Merge pull request #693 from puppetlabs/release_prep_3-8-0

Added a action to generate release notes

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10

.github/workflows/auto_release_prep.yml

@@ -0,0 +1,12 @@
+name: Automated release prep
+
+on:
+  workflow_dispatch:
+
+jobs:
+  auto_release_prep:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/auto_release_prep.yml@v1
+    secrets: inherit
+    with:
+      project-type: ruby
+      version-file-path: lib/vmpooler/version.rb

.github/workflows/dependabot_merge.yml

@@ -0,0 +1,8 @@
+name: Dependabot auto-merge
+
+on: pull_request
+
+jobs:
+  dependabot_merge:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/dependabot_merge.yml@v1
+    secrets: inherit

.github/workflows/ensure_label.yml

@@ -0,0 +1,8 @@
+name: Ensure label
+
+on: pull_request
+
+jobs:
+  ensure_label:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/ensure_label.yml@v1
+    secrets: inherit

.github/workflows/release.yml

@@ -0,0 +1,58 @@
+name: Release Gem
+
+on: workflow_dispatch
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.repository == 'puppetlabs/vmpooler'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Current Version
+        uses: actions/github-script@v7
+        id: cv
+        with:
+          script: |
+            const { data: response } = await github.rest.repos.getLatestRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            console.log(`The latest release is ${response.tag_name}`)
+            return response.tag_name
+          result-encoding: string
+
+      - name: Get Next Version
+        id: nv
+        run: |
+          version=$(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev)
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "Found version $version from lib/vmpooler/version.rb"
+
+      - name: Tag Release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.nv.outputs.version }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          bodyfile: release-notes.md
+          draft: false
+          prerelease: false
+
+      # This step should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+      - name: Install Ruby jruby-9.4.12.1
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 'jruby-9.4.12.1'
+
+      - name: Build gem
+        run: gem build *.gemspec
+
+      - name: Publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: '${{ secrets.RUBYGEMS_AUTH_TOKEN }}'

.github/workflows/security.yml

@@ -0,0 +1,39 @@
+name: Security
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  scan:
+    name: Mend Scanning
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout repo content
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - name: setup ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    # setup a package lock if one doesn't exist, otherwise do nothing
+    - name: check lock
+      run: '[ -f "Gemfile.lock" ] && echo "package lock file exists, skipping" || bundle lock'
+    # install java
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+    # download mend
+    - name: download_mend
+      run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
+    - name: run mend
+      run: java -jar wss-unified-agent.jar
+      env:
+        WS_APIKEY: ${{ secrets.MEND_API_KEY }}
+        WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
+        WS_USERKEY: ${{ secrets.MEND_TOKEN }}
+        WS_PRODUCTNAME: RE
+        WS_PROJECTNAME: ${{ github.event.repository.name }}

.github/workflows/testing.yml

@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run Rubocop
+      run: bundle exec rake rubocop
+
+  spec_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run spec tests
+      run: bundle exec rake test

.github_changelog_generator

@@ -0,0 +1,5 @@
+project=vmpooler
+user=puppetlabs
+exclude_labels=maintenance
+github-api=https://api.github.com
+release-branch=main

.gitignore

@@ -1,9 +1,9 @@
+.bundle/
 .vagrant/
-results.xml
+coverage/
+vendor/
+.dccache
 .ruby-version
-Gemfile.lock
 Gemfile.local
-vendor
+results.xml
 /vmpooler.yaml
-.bundle
-coverage

.rubocop.yml

@@ -17,7 +17,7 @@ Style/Documentation:
   Enabled: false
 
 # Line length is not useful
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: false
 
 # Empty method definitions over more than one line is ok
@@ -51,6 +51,10 @@ Metrics/ModuleLength:
 Style/WordArray:
   Enabled: false
 
+# RedundantBegin is causing lib/pool_manager & vsphere.rb to fail in Ruby 2.5+
+Style/RedundantBegin:
+  Enabled: false
+
 # Either sytnax for regex is ok
 Style/RegexpLiteral:
   Enabled: false
@@ -58,10 +62,11 @@ Style/RegexpLiteral:
 # In some cases readability is better without these cops enabled
 Style/ConditionalAssignment:
   Enabled: false
-Next:
+Style/Next:
   Enabled: false
 Metrics/ParameterLists:
   Max: 10
+  MaxOptionalParameters: 10
 Style/GuardClause:
   Enabled: false
 
@@ -69,3 +74,28 @@ Style/GuardClause:
 Layout/EndOfLine:
   EnforcedStyle: lf
 
+# Added in 0.80, don't really care about the change
+Style/HashEachMethods:
+  Enabled: false
+
+# Added in 0.80, don't really care about the change
+Style/HashTransformKeys:
+  Enabled: false
+
+# Added in 0.80, don't really care about the change
+Style/HashTransformValues:
+  Enabled: false
+
+# These short variable names make sense as exceptions to the rule, but generally I think short variable names do hurt readability
+Naming/MethodParameterName:
+  AllowedNames:
+    - vm
+    - dc
+    - s
+    - x
+    - f
+
+# Standard comparisons seem more readable
+Style/NumericPredicate:
+  Enabled: true
+  EnforcedStyle: comparison

.rubocop_todo.yml

@@ -10,9 +10,9 @@
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
 # SupportedStyles: with_first_parameter, with_fixed_indentation
-Layout/AlignParameters:
+Layout/ParameterAlignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 9
 # Cop supports --auto-correct.
@@ -21,13 +21,13 @@ Layout/AlignParameters:
 Layout/CaseIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 Layout/ClosingParenthesisIndentation:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -55,17 +55,17 @@ Layout/EmptyLinesAroundModuleBody:
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
 # SupportedStyles: special_inside_parentheses, consistent, align_braces
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: Width, IgnoredPatterns.
 Layout/IndentationWidth:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -73,7 +73,7 @@ Layout/IndentationWidth:
 # SupportedStyles: symmetrical, new_line, same_line
 Layout/MultilineMethodCallBraceLayout:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -87,20 +87,14 @@ Layout/SpaceAroundEqualsInParameterDefault:
 # Cop supports --auto-correct.
 Layout/SpaceAroundKeyword:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowForAlignment.
 Layout/SpaceAroundOperators:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-Layout/SpaceInsideBrackets:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 8
 # Cop supports --auto-correct.
@@ -115,17 +109,17 @@ Layout/SpaceInsideHashLiteralBraces:
 # Cop supports --auto-correct.
 Layout/SpaceInsideParens:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Configuration parameters: AllowSafeAssignment.
 Lint/AssignmentInCondition:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
-Lint/HandleExceptions:
+Lint/SuppressedException:
   Exclude:
     - 'lib/vmpooler/api/dashboard.rb'
 
@@ -147,12 +141,6 @@ Lint/UselessAssignment:
     - 'lib/vmpooler/api/dashboard.rb'
     - 'lib/vmpooler/api/helpers.rb'
 
-# Offense count: 2
-# Cop supports --auto-correct.
-Performance/RedundantMatch:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
-
 # Offense count: 6
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -160,15 +148,7 @@ Performance/RedundantMatch:
 Style/AndOr:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: braces, no_braces, context_dependent
-Style/BracesAroundHashParameters:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 Style/CaseEquality:
@@ -189,7 +169,7 @@ Style/For:
 Style/HashSyntax:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
@@ -197,7 +177,7 @@ Style/HashSyntax:
 Style/IfUnlessModifier:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
@@ -205,13 +185,13 @@ Style/IfUnlessModifier:
 # SupportedStyles: both, prefix, postfix
 Style/NegatedIf:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
 Style/Not:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -220,33 +200,33 @@ Style/Not:
 Style/NumericPredicate:
   Exclude:
     - 'spec/**/*'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/ParallelAssignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowSafeAssignment.
 Style/ParenthesesAroundCondition:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/PerlBackrefs:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist.
 # NamePrefix: is_, has_, have_
 # NamePrefixBlacklist: is_, has_, have_
 # NameWhitelist: is_a?
-Style/PredicateName:
+Naming/PredicateName:
   Exclude:
     - 'spec/**/*'
     - 'lib/vmpooler/api/helpers.rb'
@@ -255,7 +235,7 @@ Style/PredicateName:
 # Cop supports --auto-correct.
 Style/RedundantParentheses:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -276,7 +256,7 @@ Style/RedundantSelf:
 # SupportedStyles: single_quotes, double_quotes
 Style/StringLiterals:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -289,9 +269,9 @@ Style/TernaryParentheses:
 # Offense count: 2
 # Configuration parameters: EnforcedStyle, SupportedStyles.
 # SupportedStyles: snake_case, camelCase
-Style/VariableName:
+Naming/VariableName:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.

.travis.yml

@@ -1,39 +0,0 @@
-cache: bundler
-sudo: false
-language: ruby
-services:
-  - redis-server
-
-matrix:
-  include:
-    - rvm: 2.3.7
-      env: "CHECK=rubocop"
-
-    - rvm: 2.3.7
-      env: "CHECK=test"
-
-    - rvm: 2.4.4
-      env: "CHECK=test"
-
-    - rvm: 2.5.1
-      env: "CHECK=test"
-
-    - rvm: jruby-9.1.17.0
-      env: "CHECK=test"
-
-    - rvm: jruby-9.2.0.0
-      env: "CHECK=test"
-
-  # Remove the allow_failures section once
-  # Rubocop is required for Travis to pass a build
-  allow_failures:
-    - rvm: 2.3.7
-      env: "CHECK=rubocop"
-
-install:
-  - gem update --system
-  - gem install bundler
-  - bundle --version
-  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
-script:
-  - "bundle exec rake $CHECK"

CHANGELOG.md

@@ -1,64 +1,927 @@
-# Change Log
+# Changelog
 
-All notable changes to this project will be documented in this file.
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
 
-The format is based on
-[Keep a Changelog](http://keepachangelog.com)
-& makes a strong effort to adhere to
-[Semantic Versioning](http://semver.org).
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
 
-Tracking in this Changelog began for this project with the tagging of version 0.1.0.
-If you're looking for changes from before this, refer to the project's
-git logs & PR history.
+**Implemented enhancements:**
 
-# [Unreleased](https://github.com/puppetlabs/vmpooler/compare/0.2.1...master)
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-# [0.2.1](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+**Fixed bugs:**
 
-### Fixed
-- Better handle delta disk creation errors (POOLER-130)
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-### Added
-- Re-write check\_pool in pool\_manager to improve readability
-- Add a docker-compose file for testing vmpooler
-- Add capability to weight backends when an alias spans multiple backends (POOLER-129)
+## [3.7.0](https://github.com/puppetlabs/vmpooler/tree/3.7.0) (2025-06-04)
 
-# [0.2.0](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.6.0...3.7.0)
 
-### Fixed
-- (POOLER-128) VM specific mutex objects are not dereferenced when a VM is destroyed
-- A VM that is being destroyed is reported as discovered
+**Implemented enhancements:**
 
-### Added
-- Adds a new mechanism to load providers from any gem or file path
+- \(P4DEVOPS-6096\) Include VMs that have been requested but not moved to pending when getting queue metrics [\#681](https://github.com/puppetlabs/vmpooler/pull/681) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump redis from 5.1.0 to 5.2.0 [\#675](https://github.com/puppetlabs/vmpooler/pull/675) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.1.0 to 13.2.1 [\#673](https://github.com/puppetlabs/vmpooler/pull/673) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.8 to 5.1.0 [\#665](https://github.com/puppetlabs/vmpooler/pull/665) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rspec from 3.12.0 to 3.13.0 [\#664](https://github.com/puppetlabs/vmpooler/pull/664) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.1 to 1.4.0 [\#663](https://github.com/puppetlabs/vmpooler/pull/663) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.43.0 to 0.44.0 [\#662](https://github.com/puppetlabs/vmpooler/pull/662) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.41.0 to 0.43.0 [\#658](https://github.com/puppetlabs/vmpooler/pull/658) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.18.0 to 0.19.0 [\#653](https://github.com/puppetlabs/vmpooler/pull/653) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump sinatra from 3.1.0 to 3.2.0 [\#652](https://github.com/puppetlabs/vmpooler/pull/652) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.40.0 to 0.41.0 [\#650](https://github.com/puppetlabs/vmpooler/pull/650) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.37.0 to 0.40.0 [\#643](https://github.com/puppetlabs/vmpooler/pull/643) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.0.6 to 13.1.0 [\#638](https://github.com/puppetlabs/vmpooler/pull/638) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.2 to 1.3.0 [\#635](https://github.com/puppetlabs/vmpooler/pull/635) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-# [0.1.0](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...master)
+**Fixed bugs:**
 
-### Fixed
-- Remove unused method `find_pool` and related pending tests
-- Setting `max_tries` results in an infinite loop (POOLER-124)
-- Do not evaluate folders as VMs in `get_pool_vms` (POOLER-40)
-- Expire redis VM key when clone fails (POOLER-31)
-- Remove all usage of propertyCollector
-- Replace `find_vm` search mechanism (POOLER-68)
-- Fix configuration file loading (POOLER-103)
-- Update vulnerable dependencies (POOLER-101)
+- Bump opentelemetry-sdk from 1.4.0 to 1.4.1 [\#672](https://github.com/puppetlabs/vmpooler/pull/672) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8.1 to 2.2.9 [\#671](https://github.com/puppetlabs/vmpooler/pull/671) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.3.0 to 1.3.1 [\#668](https://github.com/puppetlabs/vmpooler/pull/668) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8 to 2.2.8.1 [\#666](https://github.com/puppetlabs/vmpooler/pull/666) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump concurrent-ruby from 1.2.2 to 1.2.3 [\#660](https://github.com/puppetlabs/vmpooler/pull/660) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.1 to 6.4.2 [\#655](https://github.com/puppetlabs/vmpooler/pull/655) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.0 to 6.4.1 [\#654](https://github.com/puppetlabs/vmpooler/pull/654) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.22.2 to = 0.22.3 [\#646](https://github.com/puppetlabs/vmpooler/pull/646) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-concurrent\_ruby requirement from = 0.21.1 to = 0.21.2 [\#645](https://github.com/puppetlabs/vmpooler/pull/645) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.0 to 1.3.1 [\#642](https://github.com/puppetlabs/vmpooler/pull/642) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump prometheus-client from 4.2.1 to 4.2.2 [\#641](https://github.com/puppetlabs/vmpooler/pull/641) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.7 to 5.0.8 [\#637](https://github.com/puppetlabs/vmpooler/pull/637) ([dependabot[bot]](https://github.com/apps/dependabot))
+- \(RE-15817\) Reword fail warning and get error from redis before generating message [\#633](https://github.com/puppetlabs/vmpooler/pull/633) ([isaac-hammes](https://github.com/isaac-hammes))
 
-### Added
+**Merged pull requests:**
 
-- Allow API and manager to run separately (POOLER-109)
-- Add configuration API endpoint (POOLER-107)
-- Add option to disable VM hostname mismatch checks
-- Add a gemspec file
-- Add time remaining information (POOLER-81)
-- Ship metrics for clone to ready time (POOLER-34)
-- Reduce duplicate checking of VMs
-- Reduce object lookups when retrieving VMs and folders
-- Optionally create delta disks for pool templates
-- Drop support for any ruby before 2.3
-- Add support for multiple LDAP search base DNs (POOLER-113)
-- Ensure a VM is only destroyed once (POOLER-112)
-- Add support for setting redis server port and password
-- Greatly reduce time it takes to add disks
-- Add Dockerfile that does not bundle redis
-- Add vmpooler.service to support systemd managing the service
+- \(P4DEVOPS-6096\) Fix gems to prevent warnings in logs [\#685](https://github.com/puppetlabs/vmpooler/pull/685) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(maint\) Revert gems to last release [\#683](https://github.com/puppetlabs/vmpooler/pull/683) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump actions/setup-java from 3 to 4 [\#648](https://github.com/puppetlabs/vmpooler/pull/648) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/github-script from 6 to 7 [\#644](https://github.com/puppetlabs/vmpooler/pull/644) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.6.0](https://github.com/puppetlabs/vmpooler/tree/3.6.0) (2023-10-05)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.1...3.6.0)
+
+**Fixed bugs:**
+
+- \(maint\) Fix message for timeout notification. [\#624](https://github.com/puppetlabs/vmpooler/pull/624) ([isaac-hammes](https://github.com/isaac-hammes))
+
+**Merged pull requests:**
+
+- Bump rubocop from 1.56.3 to 1.56.4 [\#631](https://github.com/puppetlabs/vmpooler/pull/631) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.3.1 to 6.4.0 [\#630](https://github.com/puppetlabs/vmpooler/pull/630) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.2 to 1.56.3 [\#628](https://github.com/puppetlabs/vmpooler/pull/628) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/checkout from 3 to 4 [\#627](https://github.com/puppetlabs/vmpooler/pull/627) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.24.1 to = 0.24.2 [\#626](https://github.com/puppetlabs/vmpooler/pull/626) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.1 to 1.56.2 [\#625](https://github.com/puppetlabs/vmpooler/pull/625) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.0 to 1.56.1 [\#623](https://github.com/puppetlabs/vmpooler/pull/623) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.5.1](https://github.com/puppetlabs/vmpooler/tree/3.5.1) (2023-08-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.0...3.5.1)
+
+**Fixed bugs:**
+
+- \(maint\) Fix bugs from redis and timeout notification updates. [\#621](https://github.com/puppetlabs/vmpooler/pull/621) ([isaac-hammes](https://github.com/isaac-hammes))
+
+## [3.5.0](https://github.com/puppetlabs/vmpooler/tree/3.5.0) (2023-08-23)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.4.0...3.5.0)
+
+**Implemented enhancements:**
+
+- Improve LDAP auth [\#616](https://github.com/puppetlabs/vmpooler/issues/616)
+- \(maint\) Raise error when ip address is not given to vm after clone. [\#619](https://github.com/puppetlabs/vmpooler/pull/619) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(POD-8\) Add timeout\_notification config to log warning before vm is destroyed. [\#618](https://github.com/puppetlabs/vmpooler/pull/618) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(RE-15565\) Add ability to use bind\_as with a service account [\#617](https://github.com/puppetlabs/vmpooler/pull/617) ([yachub](https://github.com/yachub))
+
+**Merged pull requests:**
+
+- Bump puma from 6.3.0 to 6.3.1 [\#615](https://github.com/puppetlabs/vmpooler/pull/615) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.4.0](https://github.com/puppetlabs/vmpooler/tree/3.4.0) (2023-08-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.3.0...3.4.0)
+
+**Implemented enhancements:**
+
+- \(POD-10\) Log reason for failed VM checks. [\#611](https://github.com/puppetlabs/vmpooler/pull/611) ([isaac-hammes](https://github.com/isaac-hammes))
+
+**Closed issues:**
+
+- Log reason connection on port 22 of a failed VM [\#609](https://github.com/puppetlabs/vmpooler/issues/609)
+
+## [3.3.0](https://github.com/puppetlabs/vmpooler/tree/3.3.0) (2023-08-16)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.2.0...3.3.0)
+
+**Closed issues:**
+
+- Redis 5.x Deprecations [\#603](https://github.com/puppetlabs/vmpooler/issues/603)
+
+**Merged pull requests:**
+
+- Update rubocop requirement from ~\> 1.55.1 to ~\> 1.56.0 [\#608](https://github.com/puppetlabs/vmpooler/pull/608) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.2.0](https://github.com/puppetlabs/vmpooler/tree/3.2.0) (2023-08-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.1.0...3.2.0)
+
+**Implemented enhancements:**
+
+- \(maint\) Update opentelemetry gems. [\#606](https://github.com/puppetlabs/vmpooler/pull/606) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump jruby to 9.4.3.0 and bundle update [\#604](https://github.com/puppetlabs/vmpooler/pull/604) ([yachub](https://github.com/yachub))
+
+**Fixed bugs:**
+
+- \(RE-15692\) Do not attempt loading DNS classes if none are defined [\#602](https://github.com/puppetlabs/vmpooler/pull/602) ([yachub](https://github.com/yachub))
+
+**Closed issues:**
+
+- Fix startup error when not using any dns plugins [\#601](https://github.com/puppetlabs/vmpooler/issues/601)
+
+**Merged pull requests:**
+
+- Bump prometheus-client from 4.1.0 to 4.2.1 [\#599](https://github.com/puppetlabs/vmpooler/pull/599) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.54.2 to ~\> 1.55.1 [\#597](https://github.com/puppetlabs/vmpooler/pull/597) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.7 to 2.2.8 [\#594](https://github.com/puppetlabs/vmpooler/pull/594) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.51.0 to ~\> 1.54.2 [\#593](https://github.com/puppetlabs/vmpooler/pull/593) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.2.2 to 6.3.0 [\#586](https://github.com/puppetlabs/vmpooler/pull/586) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump connection\_pool from 2.4.0 to 2.4.1 [\#583](https://github.com/puppetlabs/vmpooler/pull/583) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.50.1 to ~\> 1.51.0 [\#582](https://github.com/puppetlabs/vmpooler/pull/582) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.1 to 1.2.2 [\#581](https://github.com/puppetlabs/vmpooler/pull/581) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.6.4 to 2.2.7 [\#579](https://github.com/puppetlabs/vmpooler/pull/579) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.1.0](https://github.com/puppetlabs/vmpooler/tree/3.1.0) (2023-05-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.0.0...3.1.0)
+
+**Merged pull requests:**
+
+- Bump rubocop from 1.50.1 to 1.50.2 [\#578](https://github.com/puppetlabs/vmpooler/pull/578) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update puma requirement from ~\> 5.0, \>= 5.0.4 to \>= 5.0.4, \< 7 [\#577](https://github.com/puppetlabs/vmpooler/pull/577) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.19.1 to = 0.23.0 [\#576](https://github.com/puppetlabs/vmpooler/pull/576) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Migrate issue management to Jira [\#575](https://github.com/puppetlabs/vmpooler/pull/575) ([yachub](https://github.com/yachub))
+- Bump jruby to 9.4.2.0 [\#574](https://github.com/puppetlabs/vmpooler/pull/574) ([yachub](https://github.com/yachub))
+- Update rubocop requirement from ~\> 1.28.2 to ~\> 1.50.1 [\#573](https://github.com/puppetlabs/vmpooler/pull/573) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update sinatra requirement from ~\> 2.0 to \>= 2, \< 4 [\#572](https://github.com/puppetlabs/vmpooler/pull/572) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.17.1 to 0.18.0 [\#571](https://github.com/puppetlabs/vmpooler/pull/571) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update prometheus-client requirement from ~\> 2.0 to \>= 2, \< 5 [\#566](https://github.com/puppetlabs/vmpooler/pull/566) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack-test from 2.0.2 to 2.1.0 [\#564](https://github.com/puppetlabs/vmpooler/pull/564) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rack requirement from ~\> 2.2 to \>= 2.2, \< 4.0 [\#562](https://github.com/puppetlabs/vmpooler/pull/562) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-exporter-jaeger requirement from = 0.20.1 to = 0.22.0 [\#524](https://github.com/puppetlabs/vmpooler/pull/524) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.0.0](https://github.com/puppetlabs/vmpooler/tree/3.0.0) (2023-03-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.5.0...3.0.0)
+
+**Breaking changes:**
+
+- Direct Users to vmpooler-deployment [\#568](https://github.com/puppetlabs/vmpooler/pull/568) ([yachub](https://github.com/yachub))
+- \(RE-15124\) Implement DNS Plugins and Remove api v1 and v2 [\#551](https://github.com/puppetlabs/vmpooler/pull/551) ([yachub](https://github.com/yachub))
+
+## [2.5.0](https://github.com/puppetlabs/vmpooler/tree/2.5.0) (2023-03-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.4.0...2.5.0)
+
+**Implemented enhancements:**
+
+- \(RE-15161\) Use timeout builtin to TCPSocket when opening sockets. [\#555](https://github.com/puppetlabs/vmpooler/pull/555) ([isaac-hammes](https://github.com/isaac-hammes))
+
+**Merged pull requests:**
+
+- Add docs and update actions [\#550](https://github.com/puppetlabs/vmpooler/pull/550) ([yachub](https://github.com/yachub))
+- \(RE-15111\) Migrate Snyk to Mend Scanning [\#546](https://github.com/puppetlabs/vmpooler/pull/546) ([yachub](https://github.com/yachub))
+- \(RE-14811\) Remove DIO as codeowners [\#517](https://github.com/puppetlabs/vmpooler/pull/517) ([yachub](https://github.com/yachub))
+- Add Snyk action and Move to RE org [\#511](https://github.com/puppetlabs/vmpooler/pull/511) ([yachub](https://github.com/yachub))
+- Add release-engineering to codeowners [\#508](https://github.com/puppetlabs/vmpooler/pull/508) ([yachub](https://github.com/yachub))
+- Update docker/Gemfile.lock [\#503](https://github.com/puppetlabs/vmpooler/pull/503) ([yachub](https://github.com/yachub))
+
+## [2.4.0](https://github.com/puppetlabs/vmpooler/tree/2.4.0) (2022-07-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.3.0...2.4.0)
+
+**Merged pull requests:**
+
+- \(maint\) Bump version to 2.4.0 [\#502](https://github.com/puppetlabs/vmpooler/pull/502) ([sbeaulie](https://github.com/sbeaulie))
+- \(bug\) Prevent failing VMs to be retried infinitely \(ondemand\) [\#501](https://github.com/puppetlabs/vmpooler/pull/501) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-3138\) vmpooler v2 api missing vm/hostname [\#500](https://github.com/puppetlabs/vmpooler/pull/500) ([sbeaulie](https://github.com/sbeaulie))
+- Update rubocop requirement from ~\> 1.1.0 to ~\> 1.28.2 [\#499](https://github.com/puppetlabs/vmpooler/pull/499) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.30.0 to 0.31.0 [\#496](https://github.com/puppetlabs/vmpooler/pull/496) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-redis requirement from = 0.21.2 to = 0.21.3 [\#494](https://github.com/puppetlabs/vmpooler/pull/494) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 5.5.2 to 5.6.4 [\#490](https://github.com/puppetlabs/vmpooler/pull/490) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.19.3 to = 0.19.4 [\#478](https://github.com/puppetlabs/vmpooler/pull/478) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [2.3.0](https://github.com/puppetlabs/vmpooler/tree/2.3.0) (2022-04-07)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.2.0...2.3.0)
+
+**Merged pull requests:**
+
+- \(maint\) Fix deprecation warning for redis ruby library [\#489](https://github.com/puppetlabs/vmpooler/pull/489) ([sbeaulie](https://github.com/sbeaulie))
+- Add OTel HttpClient Instrumentation [\#477](https://github.com/puppetlabs/vmpooler/pull/477) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Update dev tooling and related docs [\#476](https://github.com/puppetlabs/vmpooler/pull/476) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Connect domain settings to pools, create v2 API [\#475](https://github.com/puppetlabs/vmpooler/pull/475) ([genebean](https://github.com/genebean))
+
+## [2.2.0](https://github.com/puppetlabs/vmpooler/tree/2.2.0) (2021-12-30)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.1.0...2.2.0)
+
+**Merged pull requests:**
+
+- Bump version to 2.2.0 [\#473](https://github.com/puppetlabs/vmpooler/pull/473) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Fix EXTRA\_CONFIG merge behavior [\#472](https://github.com/puppetlabs/vmpooler/pull/472) ([sbeaulie](https://github.com/sbeaulie))
+- Update to latest OTel gems [\#471](https://github.com/puppetlabs/vmpooler/pull/471) ([genebean](https://github.com/genebean))
+- Add additional data to spans in api/v1.rb [\#400](https://github.com/puppetlabs/vmpooler/pull/400) ([genebean](https://github.com/genebean))
+
+## [2.1.0](https://github.com/puppetlabs/vmpooler/tree/2.1.0) (2021-12-13)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.0.0...2.1.0)
+
+**Merged pull requests:**
+
+- Ensure all configured providers are loaded [\#470](https://github.com/puppetlabs/vmpooler/pull/470) ([genebean](https://github.com/genebean))
+- \(maint\) Adding a provider method tag\_vm\_user [\#469](https://github.com/puppetlabs/vmpooler/pull/469) ([sbeaulie](https://github.com/sbeaulie))
+- Update testing.yml [\#468](https://github.com/puppetlabs/vmpooler/pull/468) ([sbeaulie](https://github.com/sbeaulie))
+- Move vsphere specific methods out of vmpooler [\#467](https://github.com/puppetlabs/vmpooler/pull/467) ([sbeaulie](https://github.com/sbeaulie))
+- Release prep for v2.0.0 [\#465](https://github.com/puppetlabs/vmpooler/pull/465) ([genebean](https://github.com/genebean))
+
+## [2.0.0](https://github.com/puppetlabs/vmpooler/tree/2.0.0) (2021-12-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.3.0...2.0.0)
+
+**Merged pull requests:**
+
+- Use credentials file for Rubygems auth [\#466](https://github.com/puppetlabs/vmpooler/pull/466) ([genebean](https://github.com/genebean))
+- Add Gem release workflow [\#464](https://github.com/puppetlabs/vmpooler/pull/464) ([genebean](https://github.com/genebean))
+- Update icon in the readme to reference this repo [\#463](https://github.com/puppetlabs/vmpooler/pull/463) ([genebean](https://github.com/genebean))
+- \(DIO-2769\) Move vsphere provider to its own gem [\#462](https://github.com/puppetlabs/vmpooler/pull/462) ([genebean](https://github.com/genebean))
+
+## [1.3.0](https://github.com/puppetlabs/vmpooler/tree/1.3.0) (2021-11-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.2.0...1.3.0)
+
+**Merged pull requests:**
+
+- \(DIO-2675\) Undo pool size & template overrides [\#461](https://github.com/puppetlabs/vmpooler/pull/461) ([genebean](https://github.com/genebean))
+- \(DIO-2186\) Token migration [\#460](https://github.com/puppetlabs/vmpooler/pull/460) ([genebean](https://github.com/genebean))
+
+## [1.2.0](https://github.com/puppetlabs/vmpooler/tree/1.2.0) (2021-09-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.2...1.2.0)
+
+**Merged pull requests:**
+
+- \(DIO-2621\) Make LDAP encryption configurable [\#459](https://github.com/puppetlabs/vmpooler/pull/459) ([genebean](https://github.com/genebean))
+
+## [1.1.2](https://github.com/puppetlabs/vmpooler/tree/1.1.2) (2021-08-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.1...1.1.2)
+
+**Merged pull requests:**
+
+- \(DIO-541\) Fix jenkins and user usage metrics [\#458](https://github.com/puppetlabs/vmpooler/pull/458) ([yachub](https://github.com/yachub))
+
+## [1.1.1](https://github.com/puppetlabs/vmpooler/tree/1.1.1) (2021-08-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0...1.1.1)
+
+**Merged pull requests:**
+
+- \(POOLER-198\) Fix otel warning: Bump otel gems to 0.17.0 [\#457](https://github.com/puppetlabs/vmpooler/pull/457) ([yachub](https://github.com/yachub))
+
+## [1.1.0](https://github.com/puppetlabs/vmpooler/tree/1.1.0) (2021-08-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0-rc.1...1.1.0)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#455](https://github.com/puppetlabs/vmpooler/pull/455) ([yachub](https://github.com/yachub))
+
+## [1.1.0-rc.1](https://github.com/puppetlabs/vmpooler/tree/1.1.0-rc.1) (2021-08-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.0.0...1.1.0-rc.1)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#454](https://github.com/puppetlabs/vmpooler/pull/454) ([yachub](https://github.com/yachub))
+- Update OTel gems to 0.15.0 [\#450](https://github.com/puppetlabs/vmpooler/pull/450) ([genebean](https://github.com/genebean))
+- Migrate testing to GH Actions from Travis [\#446](https://github.com/puppetlabs/vmpooler/pull/446) ([genebean](https://github.com/genebean))
+
+## [1.0.0](https://github.com/puppetlabs/vmpooler/tree/1.0.0) (2021-02-02)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.2...1.0.0)
+
+**Merged pull requests:**
+
+- Update OTel gems to 0.13.z [\#447](https://github.com/puppetlabs/vmpooler/pull/447) ([genebean](https://github.com/genebean))
+- \(DIO-1503\) Fix regex for ondemand instances [\#445](https://github.com/puppetlabs/vmpooler/pull/445) ([genebean](https://github.com/genebean))
+- \(maint\) Update lightstep pre-deploy ghaction to v0.2.6 [\#440](https://github.com/puppetlabs/vmpooler/pull/440) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.2](https://github.com/puppetlabs/vmpooler/tree/0.18.2) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.1...0.18.2)
+
+**Merged pull requests:**
+
+- Remove usage of redis multi from api [\#438](https://github.com/puppetlabs/vmpooler/pull/438) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Fix checkout counter allocation [\#437](https://github.com/puppetlabs/vmpooler/pull/437) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.18.1](https://github.com/puppetlabs/vmpooler/tree/0.18.1) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.0...0.18.1)
+
+**Merged pull requests:**
+
+- Update Puma to 5.0.4 from ~4.3 [\#436](https://github.com/puppetlabs/vmpooler/pull/436) ([genebean](https://github.com/genebean))
+- \(MAINT\) Fix checkout counter allocation [\#435](https://github.com/puppetlabs/vmpooler/pull/435) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-193\) Mark checked out VM as active [\#434](https://github.com/puppetlabs/vmpooler/pull/434) ([mattkirby](https://github.com/mattkirby))
+- Update to OTel 0.8.0 [\#432](https://github.com/puppetlabs/vmpooler/pull/432) ([genebean](https://github.com/genebean))
+- \(POOLER-192\) Use Rubocop 1.0 [\#423](https://github.com/puppetlabs/vmpooler/pull/423) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.0](https://github.com/puppetlabs/vmpooler/tree/0.18.0) (2020-10-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.17.0...0.18.0)
+
+**Merged pull requests:**
+
+- \(maint\) Speedup the tagging method [\#422](https://github.com/puppetlabs/vmpooler/pull/422) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-1065\) Add lightstep gh action [\#421](https://github.com/puppetlabs/vmpooler/pull/421) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.17.0](https://github.com/puppetlabs/vmpooler/tree/0.17.0) (2020-10-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.3...0.17.0)
+
+**Merged pull requests:**
+
+- \(DIO-1059\) Optionally add snapshot tuning params at clone time [\#419](https://github.com/puppetlabs/vmpooler/pull/419) ([suckatrash](https://github.com/suckatrash))
+
+## [0.16.3](https://github.com/puppetlabs/vmpooler/tree/0.16.3) (2020-10-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.2...0.16.3)
+
+**Merged pull requests:**
+
+- \(POOLER-191\) Add checking for running instances that are not in active [\#418](https://github.com/puppetlabs/vmpooler/pull/418) ([mattkirby](https://github.com/mattkirby))
+
+## [0.16.2](https://github.com/puppetlabs/vmpooler/tree/0.16.2) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.1...0.16.2)
+
+**Merged pull requests:**
+
+- Bump OTel Sinatra to 0.7.1 [\#417](https://github.com/puppetlabs/vmpooler/pull/417) ([genebean](https://github.com/genebean))
+
+## [0.16.1](https://github.com/puppetlabs/vmpooler/tree/0.16.1) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.0...0.16.1)
+
+## [0.16.0](https://github.com/puppetlabs/vmpooler/tree/0.16.0) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.15.0...0.16.0)
+
+**Merged pull requests:**
+
+- Update to OTel 0.7.0 [\#416](https://github.com/puppetlabs/vmpooler/pull/416) ([genebean](https://github.com/genebean))
+
+## [0.15.0](https://github.com/puppetlabs/vmpooler/tree/0.15.0) (2020-09-30)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.9...0.15.0)
+
+**Merged pull requests:**
+
+- \(maint\) Centralize dependency management in the gemspec [\#407](https://github.com/puppetlabs/vmpooler/pull/407) ([sbeaulie](https://github.com/sbeaulie))
+- \(pooler-180\) Add healthcheck endpoint, spec testing [\#406](https://github.com/puppetlabs/vmpooler/pull/406) ([suckatrash](https://github.com/suckatrash))
+
+## [0.14.9](https://github.com/puppetlabs/vmpooler/tree/0.14.9) (2020-09-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.8...0.14.9)
+
+**Merged pull requests:**
+
+- Adding make to the other two Dockerfiles [\#405](https://github.com/puppetlabs/vmpooler/pull/405) ([genebean](https://github.com/genebean))
+
+## [0.14.8](https://github.com/puppetlabs/vmpooler/tree/0.14.8) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.7...0.14.8)
+
+**Merged pull requests:**
+
+- Fix mixup of gem placement. [\#404](https://github.com/puppetlabs/vmpooler/pull/404) ([genebean](https://github.com/genebean))
+
+## [0.14.7](https://github.com/puppetlabs/vmpooler/tree/0.14.7) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.6...0.14.7)
+
+**Merged pull requests:**
+
+- Add OTel resource detectors [\#401](https://github.com/puppetlabs/vmpooler/pull/401) ([genebean](https://github.com/genebean))
+- Add distributed tracing [\#399](https://github.com/puppetlabs/vmpooler/pull/399) ([genebean](https://github.com/genebean))
+
+## [0.14.6](https://github.com/puppetlabs/vmpooler/tree/0.14.6) (2020-09-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.5...0.14.6)
+
+**Merged pull requests:**
+
+- \(POOLER-184\) Pool manager retry and exit on failure [\#398](https://github.com/puppetlabs/vmpooler/pull/398) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Add promstats component check [\#397](https://github.com/puppetlabs/vmpooler/pull/397) ([rooneyshuman](https://github.com/rooneyshuman))
+- Test vmpooler on latest 2.5 [\#393](https://github.com/puppetlabs/vmpooler/pull/393) ([mattkirby](https://github.com/mattkirby))
+- Update rbvmomi requirement from ~\> 2.1 to \>= 2.1, \< 4.0 [\#391](https://github.com/puppetlabs/vmpooler/pull/391) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [0.14.5](https://github.com/puppetlabs/vmpooler/tree/0.14.5) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.4...0.14.5)
+
+**Merged pull requests:**
+
+- \(MAINT\) Fix Staledns error counter [\#396](https://github.com/puppetlabs/vmpooler/pull/396) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.4](https://github.com/puppetlabs/vmpooler/tree/0.14.4) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.3...0.14.4)
+
+**Merged pull requests:**
+
+- \(MAINT\) Normalise all tokens for stats [\#395](https://github.com/puppetlabs/vmpooler/pull/395) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.3](https://github.com/puppetlabs/vmpooler/tree/0.14.3) (2020-08-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.2...0.14.3)
+
+**Merged pull requests:**
+
+- \(POOLER-186\) Fix template alias evaluation with backend weight of 0 [\#394](https://github.com/puppetlabs/vmpooler/pull/394) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Clarity refactor of Prom Stats code [\#390](https://github.com/puppetlabs/vmpooler/pull/390) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.2](https://github.com/puppetlabs/vmpooler/tree/0.14.2) (2020-08-03)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.1...0.14.2)
+
+**Merged pull requests:**
+
+- Ensure lifetime is set when creating ondemand instances [\#392](https://github.com/puppetlabs/vmpooler/pull/392) ([mattkirby](https://github.com/mattkirby))
+- Fix vmpooler folder purging [\#389](https://github.com/puppetlabs/vmpooler/pull/389) ([mattkirby](https://github.com/mattkirby))
+
+## [0.14.1](https://github.com/puppetlabs/vmpooler/tree/0.14.1) (2020-07-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.0...0.14.1)
+
+**Merged pull requests:**
+
+- Correctly handle multiple pools of same alias in ondemand checkout [\#388](https://github.com/puppetlabs/vmpooler/pull/388) ([mattkirby](https://github.com/mattkirby))
+- Update travis config to remove deprecated style [\#387](https://github.com/puppetlabs/vmpooler/pull/387) ([rooneyshuman](https://github.com/rooneyshuman))
+- Update Dependabot config file [\#386](https://github.com/puppetlabs/vmpooler/pull/386) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.14.0](https://github.com/puppetlabs/vmpooler/tree/0.14.0) (2020-07-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.3...0.14.0)
+
+**Merged pull requests:**
+
+- Add a note on jruby 9.2.11 and redis connection pooling changes [\#384](https://github.com/puppetlabs/vmpooler/pull/384) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-167\) Allow for network configuration at vm clone time [\#382](https://github.com/puppetlabs/vmpooler/pull/382) ([rooneyshuman](https://github.com/rooneyshuman))
+- \(POOLER-160\) Add Prometheus Metrics to vmpooler [\#372](https://github.com/puppetlabs/vmpooler/pull/372) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.13.3](https://github.com/puppetlabs/vmpooler/tree/0.13.3) (2020-06-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.2...0.13.3)
+
+**Merged pull requests:**
+
+- \(POOLER-174\) Reduce duplicate of on demand code introduced in POOLER-158 [\#383](https://github.com/puppetlabs/vmpooler/pull/383) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.2](https://github.com/puppetlabs/vmpooler/tree/0.13.2) (2020-06-05)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.1...0.13.2)
+
+**Merged pull requests:**
+
+- Rescue and warn when graphite connection cannot be opened [\#379](https://github.com/puppetlabs/vmpooler/pull/379) ([mattkirby](https://github.com/mattkirby))
+
+## [0.13.1](https://github.com/puppetlabs/vmpooler/tree/0.13.1) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.0...0.13.1)
+
+**Merged pull requests:**
+
+- \(maint\) Fix merge issue [\#378](https://github.com/puppetlabs/vmpooler/pull/378) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.0](https://github.com/puppetlabs/vmpooler/tree/0.13.0) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.12.0...0.13.0)
+
+**Merged pull requests:**
+
+- \(POOLER-166\) Check for stale dns records [\#377](https://github.com/puppetlabs/vmpooler/pull/377) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-158\) Add support for ondemand provisioning [\#375](https://github.com/puppetlabs/vmpooler/pull/375) ([mattkirby](https://github.com/mattkirby))
+
+## [0.12.0](https://github.com/puppetlabs/vmpooler/tree/0.12.0) (2020-05-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.3...0.12.0)
+
+**Merged pull requests:**
+
+- \(POOLER-171\) Enable support for multiple user objects [\#376](https://github.com/puppetlabs/vmpooler/pull/376) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.11.3](https://github.com/puppetlabs/vmpooler/tree/0.11.3) (2020-04-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.2...0.11.3)
+
+**Merged pull requests:**
+
+- \(DIO-608\) vmpooler SUT handed out multiple times [\#374](https://github.com/puppetlabs/vmpooler/pull/374) ([sbeaulie](https://github.com/sbeaulie))
+- \(MAINT\) Update CODEOWNERS [\#373](https://github.com/puppetlabs/vmpooler/pull/373) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.11.2](https://github.com/puppetlabs/vmpooler/tree/0.11.2) (2020-04-16)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.1...0.11.2)
+
+**Merged pull requests:**
+
+- \(POOLER-161\) Fix extending vm lifetime when max lifetime is set [\#371](https://github.com/puppetlabs/vmpooler/pull/371) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-165\) Fix purge\_unconfigured\_folders [\#370](https://github.com/puppetlabs/vmpooler/pull/370) ([mattkirby](https://github.com/mattkirby))
+- Update rake requirement from ~\> 12.3 to \>= 12.3, \< 14.0 [\#369](https://github.com/puppetlabs/vmpooler/pull/369) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.11.1](https://github.com/puppetlabs/vmpooler/tree/0.11.1) (2020-03-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.0...0.11.1)
+
+**Merged pull requests:**
+
+- Remove providers addition to docker-compose.yml [\#368](https://github.com/puppetlabs/vmpooler/pull/368) ([mattkirby](https://github.com/mattkirby))
+- Add Dependabot to keep gems updated [\#367](https://github.com/puppetlabs/vmpooler/pull/367) ([genebean](https://github.com/genebean))
+- Update gem dependencies to latest versions [\#366](https://github.com/puppetlabs/vmpooler/pull/366) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-156\) Detect redis connection failures [\#365](https://github.com/puppetlabs/vmpooler/pull/365) ([mattkirby](https://github.com/mattkirby))
+- Add a .dockerignore file [\#363](https://github.com/puppetlabs/vmpooler/pull/363) ([mattkirby](https://github.com/mattkirby))
+
+## [0.11.0](https://github.com/puppetlabs/vmpooler/tree/0.11.0) (2020-03-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.3...0.11.0)
+
+**Merged pull requests:**
+
+- Pin to JRuby 9.2.9 in Dockerfiles [\#362](https://github.com/puppetlabs/vmpooler/pull/362) ([highb](https://github.com/highb))
+- Manual Rubocop Fixes [\#361](https://github.com/puppetlabs/vmpooler/pull/361) ([highb](https://github.com/highb))
+- "Unsafe" rubocop fixes [\#360](https://github.com/puppetlabs/vmpooler/pull/360) ([highb](https://github.com/highb))
+- Fix Rubocop "safe" auto-corrections [\#359](https://github.com/puppetlabs/vmpooler/pull/359) ([highb](https://github.com/highb))
+- Remove duplicate of 0.10.2 from CHANGELOG [\#358](https://github.com/puppetlabs/vmpooler/pull/358) ([highb](https://github.com/highb))
+- \(POOLER-157\) Add extra\_config option to vmpooler [\#357](https://github.com/puppetlabs/vmpooler/pull/357) ([mattkirby](https://github.com/mattkirby))
+
+## [0.10.3](https://github.com/puppetlabs/vmpooler/tree/0.10.3) (2020-03-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.2...0.10.3)
+
+**Merged pull requests:**
+
+- Release 0.10.3 [\#356](https://github.com/puppetlabs/vmpooler/pull/356) ([highb](https://github.com/highb))
+- \(POOLER-154\) Delay vm host update until after migration completes [\#355](https://github.com/puppetlabs/vmpooler/pull/355) ([highb](https://github.com/highb))
+
+## [0.10.2](https://github.com/puppetlabs/vmpooler/tree/0.10.2) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.1...0.10.2)
+
+## [0.10.1](https://github.com/puppetlabs/vmpooler/tree/0.10.1) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.0...0.10.1)
+
+## [0.10.0](https://github.com/puppetlabs/vmpooler/tree/0.10.0) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.1...0.10.0)
+
+**Merged pull requests:**
+
+- Update changelog for 0.10.0 release [\#354](https://github.com/puppetlabs/vmpooler/pull/354) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-153\) Add endpoint for resetting a pool [\#353](https://github.com/puppetlabs/vmpooler/pull/353) ([mattkirby](https://github.com/mattkirby))
+
+## [0.9.1](https://github.com/puppetlabs/vmpooler/tree/0.9.1) (2020-01-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.0...0.9.1)
+
+**Merged pull requests:**
+
+- Generate a wider set of legal names [\#351](https://github.com/puppetlabs/vmpooler/pull/351) ([nicklewis](https://github.com/nicklewis))
+
+## [0.9.0](https://github.com/puppetlabs/vmpooler/tree/0.9.0) (2019-12-12)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.2...0.9.0)
+
+**Closed issues:**
+
+- find\_cluster in vsphere\_helper doesn't support host folders [\#205](https://github.com/puppetlabs/vmpooler/issues/205)
+
+**Merged pull requests:**
+
+- \(QENG-7531\) Add Marked as Failed Stat [\#350](https://github.com/puppetlabs/vmpooler/pull/350) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-123\) Implement a max TTL [\#349](https://github.com/puppetlabs/vmpooler/pull/349) ([sbeaulie](https://github.com/sbeaulie))
+- Support nested host folders in find\_cluster\(\) [\#348](https://github.com/puppetlabs/vmpooler/pull/348) ([seanmil](https://github.com/seanmil))
+- Update CHANGELOG for 0.8.2 [\#347](https://github.com/puppetlabs/vmpooler/pull/347) ([highb](https://github.com/highb))
+
+## [0.8.2](https://github.com/puppetlabs/vmpooler/tree/0.8.2) (2019-11-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.1...0.8.2)
+
+**Merged pull requests:**
+
+- Update rubocop configs [\#346](https://github.com/puppetlabs/vmpooler/pull/346) ([highb](https://github.com/highb))
+- \(QENG-7530\) Add check for unique hostnames [\#345](https://github.com/puppetlabs/vmpooler/pull/345) ([highb](https://github.com/highb))
+- \(QENG-7530\) Fix hostname\_shorten regex [\#344](https://github.com/puppetlabs/vmpooler/pull/344) ([highb](https://github.com/highb))
+- Update changelog for 0.8.1 release [\#343](https://github.com/puppetlabs/vmpooler/pull/343) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.1](https://github.com/puppetlabs/vmpooler/tree/0.8.1) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.0...0.8.1)
+
+**Merged pull requests:**
+
+- Add spicy-proton to vmpooler.gemspec [\#342](https://github.com/puppetlabs/vmpooler/pull/342) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.0](https://github.com/puppetlabs/vmpooler/tree/0.8.0) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.2...0.8.0)
+
+**Merged pull requests:**
+
+- \(QENG-7530\) Make VM names more human readable [\#341](https://github.com/puppetlabs/vmpooler/pull/341) ([highb](https://github.com/highb))
+
+## [0.7.2](https://github.com/puppetlabs/vmpooler/tree/0.7.2) (2019-10-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.1...0.7.2)
+
+**Merged pull requests:**
+
+- Simplify declaration of checkoutlock mutex [\#340](https://github.com/puppetlabs/vmpooler/pull/340) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-150\) Synchronize checkout operations for API [\#339](https://github.com/puppetlabs/vmpooler/pull/339) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-148\) Fix undefined variable bug in \_check\_ready\_vm. [\#338](https://github.com/puppetlabs/vmpooler/pull/338) ([quorten](https://github.com/quorten))
+- Add CODEOWNERS file to vmpooler [\#337](https://github.com/puppetlabs/vmpooler/pull/337) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.1](https://github.com/puppetlabs/vmpooler/tree/0.7.1) (2019-08-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.0...0.7.1)
+
+**Merged pull requests:**
+
+- \(POOLER-147\) Fix create\_linked\_clone pool option [\#336](https://github.com/puppetlabs/vmpooler/pull/336) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update changelog for 0.7.0 release [\#335](https://github.com/puppetlabs/vmpooler/pull/335) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.0](https://github.com/puppetlabs/vmpooler/tree/0.7.0) (2019-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.3...0.7.0)
+
+**Merged pull requests:**
+
+- \(POOLER-142\) Add running host to vm API data [\#334](https://github.com/puppetlabs/vmpooler/pull/334) ([mattkirby](https://github.com/mattkirby))
+- Make it possible to disable linked clones [\#333](https://github.com/puppetlabs/vmpooler/pull/333) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.3](https://github.com/puppetlabs/vmpooler/tree/0.6.3) (2019-07-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.2...0.6.3)
+
+**Closed issues:**
+
+- Named snapshots? [\#140](https://github.com/puppetlabs/vmpooler/issues/140)
+
+**Merged pull requests:**
+
+- \(POOLER-143\) Add clone\_target config change to API [\#332](https://github.com/puppetlabs/vmpooler/pull/332) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Update changelog for 0.6.2 [\#331](https://github.com/puppetlabs/vmpooler/pull/331) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.2](https://github.com/puppetlabs/vmpooler/tree/0.6.2) (2019-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.1...0.6.2)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Fix typo in domain [\#330](https://github.com/puppetlabs/vmpooler/pull/330) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-140\) Ensure a VM is alive at checkout [\#329](https://github.com/puppetlabs/vmpooler/pull/329) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.1](https://github.com/puppetlabs/vmpooler/tree/0.6.1) (2019-05-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.0...0.6.1)
+
+**Merged pull requests:**
+
+- Update Changelog ahead of building 0.6.1 [\#328](https://github.com/puppetlabs/vmpooler/pull/328) ([sbeaulie](https://github.com/sbeaulie))
+- Update API.md \[skip ci\] [\#327](https://github.com/puppetlabs/vmpooler/pull/327) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Optimize the status api using redis pipeline [\#326](https://github.com/puppetlabs/vmpooler/pull/326) ([sbeaulie](https://github.com/sbeaulie))
+- Update changelog ahead of 0.6.0 release. [\#325](https://github.com/puppetlabs/vmpooler/pull/325) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.0](https://github.com/puppetlabs/vmpooler/tree/0.6.0) (2019-04-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.1...0.6.0)
+
+**Merged pull requests:**
+
+- \(QENG-7201\) Vmpooler pool statistic endpoint optimization [\#324](https://github.com/puppetlabs/vmpooler/pull/324) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-141\) Fix order of processing migrating and pending queues [\#323](https://github.com/puppetlabs/vmpooler/pull/323) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Add bundler to dockerfile\_local [\#322](https://github.com/puppetlabs/vmpooler/pull/322) ([mattkirby](https://github.com/mattkirby))
+- Update changelog to 0.5.1 [\#321](https://github.com/puppetlabs/vmpooler/pull/321) ([mattkirby](https://github.com/mattkirby))
+
+## [0.5.1](https://github.com/puppetlabs/vmpooler/tree/0.5.1) (2019-04-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.0...0.5.1)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Ensure a running VM stays in a queue [\#320](https://github.com/puppetlabs/vmpooler/pull/320) ([mattkirby](https://github.com/mattkirby))
+- Fix Dockerfile link in readme and add note about http requests for dev [\#316](https://github.com/puppetlabs/vmpooler/pull/316) ([briancain](https://github.com/briancain))
+
+## [0.5.0](https://github.com/puppetlabs/vmpooler/tree/0.5.0) (2019-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.4.0...0.5.0)
+
+**Merged pull requests:**
+
+- \(POOLER-139\) Fix discovering checked out VM [\#318](https://github.com/puppetlabs/vmpooler/pull/318) ([mattkirby](https://github.com/mattkirby))
+
+## [0.4.0](https://github.com/puppetlabs/vmpooler/tree/0.4.0) (2019-02-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.3.0...0.4.0)
+
+**Merged pull requests:**
+
+- \(MAINT\) Update changelog for 0.4.0 release [\#315](https://github.com/puppetlabs/vmpooler/pull/315) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-138\) Support multiple pools per alias [\#314](https://github.com/puppetlabs/vmpooler/pull/314) ([mattkirby](https://github.com/mattkirby))
+- Update dockerfile jruby to 9.2 [\#313](https://github.com/puppetlabs/vmpooler/pull/313) ([mattkirby](https://github.com/mattkirby))
+- Stop testing ruby 2.3.x [\#312](https://github.com/puppetlabs/vmpooler/pull/312) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-137\) Support integer environment variables [\#311](https://github.com/puppetlabs/vmpooler/pull/311) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update travis to test latest ruby [\#309](https://github.com/puppetlabs/vmpooler/pull/309) ([mattkirby](https://github.com/mattkirby))
+
+## [0.3.0](https://github.com/puppetlabs/vmpooler/tree/0.3.0) (2018-12-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.2...0.3.0)
+
+**Merged pull requests:**
+
+- Change version 0.2.2 to 0.3.0 [\#310](https://github.com/puppetlabs/vmpooler/pull/310) ([mattkirby](https://github.com/mattkirby))
+- Ensure nodes are consistent for usage stats [\#308](https://github.com/puppetlabs/vmpooler/pull/308) ([mattkirby](https://github.com/mattkirby))
+- Update changelog for 0.2.3 [\#307](https://github.com/puppetlabs/vmpooler/pull/307) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-134\) Ship VM usage stats [\#306](https://github.com/puppetlabs/vmpooler/pull/306) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-133\) Identify when a ready VM has failed [\#305](https://github.com/puppetlabs/vmpooler/pull/305) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-37\) Document HTTP responses [\#304](https://github.com/puppetlabs/vmpooler/pull/304) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-132\) Sync pool size on dashboard start [\#303](https://github.com/puppetlabs/vmpooler/pull/303) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.2](https://github.com/puppetlabs/vmpooler/tree/0.2.2) (2018-10-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
+
+**Merged pull requests:**
+
+- Update changelog version in preparation for release [\#302](https://github.com/puppetlabs/vmpooler/pull/302) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-131\) Return requested name when getting VMs [\#301](https://github.com/puppetlabs/vmpooler/pull/301) ([mattkirby](https://github.com/mattkirby))
+- Add docker-compose and dockerfile to support it [\#300](https://github.com/puppetlabs/vmpooler/pull/300) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.1](https://github.com/puppetlabs/vmpooler/tree/0.2.1) (2018-09-19)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+
+**Merged pull requests:**
+
+- Bump version for vmpooler in changelog [\#299](https://github.com/puppetlabs/vmpooler/pull/299) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-129\) Allow setting weights for backends [\#298](https://github.com/puppetlabs/vmpooler/pull/298) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-130\) Improve delta disk creation handling [\#297](https://github.com/puppetlabs/vmpooler/pull/297) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-114\) Refactor check\_pool in pool\_manager [\#296](https://github.com/puppetlabs/vmpooler/pull/296) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.0](https://github.com/puppetlabs/vmpooler/tree/0.2.0) (2018-07-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+
+**Closed issues:**
+
+- create release [\#262](https://github.com/puppetlabs/vmpooler/issues/262)
+- Add API to delete a snapshot [\#163](https://github.com/puppetlabs/vmpooler/issues/163)
+
+**Merged pull requests:**
+
+- \(MAINT\) release 0.2.0 [\#294](https://github.com/puppetlabs/vmpooler/pull/294) ([mattkirby](https://github.com/mattkirby))
+- Remove VM from completed only after destroy [\#293](https://github.com/puppetlabs/vmpooler/pull/293) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-128\) Remove references to VM mutex when destroying [\#292](https://github.com/puppetlabs/vmpooler/pull/292) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Document config via environment [\#291](https://github.com/puppetlabs/vmpooler/pull/291) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) change domain to example.com [\#290](https://github.com/puppetlabs/vmpooler/pull/290) ([steveax](https://github.com/steveax))
+- Update entrypoint in dockerfile for vmpooler gem [\#289](https://github.com/puppetlabs/vmpooler/pull/289) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) release 0.1.0 [\#288](https://github.com/puppetlabs/vmpooler/pull/288) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-66\) Purge vms and folders no longer configured [\#274](https://github.com/puppetlabs/vmpooler/pull/274) ([mattkirby](https://github.com/mattkirby))
+- Adds a new mechanism to load providers from any gem or file path automatically [\#263](https://github.com/puppetlabs/vmpooler/pull/263) ([logicminds](https://github.com/logicminds))
+
+## [0.1.0](https://github.com/puppetlabs/vmpooler/tree/0.1.0) (2018-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...0.1.0)
+
+**Closed issues:**
+
+- jruby 1.7.8 does not support safe\_load [\#243](https://github.com/puppetlabs/vmpooler/issues/243)
+- YAML.safe\_load does not work with symbols in config file [\#240](https://github.com/puppetlabs/vmpooler/issues/240)
+- vmpooler fails to fetch vm with dummy provider [\#238](https://github.com/puppetlabs/vmpooler/issues/238)
+- Any interest in VRA7 support? [\#235](https://github.com/puppetlabs/vmpooler/issues/235)
+- Do not have a hardcoded list of VM providers [\#230](https://github.com/puppetlabs/vmpooler/issues/230)
+- Use a dynamic check\_pool period [\#226](https://github.com/puppetlabs/vmpooler/issues/226)
+- vmpooler doesn't seem to recognize ready VMs [\#218](https://github.com/puppetlabs/vmpooler/issues/218)
+- `find\_vmdks` in `vsphere\_helper` should not use `vmdk\_datastore.\_connection` [\#213](https://github.com/puppetlabs/vmpooler/issues/213)
+- `get\_base\_vm\_container\_from` in `vsphere\_helper` ensures the wrong connection [\#212](https://github.com/puppetlabs/vmpooler/issues/212)
+- `close` in vsphere\_helper throws an error if a connection was never made [\#211](https://github.com/puppetlabs/vmpooler/issues/211)
+- `find\_pool` in vsphere\_helper.rb has subtle errors [\#210](https://github.com/puppetlabs/vmpooler/issues/210)
+- `find\_pool` in vsphere\_helper tends to throw instead of returning nil for missing pools [\#209](https://github.com/puppetlabs/vmpooler/issues/209)
+- Vsphere connections are always insecure \(Ignore cert errors\) [\#207](https://github.com/puppetlabs/vmpooler/issues/207)
+- `find\_folder` in vsphere\_helper.rb has subtle errors [\#204](https://github.com/puppetlabs/vmpooler/issues/204)
+- Should not use `abort` in vsphere\_helper [\#203](https://github.com/puppetlabs/vmpooler/issues/203)
+- No reason why get\_snapshot\_list is defined in vsphere\_helper [\#202](https://github.com/puppetlabs/vmpooler/issues/202)
+- Setting max\_tries in configuration results in vSphereHelper going into infinite loop [\#199](https://github.com/puppetlabs/vmpooler/issues/199)
+- "connect.open" metric is doubled up if a connection is broken [\#195](https://github.com/puppetlabs/vmpooler/issues/195)
+- Remove the use of global variables in the vSphere helper [\#194](https://github.com/puppetlabs/vmpooler/issues/194)
+- Should exit Threads cleanly [\#193](https://github.com/puppetlabs/vmpooler/issues/193)
+- check\_ready\_vm unnecessarily calls open\_socket [\#185](https://github.com/puppetlabs/vmpooler/issues/185)
+- Feature Request: Add provider support [\#181](https://github.com/puppetlabs/vmpooler/issues/181)
+- Document all possible HTTP response codes for endpoints [\#166](https://github.com/puppetlabs/vmpooler/issues/166)
+- Add API to clone new VM from existing VM snapshot [\#165](https://github.com/puppetlabs/vmpooler/issues/165)
+- vsphere\_helper.rb: find\_least\_used\_host should warn if no suitable hosts are found [\#164](https://github.com/puppetlabs/vmpooler/issues/164)
+- find\_vm uses just hostname delta, vSphere searchIndex matches on FQDN [\#141](https://github.com/puppetlabs/vmpooler/issues/141)
+- Tagging does not support boolean values [\#135](https://github.com/puppetlabs/vmpooler/issues/135)
+- POST to /api/v1/token returns WEBrick::HTTPStatus::LengthRequired error [\#132](https://github.com/puppetlabs/vmpooler/issues/132)
+- vmpooler throwing exceptions [\#129](https://github.com/puppetlabs/vmpooler/issues/129)
+- NilClass error when running API without Graphite configured [\#81](https://github.com/puppetlabs/vmpooler/issues/81)
+- Manually removing VM's result in state mis-match [\#80](https://github.com/puppetlabs/vmpooler/issues/80)
+- Add support for customization specs [\#79](https://github.com/puppetlabs/vmpooler/issues/79)
+
+**Merged pull requests:**
+
+- \(maint\) Fix vmpooler require in bin/vmpooler [\#287](https://github.com/puppetlabs/vmpooler/pull/287) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Remove ruby 2.2.10 from travis config [\#286](https://github.com/puppetlabs/vmpooler/pull/286) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Add changelog and contributing guidlines [\#285](https://github.com/puppetlabs/vmpooler/pull/285) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Remove find\_pool and update pending tests [\#283](https://github.com/puppetlabs/vmpooler/pull/283) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-109\) Allow API to run independently [\#281](https://github.com/puppetlabs/vmpooler/pull/281) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-81\) Add time remaining information [\#280](https://github.com/puppetlabs/vmpooler/pull/280) ([smcelmurry](https://github.com/smcelmurry))
+- Revert "\(POOLER-81\) Add time\_remaining information" [\#279](https://github.com/puppetlabs/vmpooler/pull/279) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Fix test reference to find\_vm [\#278](https://github.com/puppetlabs/vmpooler/pull/278) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-34\) Ship clone request to ready time to metrics [\#277](https://github.com/puppetlabs/vmpooler/pull/277) ([smcelmurry](https://github.com/smcelmurry))
+- \(POOLER-81\) Add time\_remaining information [\#276](https://github.com/puppetlabs/vmpooler/pull/276) ([smcelmurry](https://github.com/smcelmurry))
+- Add jruby 9.2 to travis testing [\#275](https://github.com/puppetlabs/vmpooler/pull/275) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-124\) Fix evaluation of max\_tries [\#273](https://github.com/puppetlabs/vmpooler/pull/273) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-40\) Do not return folders with get\_pool\_vms [\#272](https://github.com/puppetlabs/vmpooler/pull/272) ([mattkirby](https://github.com/mattkirby))
+- Ensure template deltas are created once [\#271](https://github.com/puppetlabs/vmpooler/pull/271) ([mattkirby](https://github.com/mattkirby))
+- Do not run duplicate instances of inventory check for a pool [\#270](https://github.com/puppetlabs/vmpooler/pull/270) ([mattkirby](https://github.com/mattkirby))
+- Eliminate duplicate VM object lookups where possible [\#269](https://github.com/puppetlabs/vmpooler/pull/269) ([mattkirby](https://github.com/mattkirby))
+- Reduce object lookups for finding folders [\#268](https://github.com/puppetlabs/vmpooler/pull/268) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-113\) Add support for multiple LDAP search bases [\#267](https://github.com/puppetlabs/vmpooler/pull/267) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-31\) Expire redis vm key when clone fails [\#266](https://github.com/puppetlabs/vmpooler/pull/266) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-112\) Ensure a VM is only destroyed once [\#265](https://github.com/puppetlabs/vmpooler/pull/265) ([mattkirby](https://github.com/mattkirby))
+- Adds a gemspec file [\#264](https://github.com/puppetlabs/vmpooler/pull/264) ([logicminds](https://github.com/logicminds))
+- Change default vsphere connection behavior [\#261](https://github.com/puppetlabs/vmpooler/pull/261) ([mattkirby](https://github.com/mattkirby))
+- Remove propertyCollector from add\_disk [\#260](https://github.com/puppetlabs/vmpooler/pull/260) ([mattkirby](https://github.com/mattkirby))
+- Update ruby versions for travis [\#259](https://github.com/puppetlabs/vmpooler/pull/259) ([mattkirby](https://github.com/mattkirby))
+- Update to generic launcher [\#258](https://github.com/puppetlabs/vmpooler/pull/258) ([frozenfoxx](https://github.com/frozenfoxx))
+- Add support for setting redis port and password [\#257](https://github.com/puppetlabs/vmpooler/pull/257) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-107\) Add configuration API endpoint [\#256](https://github.com/puppetlabs/vmpooler/pull/256) ([mattkirby](https://github.com/mattkirby))
+- Create vmpooler.service [\#255](https://github.com/puppetlabs/vmpooler/pull/255) ([frozenfoxx](https://github.com/frozenfoxx))
+- \(POOLER-101\) Update nokogiri and net-ldap [\#254](https://github.com/puppetlabs/vmpooler/pull/254) ([mattkirby](https://github.com/mattkirby))
+- Add dockerfile without redis [\#253](https://github.com/puppetlabs/vmpooler/pull/253) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-103\) Fix configuration file loading [\#252](https://github.com/puppetlabs/vmpooler/pull/252) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-68\) Replace find\_vm search mechanism [\#251](https://github.com/puppetlabs/vmpooler/pull/251) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Add the last boot time for each pool [\#250](https://github.com/puppetlabs/vmpooler/pull/250) ([sbeaulie](https://github.com/sbeaulie))
+- Fix typo in error message [\#249](https://github.com/puppetlabs/vmpooler/pull/249) ([teancom](https://github.com/teancom))
+- Identify when ESXi host quickstats do not return [\#248](https://github.com/puppetlabs/vmpooler/pull/248) ([mattkirby](https://github.com/mattkirby))
+- Update jruby version for travis to 9.1.13.0 [\#247](https://github.com/puppetlabs/vmpooler/pull/247) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-96\) Setting the Rubygems version [\#246](https://github.com/puppetlabs/vmpooler/pull/246) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-93\) Extend API endpoint to provide just what is needed [\#245](https://github.com/puppetlabs/vmpooler/pull/245) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-92\) Add the alias information in the API status page for each… [\#244](https://github.com/puppetlabs/vmpooler/pull/244) ([sbeaulie](https://github.com/sbeaulie))
+- \(QENG-5305\) Improve vmpooler host selection [\#242](https://github.com/puppetlabs/vmpooler/pull/242) ([mattkirby](https://github.com/mattkirby))
+- Allow user to specify a configuration file in VMPOOLER\_CONFIG\_FILE variable [\#241](https://github.com/puppetlabs/vmpooler/pull/241) ([amcdson](https://github.com/amcdson))
+- Fix no implicit conversion to rational from nil [\#239](https://github.com/puppetlabs/vmpooler/pull/239) ([sbeaulie](https://github.com/sbeaulie))
+- Updated Vagrant box and associated docs [\#237](https://github.com/puppetlabs/vmpooler/pull/237) ([genebean](https://github.com/genebean))
+- \(GH-226\) Respond quickly to VMs being consumed [\#236](https://github.com/puppetlabs/vmpooler/pull/236) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-89\) Identify when config issue is present [\#234](https://github.com/puppetlabs/vmpooler/pull/234) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Update template delta script for moved vsphere credentials [\#233](https://github.com/puppetlabs/vmpooler/pull/233) ([ScottGarman](https://github.com/ScottGarman))
+- Fix rubocop [\#232](https://github.com/puppetlabs/vmpooler/pull/232) ([glennsarti](https://github.com/glennsarti))
+- \(GH-230\) Dynamically load VM Providers [\#231](https://github.com/puppetlabs/vmpooler/pull/231) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Remove phantom VMs that are in Redis but don't exist in provider [\#229](https://github.com/puppetlabs/vmpooler/pull/229) ([glennsarti](https://github.com/glennsarti))
+- Update find\_least\_used\_compatible\_host to specify pool [\#228](https://github.com/puppetlabs/vmpooler/pull/228) ([mattkirby](https://github.com/mattkirby))
+- \(GH-226\) Use a dynamic pool\_check loop period [\#227](https://github.com/puppetlabs/vmpooler/pull/227) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Update development documentation [\#225](https://github.com/puppetlabs/vmpooler/pull/225) ([glennsarti](https://github.com/glennsarti))
+- \(GH-213\) Remove use of private \_connection method [\#224](https://github.com/puppetlabs/vmpooler/pull/224) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-83\) Add ability to specify a datacenter for vsphere [\#223](https://github.com/puppetlabs/vmpooler/pull/223) ([glennsarti](https://github.com/glennsarti))
+- Added Vagrant setup and fixed the Dockerfile so it actually works [\#222](https://github.com/puppetlabs/vmpooler/pull/222) ([genebean](https://github.com/genebean))
+- Adding support for multiple vsphere providers [\#221](https://github.com/puppetlabs/vmpooler/pull/221) ([sbeaulie](https://github.com/sbeaulie))
+- Refactor get\_cluster\_host\_utilization method [\#220](https://github.com/puppetlabs/vmpooler/pull/220) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Pin rack to 1.x [\#219](https://github.com/puppetlabs/vmpooler/pull/219) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-72\)\(POOLER-70\)\(POOLER-52\) Move Pool Manager to use the VM Provider [\#216](https://github.com/puppetlabs/vmpooler/pull/216) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Emit console messages when debugging is enabled [\#215](https://github.com/puppetlabs/vmpooler/pull/215) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\)\(POOLER-52\) Create a functional vSphere Provider  [\#214](https://github.com/puppetlabs/vmpooler/pull/214) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop violations  [\#208](https://github.com/puppetlabs/vmpooler/pull/208) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix credentials in vsphere\_helper [\#200](https://github.com/puppetlabs/vmpooler/pull/200) ([glennsarti](https://github.com/glennsarti))
+- Update usage of global variablesin vsphere\_helper [\#198](https://github.com/puppetlabs/vmpooler/pull/198) ([mattkirby](https://github.com/mattkirby))
+- Remove duplicate of metrics.connect.open [\#197](https://github.com/puppetlabs/vmpooler/pull/197) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-73\) Add spec tests for vsphere\_helper [\#196](https://github.com/puppetlabs/vmpooler/pull/196) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop offenses [\#191](https://github.com/puppetlabs/vmpooler/pull/191) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Prepare to refactor VSphere code into a VM Provider [\#190](https://github.com/puppetlabs/vmpooler/pull/190) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Refactor clone\_vm to take pool configuration object [\#189](https://github.com/puppetlabs/vmpooler/pull/189) ([glennsarti](https://github.com/glennsarti))
+- \(GH-185\) Remove unnecessary checks in check\_ready\_vm  [\#188](https://github.com/puppetlabs/vmpooler/pull/188) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Only load rubocop rake tasks if gem is available [\#187](https://github.com/puppetlabs/vmpooler/pull/187) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Add rubocop and allow failures in Travis CI [\#183](https://github.com/puppetlabs/vmpooler/pull/183) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-73\) Update unit tests prior to refactoring [\#182](https://github.com/puppetlabs/vmpooler/pull/182) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-71\) Add dummy authentication provider [\#180](https://github.com/puppetlabs/vmpooler/pull/180) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Enhance VM Pooler developer experience [\#177](https://github.com/puppetlabs/vmpooler/pull/177) ([glennsarti](https://github.com/glennsarti))
+
+
+
+\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*

CODEOWNERS

@@ -0,0 +1,10 @@
+
+# This will cause RE to be assigned review of any opened PRs against
+# the branches containing this file.
+# See https://help.github.com/en/articles/about-code-owners for info on how to
+# take ownership of parts of the code base that should be reviewed by another
+# team.
+
+# RE will be the default owners for everything in the repo.
+*                                     @puppetlabs/release-engineering
+

CONTRIBUTING.md

@@ -1,9 +1,6 @@
 # How to contribute
 
-Third-party patches are essential for keeping vmpooler great. We want to keep it as easy as possible to contribute changes that
-get things working in your environment. There are a few guidelines that we
-need contributors to follow so that we can have a chance of keeping on
-top of things.
+Third-party patches are essential for keeping VMPooler great. We want to keep it as easy as possible to contribute changes that get things working in your environment. There are a few guidelines that we need contributors to follow so that we can have a chance of keeping on top of things.
 
 ## Getting Started
 
@@ -18,28 +15,25 @@ top of things.
 
 * Create a topic branch from where you want to base your work.
   * This is usually the master branch.
-  * Only target release branches if you are certain your fix must be on that
-    branch.
-  * To quickly create a topic branch based on master; `git checkout -b
-    fix/master/my_contribution master`. Please avoid working directly on the
-    `master` branch.
+  * Only target release branches if you are certain your fix must be on that branch.
+  * To quickly create a topic branch based on master: `git checkout -b fix/master/my_contribution master`. Please avoid working directly on the `master` branch.
 * Make commits of logical units.
 * Check for unnecessary whitespace with `git diff --check` before committing.
 * Make sure your commit messages are in the proper format.
 
-````
-    (POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
+```plain
+(POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
 
-    Without this patch applied the example commit message in the CONTRIBUTING
-    document is not a concrete example.  This is a problem because the
-    contributor is left to imagine what the commit message should look like
-    based on a description rather than an example.  This patch fixes the
-    problem by making the example concrete and imperative.
+Without this patch applied the example commit message in the CONTRIBUTING
+document is not a concrete example.  This is a problem because the
+contributor is left to imagine what the commit message should look like
+based on a description rather than an example.  This patch fixes the
+problem by making the example concrete and imperative.
 
-    The first line is a real life imperative statement with a ticket number
-    from our issue tracker.  The body describes the behavior without the patch,
-    why this is a problem, and how the patch fixes the problem when applied.
-````
+The first line is a real life imperative statement with a ticket number
+from our issue tracker.  The body describes the behavior without the patch,
+why this is a problem, and how the patch fixes the problem when applied.
+```
 
 * Make sure you have added the necessary tests for your changes.
 * Run _all_ the tests to assure nothing else was accidentally broken.
@@ -48,36 +42,32 @@ top of things.
 
 ### Documentation
 
-For changes of a trivial nature to comments and documentation, it is not
-always necessary to create a new ticket in Jira. In this case, it is
-appropriate to start the first line of a commit with '(doc)' instead of
-a ticket number.
+For changes of a trivial nature to comments and documentation, it is not always necessary to create a new ticket in Jira. In this case, it is appropriate to start the first line of a commit with '(doc)' instead of a ticket number.
 
-````
-    (doc) Add documentation commit example to CONTRIBUTING
+```plain
+(doc) Add documentation commit example to CONTRIBUTING
 
-    There is no example for contributing a documentation commit
-    to the Puppet repository. This is a problem because the contributor
-    is left to assume how a commit of this nature may appear.
+There is no example for contributing a documentation commit
+to the Puppet repository. This is a problem because the contributor
+is left to assume how a commit of this nature may appear.
 
-    The first line is a real life imperative statement with '(doc)' in
-    place of what would have been the ticket number in a
-    non-documentation related commit. The body describes the nature of
-    the new documentation or comments added.
-````
+The first line is a real life imperative statement with '(doc)' in
+place of what would have been the ticket number in a
+non-documentation related commit. The body describes the nature of
+the new documentation or comments added.
+```
 
 ## Submitting Changes
 
-* Sign the [Contributor License Agreement](http://links.puppetlabs.com/cla).
+* Sign the Contributor License Agreement.
 * Push your changes to a topic branch in your fork of the repository.
 * Submit a pull request to the repository in the puppetlabs organization.
 * Update your Jira ticket to mark that you have submitted code and are ready for it to be reviewed (Status: Ready for Merge).
   * Include a link to the pull request in the ticket.
-* The Puppet SRE team looks at Pull Requests on a regular basis.
-* After feedback has been given we expect responses within two weeks. After two
-  weeks we may close the pull request if it isn't showing any activity.
+* The Puppet Release Engineering team looks at Pull Requests on a regular basis.
+* After feedback has been given we expect responses within two weeks. After two weeks we may close the pull request if it isn't showing any activity.
 
-# Additional Resources
+## Additional Resources
 
 * [Puppet Labs community guildelines](http://docs.puppetlabs.com/community/community_guidelines.html)
 * [Bug tracker (Jira)](http://tickets.puppetlabs.com)

Gemfile

@@ -1,42 +1,13 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
-gem 'json', '>= 1.8'
-gem 'pickup', '~> 0.0.11'
-gem 'puma', '~> 3.11'
-gem 'rack', '~> 2.0'
-gem 'rake', '~> 12.3'
-gem 'redis', '~> 4.0'
-gem 'rbvmomi', '~> 1.13'
-gem 'sinatra', '~> 2.0'
-gem 'net-ldap', '~> 0.16'
-gem 'statsd-ruby', '~> 1.4.0', :require => 'statsd'
-gem 'connection_pool', '~> 2.2'
-gem 'nokogiri', '~> 1.8'
-
-group :development do
-  gem 'pry'
-end
-
-# Test deps
-group :test do
-  # required in order for the providers auto detect mechanism to work
-  gem 'vmpooler', path: './'
-  gem 'mock_redis', '>= 0.17.0'
-  gem 'rack-test', '>= 0.6'
-  gem 'rspec', '>= 3.2'
-  gem 'simplecov', '>= 0.11.2'
-  gem 'yarjuf', '>= 2.0'
-  # Rubocop would be ok jruby but for now we only use it on
-  # MRI or Windows platforms
-  gem "rubocop", :platforms => [:ruby, :x64_mingw]
-end
+gemspec
 
 # Evaluate Gemfile.local if it exists
-if File.exists? "#{__FILE__}.local"
+if File.exist? "#{__FILE__}.local"
   instance_eval(File.read("#{__FILE__}.local"))
 end
 
 # Evaluate ~/.gemfile if it exists
-if File.exists?(File.join(Dir.home, '.gemfile'))
+if File.exist?(File.join(Dir.home, '.gemfile'))
   instance_eval(File.read(File.join(Dir.home, '.gemfile')))
 end

Gemfile.lock

@@ -0,0 +1,219 @@
+PATH
+  remote: .
+  specs:
+    vmpooler (3.8.1)
+      concurrent-ruby (~> 1.1)
+      connection_pool (~> 2.4)
+      deep_merge (~> 1.2)
+      net-ldap (~> 0.16)
+      opentelemetry-exporter-jaeger (= 0.23.0)
+      opentelemetry-instrumentation-concurrent_ruby (= 0.21.1)
+      opentelemetry-instrumentation-http_client (= 0.22.2)
+      opentelemetry-instrumentation-rack (= 0.23.4)
+      opentelemetry-instrumentation-redis (= 0.25.3)
+      opentelemetry-instrumentation-sinatra (= 0.23.2)
+      opentelemetry-resource_detectors (= 0.24.2)
+      opentelemetry-sdk (~> 1.8)
+      pickup (~> 0.0.11)
+      prometheus-client (>= 2, < 5)
+      puma (>= 5.0.4, < 7)
+      rack (>= 2.2, < 4.0)
+      rake (~> 13.0)
+      redis (~> 5.0)
+      sinatra (>= 2, < 4)
+      spicy-proton (~> 2.1)
+      statsd-ruby (~> 1.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    base64 (0.1.2)
+    bindata (2.5.1)
+    builder (3.3.0)
+    climate_control (1.2.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    deep_merge (1.2.2)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    faraday (2.13.1)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
+    ffi (1.17.2-java)
+    google-cloud-env (2.2.1)
+      faraday (>= 1.0, < 3.a)
+    json (2.12.2)
+    json (2.12.2-java)
+    language_server-protocol (3.17.0.5)
+    logger (1.7.0)
+    method_source (1.1.0)
+    mock_redis (0.37.0)
+    mustermann (3.0.3)
+      ruby2_keywords (~> 0.0.1)
+    net-http (0.6.0)
+      uri
+    net-ldap (0.19.0)
+    nio4r (2.7.4)
+    nio4r (2.7.4-java)
+    opentelemetry-api (1.5.0)
+    opentelemetry-common (0.20.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-jaeger (0.23.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+      thrift
+    opentelemetry-instrumentation-base (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-concurrent_ruby (0.21.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http_client (0.22.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rack (0.23.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-redis (0.25.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-sinatra (0.23.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-resource_detectors (0.24.2)
+      google-cloud-env
+      opentelemetry-sdk (~> 1.0)
+    opentelemetry-sdk (1.8.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.11.0)
+      opentelemetry-api (~> 1.0)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
+    pickup (0.0.11)
+    prism (1.4.0)
+    prometheus-client (4.2.4)
+      base64
+    pry (0.15.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry (0.15.2-java)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+      spoon (~> 0.0)
+    puma (6.6.0)
+      nio4r (~> 2.0)
+    puma (6.6.0-java)
+      nio4r (~> 2.0)
+    racc (1.8.1)
+    racc (1.8.1-java)
+    rack (2.2.17)
+    rack-protection (3.2.0)
+      base64 (>= 0.1.0)
+      rack (~> 2.2, >= 2.2.4)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    redis (5.4.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.24.0)
+      connection_pool
+    regexp_parser (2.10.0)
+    rexml (3.4.1)
+    rspec (3.13.1)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.4)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.4)
+    rubocop (1.56.4)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.1)
+    simplecov_json_formatter (0.1.4)
+    sinatra (3.2.0)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.2.0)
+      tilt (~> 2.0)
+    spicy-proton (2.1.15)
+      bindata (~> 2.3)
+    spoon (0.0.6)
+      ffi
+    statsd-ruby (1.5.0)
+    thor (1.3.2)
+    thrift (0.22.0)
+    tilt (2.6.0)
+    unicode-display_width (2.6.0)
+    uri (1.0.3)
+    yarjuf (2.0.0)
+      builder
+      rspec (~> 3)
+
+PLATFORMS
+  arm64-darwin-22
+  arm64-darwin-23
+  arm64-darwin-25
+  universal-java-11
+  universal-java-17
+  x86_64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  climate_control (>= 0.2.0)
+  mock_redis (= 0.37.0)
+  pry
+  rack-test (>= 0.6)
+  rspec (>= 3.2)
+  rubocop (~> 1.56.0)
+  simplecov (>= 0.11.2)
+  thor (~> 1.0, >= 1.0.1)
+  vmpooler!
+  yarjuf (>= 2.0)
+
+BUNDLED WITH
+   2.4.18

PROVIDER_API.md

@@ -1,83 +0,0 @@
-# Provider API
-
-## Create a new provider gem from scratch
-
-### Requirements
-1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
-2. the main provider code file should be named the same at the name of the provider. ie. (vpshere == lib/vmpooler/providers/vsphere.rb)
-3. The gem must be installed on the same machine as vmpooler
-4. The provider name must be referenced in the vmpooler config file in order for it to be loaded.
-5. Your gem name or repository name should contain vmpooler-<name>-provider so the community can easily search provider plugins
-   for vmpooler.
-### 1. Use bundler to create the provider scaffolding
-
-```
-bundler gem --test=rspec --no-exe --no-ext vmpooler-spoof-provider
-cd vmpooler-providers-spoof/
-mkdir -p ./lib/vmpooler/providers
-cd ./lib/vmpooler/providers
-touch spoof.rb
-
-```
-
-There may be some boilerplate files there were generated, just delete those.
-
-### 2. Create the main provider file
-Ensure the main provider file uses the following code.
-
-
-```ruby
-# lib/vmpooler/providers/spoof.rb
-require 'yaml'
-require 'vmpooler/providers/base'
-
-module Vmpooler
-  class PoolManager
-    class Provider
-      class Spoof < Vmpooler::PoolManager::Provider::Base
-      
-      # at this time it is not documented which methods should be implemented
-      # have a look at the vmpooler/providers/vpshere provider for examples
-
-      end
-
-    end
-  end
-end
-  
-
-```
-
-### 3. Fill out your gemspec
-Ensure you fill out your gemspec file to your specifications.  If you need a dependency please make sure you require them.
-
-`spec.add_dependency "vmware", "~> 1.15"`.
-
-At a minimum you may want to add the vmpooler gem as a dev dependency so you can use it during testing.
-
-`spec.add_dev_dependency "vmpooler", "~> 1.15"`
-
-or in your Gemfile
-
-```ruby
-
-gem 'vmpooler', github: 'puppetlabs/vmpooler'
-```
-
-Also make sure this dependency can be loaded by jruby.  If the dependency cannot be used by jruby don't use it.
-
-### 4. Create some tests
-Your provider code should be tested before releasing.  Copy and refactor some tests from the vmpooler gem under 
-`spec/unit/providers/dummy_spec.rb`
-
-### 5. Publish
-Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
-
-
-## Available Third Party Providers
-Be the first to update this list.  Create a provider today!
-
-
-## Example provider
-You can use the following [repo as an example](https://github.com/logicminds/vmpooler-vsphere-provider) of how to setup your provider gem.
-

README.md

@@ -1,36 +1,101 @@
-![vmpooler](https://raw.github.com/sschneid/vmpooler/master/lib/vmpooler/public/img/logo.gif)
+![VMPooler](lib/vmpooler/public/img/logo.png)
 
-# vmpooler
+# VMPooler
 
-vmpooler provides configurable 'pools' of instantly-available (running) virtual machines.
+- [VMPooler](#vmpooler)
+  - [Usage](#usage)
+    - [Migrating to v3](#migrating-to-v3)
+    - [v2.0.0 note](#v200-note)
+  - [Installation](#installation)
+    - [Dependencies](#dependencies)
+      - [Redis](#redis)
+      - [Other gems](#other-gems)
+  - [Configuration](#configuration)
+  - [Components](#components)
+    - [API](#api)
+    - [Dashboard](#dashboard)
+  - [Related tools and resources](#related-tools-and-resources)
+    - [Command-line Utility](#command-line-utility)
+    - [Vagrant plugin](#vagrant-plugin)
+  - [Development](#development)
+    - [docker-compose](#docker-compose)
+    - [Running docker-compose inside Vagrant](#running-docker-compose-inside-vagrant)
+    - [URLs when using docker-compose](#urls-when-using-docker-compose)
+  - [Update the Gemfile Lock](#update-the-gemfile-lock)
+  - [Releasing](#releasing)
+  - [License](#license)
 
+VMPooler provides configurable 'pools' of instantly-available (pre-provisioned) and/or on-demand (provisioned on request) virtual machines.
 
 ## Usage
 
-At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day.  Dynamic cloning of VM templates initially worked fine for this, but added several seconds to each test run and was unable to account for failed clone tasks.  By pushing these operations to a backend service, we were able to both speed up tests and eliminate test failures due to underlying infrastructure failures.
+At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. VMPooler manages the life cycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand.
 
+The recommended method for deploying VMPooler is via [https://github.com/puppetlabs/vmpooler-deployment](vmpooler-deployment).
+
+### Migrating to v3
+
+Starting with the v3.x release, management of DNS records is implemented as DNS plugins, similar to compute providers. This means each pool configuration should be pointing to a configuration object in `:dns_config` to determine it's method of record management.
+
+For those using the global `DOMAIN` environment variable or global `:config.domain` key, this means records were not previously being managed by VMPooler (presumably managed via dynamic dns), so it's value should be moved to `:dns_configs:<INSERT_YOUR_OWN_SYMBOL>:domain` with the value for `dns_class` for the config set to `dynamic-dns`.
+
+For example, the following < v3.x configuration:
+
+```yaml
+:config:
+  domain: 'example.com'
+```
+
+becomes:
+
+```yaml
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+```
+
+Then any pools that should have records created via the dns config above should now reference the named dns config in the `dns_plugin` key:
+
+```yaml
+:pools:
+  - name: 'debian-8-x86_64'
+    dns_plugin: 'example'
+```
+
+For those using the GCE provider, [vmpooler-provider-gce](https://github.com/puppetlabs/vmpooler-provider-gce), as of version 1.x the DNS management has been decoupled. See <https://github.com/puppetlabs/vmpooler-provider-gce#migrating-to-v1>
+
+### v2.0.0 note
+
+As of version 2.0.0, all providers other than the dummy one are now separate gems. Historically the vSphere provider was included within VMPooler itself. That code has been moved to the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository and the `vmpooler-provider-vsphere` gem. To migrate from VMPooler 1.x to 2.0 you will need to ensure that `vmpooler-provider-vsphere` is installed along side the `vmpooler` gem. See the  [Provider API](docs/PROVIDER_API.md) docs for more information.
 
 ## Installation
 
-### Prerequisites
-
-vmpooler is available as a gem
-
-To use the gem `gem install vmpooler`
+The recommended method of installation is via the Helm chart located in [puppetlabs/vmpooler-deployment](https://github.com/puppetlabs/vmpooler-deployment). That repository also provides Docker images of VMPooler.
 
 ### Dependencies
 
-Vmpooler requires a [Redis](http://redis.io/) server. This is the datastore used for vmpooler's inventory and queueing services.
+#### Redis
 
-### Configuration
+VMPooler requires a [Redis](http://redis.io/) server. This is the data store used for VMPooler's inventory and queuing services.
 
-Configuration for vmpooler may be provided via environment variables, or a configuration file.
+#### Other gems
 
-Please see this [configuration](docs/configuration.md) document for more details about configuring vmpooler via environment variables.
+VMPooler itself and the dev environment talked about below require additional Ruby gems to function. You can update the currently required ones for VMPooler by running `./update-gemfile-lock.sh`. The gems for the dev environment can be updated by running `./docker/update-gemfile-lock.sh`. These scripts will utilize the container on the FROM line of the Dockerfile to update the Gemfile.lock in the root of this repo and in the docker folder, respectively.
+
+## Configuration
+
+Configuration for VMPooler may be provided via environment variables, or a configuration file.
+
+The provided configuration defaults are reasonable for  small VMPooler instances with a few pools. If you plan to run a large VMPooler instance it is important to consider configuration values appropriate for the instance of your size in order to avoid starving the provider, or Redis, of connections.
+
+VMPooler uses a connection pool for Redis to improve efficiency and ensure thread safe usage. At Puppet, we run an instance with about 100 pools at any given time. We have to provide it with 200 Redis connections to the Redis connection pool, and a timeout for connections of 40 seconds, to avoid timeouts. Because metrics are generated for connection available and waited, your metrics provider will need to be able to cope with this volume. Prometheus or StatsD is recommended to ensure metrics get delivered reliably.
+
+Please see this [configuration](docs/configuration.md) document for more details about configuring VMPooler via environment variables.
 
 The following YAML configuration sets up two pools, `debian-7-i386` and `debian-7-x86_64`, which contain 5 running VMs each:
 
-```
+```yaml
 ---
 :providers:
   :vsphere:
@@ -63,45 +128,13 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian-
 
 See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for additional configuration options and parameters or for supporting multiple providers.
 
-### Running via Docker
+## Components
 
-A [Dockerfile](Dockerfile) is included in this repository to allow running vmpooler inside a Docker container. A configuration file can be used via volume mapping, and specifying the destination as the configuration file via environment variables, or the application can be configured with environment variables alone. The Dockerfile provides an entrypoint so you may choose whether to run API, or manager services. The default behavior will run both. To build and run:
-
-```
-docker build -t vmpooler . && docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-```
-
-To run only the API and dashboard
-
-```
-docker run -p 80:4567 -it vmpooler api
-```
-
-To run only the manager component
-
-```
-docker run -it vmpooler manager
-```
-
-### docker-compose
-
-A docker-compose file is provided to support running vmpooler easily via docker-compose.
-
-```
-docker-compose -f docker/docker-compose.yml up
-```
-
-### Running Docker inside Vagrant
-
-A vagrantfile is included in this repository. Please see [vagrant instructions](docs/vagrant.md) for details.
-
-## API and Dashboard
-
-vmpooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
+VMPooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
 
 ### API
 
-vmpooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
+VMPooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
 
 ### Dashboard
 
@@ -111,24 +144,71 @@ A dashboard is provided to offer real-time statistics and historical graphs.  It
 
 [Graphite](http://graphite.wikidot.com/) is required for historical data retrieval.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for details.
 
-## Command-line Utility
+## Related tools and resources
 
-- The [vmpooler_client.py](https://github.com/puppetlabs/vmpooler-client) CLI utility provides easy access to the vmpooler service. The tool is cross-platform and written in Python.
-- [vmfloaty](https://github.com/briancain/vmfloaty) is a ruby based CLI tool and scripting library written in ruby.
+### Command-line Utility
 
-## Vagrant plugin
+- [vmfloaty](https://github.com/puppetlabs/vmfloaty) is a ruby based CLI tool and scripting library. We consider it the primary way for users to interact with VMPooler.
 
-- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler) Use Vagrant to create and manage your vmpooler instances.
+### Vagrant plugin
 
-## Development and further documentation
+- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler): Use Vagrant to create and manage your VMPooler instances.
 
-For more information about setting up a development instance of vmpooler or other subjects, see the [docs/](docs) directory.
+## Development
 
-## Build status
+### docker-compose
 
-[![Build Status](https://travis-ci.org/puppetlabs/vmpooler.png?branch=master)](https://travis-ci.org/puppetlabs/vmpooler)
+A docker-compose file is provided to support running VMPooler and associated tools locally. This is useful for development because your local code is used to build the gem used in the docker-compose environment. The compose environment also pulls in the latest providers via git. Details of this setup are stored in the `docker/` folder.
 
+```bash
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+### Running docker-compose inside Vagrant
+
+A Vagrantfile is included in this repository so as to provide a reproducible development environment.
+
+```bash
+vagrant up
+vagrant ssh
+cd /vagrant
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+The Vagrant environment also contains multiple rubies you can utilize for spec test and the like. You can see a list of the pre-installed ones when you log in as part of the message of the day.
+
+For more information about setting up a development instance of VMPooler or other subjects, see the [docs/](docs) directory.
+
+### URLs when using docker-compose
+
+| Endpoint          | URL                                                                   |
+|-------------------|-----------------------------------------------------------------------|
+| Redis Commander   | [http://localhost:8079](http://localhost:8079)                        |
+| API               | [http://localhost:8080/api/v1]([http://localhost:8080/api/v1)         |
+| Dashboard         | [http://localhost:8080/dashboard/](http://localhost:8080/dashboard/)  |
+| Metrics (API)     | [http://localhost:8080/prometheus]([http://localhost:8080/prometheus) |
+| Metrics (Manager) | [http://localhost:8081/prometheus]([http://localhost:8081/prometheus) |
+| Jaeger            | [http://localhost:8082](http://localhost:8082)                        |
+
+Additionally, the Redis instance can be accessed at `localhost:6379`.
+
+## Update the Gemfile Lock
+
+To update the `Gemfile.lock` run `./update-gemfile-lock`.
+
+Verify, and update if needed, that the docker tag in the script and GitHub action workflows matches what is used in the [vmpooler-deployment Dockerfile](https://github.com/puppetlabs/vmpooler-deployment/blob/main/docker/Dockerfile).
+
+## Releasing
+
+Follow these steps to publish a new GitHub release, and build and push the gem to <https://rubygems.org>.
+
+1. Bump the "VERSION" in `lib/vmpooler/version.rb` appropriately based on changes in `CHANGELOG.md` since the last release.
+2. Run `./release-prep` to update `Gemfile.lock` and `CHANGELOG.md`.
+3. Commit and push changes to a new branch, then open a pull request against `main` and be sure to add the "maintenance" label.
+4. After the pull request is approved and merged, then navigate to Actions --> Release Gem --> run workflow --> Branch: main --> Run workflow.
 
 ## License
 
-vmpooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.
+VMPooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.

Vagrantfile

@@ -1,8 +1,12 @@
 # vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
 Vagrant.configure("2") do |config|
   config.vm.box = "genebean/centos-7-rvm-multi"
-  config.vm.network "forwarded_port", guest: 4567, host: 4567
-  config.vm.network "forwarded_port", guest: 8080, host: 8080
+  config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose
+  config.vm.network "forwarded_port", guest: 6379, host: 6379 # Redis
+  config.vm.network "forwarded_port", guest: 8079, host: 8079 # Redis Commander
+  config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose
+  config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose
+  config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose
   config.vm.provision "shell", inline: <<-SCRIPT
     mkdir /var/log/vmpooler
     chown vagrant:vagrant /var/log/vmpooler
@@ -11,9 +15,18 @@ Vagrant.configure("2") do |config|
     usermod -aG docker vagrant
     systemctl enable docker
     systemctl start docker
-    docker build -t vmpooler /vagrant
+    curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+    chmod +x /usr/local/bin/docker-compose
+    ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+    docker-compose --version
+    cd /vagrant
+    docker-compose -f docker/docker-compose.yml build
     docker images
-    echo 'To use the container with the dummy provider do this after "vagrant ssh":'
-    echo "docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler"
   SCRIPT
+
+  # config.vm.provider "virtualbox" do |v|
+  #   v.memory = 2048
+  #   v.cpus = 2
+  # end
+  
 end

bin/vmpooler

@@ -1,41 +1,59 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'vmpooler'
+require 'vmpooler/version'
 
 config = Vmpooler.config
+logger_file = config[:config]['logfile']
+prefix = config[:config]['prefix']
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
-logger_file = config[:config]['logfile']
+redis_connection_pool_size = config[:redis]['connection_pool_size']
+redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
+redis_reconnect_attempts = config[:redis]['reconnect_attempts']
+tracing_enabled = config[:tracing]['enabled']
+tracing_jaeger_host = config[:tracing]['jaeger_host']
 
-metrics = Vmpooler.new_metrics(config)
+logger = Vmpooler::Logger.new logger_file
+metrics = Vmpooler::Metrics.init(logger, config)
+
+version = Vmpooler::VERSION
+
+startup_args = ARGV
+Vmpooler.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
 
 torun_threads = []
 if ARGV.count == 0
-  torun = ['api', 'manager']
+  torun = %i[api manager]
 else
   torun = []
-  torun << 'api' if ARGV.include? 'api'
-  torun << 'manager' if ARGV.include? 'manager'
+  torun << :api if ARGV.include?('api')
+  torun << :manager if ARGV.include?('manager')
   exit(2) if torun.empty?
 end
 
-if torun.include? 'api'
+if torun.include?(:api)
   api = Thread.new do
-    thr = Vmpooler::API.new
     redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
-    thr.helpers.configure(config, redis, metrics)
-    thr.helpers.execute!
+    Vmpooler::API.execute(torun, config, redis, metrics, logger)
   end
   torun_threads << api
+elsif metrics.respond_to?(:setup_prometheus_metrics)
+  # Run the cut down API - Prometheus Metrics only.
+  prometheus_only_api = Thread.new do
+    Vmpooler::API.execute(torun, config, nil, metrics, logger)
+  end
+  torun_threads << prometheus_only_api
 end
 
-if torun.include? 'manager'
+if torun.include?(:manager)
   manager = Thread.new do
     Vmpooler::PoolManager.new(
       config,
-      Vmpooler.new_logger(logger_file),
-      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      logger,
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics, redis_reconnect_attempts),
       metrics
     ).execute!
   end
@@ -49,6 +67,4 @@ if ENV['VMPOOLER_DEBUG']
   end
 end
 
-torun_threads.each do |th|
-  th.join
-end
+torun_threads.each(&:join)

docker/Dockerfile

@@ -1,21 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.1-jdk
-
-COPY docker/docker-entrypoint.sh /usr/local/bin/
-
-ENV LOGFILE=/dev/stdout \
-      RACK_ENV=production
-
-RUN gem install vmpooler && \
-      chmod +x /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]

docker/Dockerfile-aio

@@ -1,32 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.1-jdk
-
-RUN mkdir -p /var/lib/vmpooler
-
-WORKDIR /var/lib/vmpooler
-
-ADD Gemfile* /var/lib/vmpooler/
-RUN bundle install --system
-
-RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby
-
-RUN echo "deb http://httpredir.debian.org/debian jessie main" >/etc/apt/sources.list.d/jessie-main.list
-RUN apt-get update && apt-get install -y redis-server && rm -rf /var/lib/apt/lists/*
-
-COPY . /var/lib/vmpooler
-
-ENV VMPOOLER_LOG /var/log/vmpooler.log
-CMD \
-    /etc/init.d/redis-server start \
-    && /var/lib/vmpooler/scripts/vmpooler_init.sh start \
-    && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \
-    tail -f ${VMPOOLER_LOG}

docker/docker-entrypoint.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -e
-
-set -- vmpooler "$@"
-
-exec "$@"

docs/API-v3.md

@@ -6,11 +6,36 @@
 5. [VM snapshots](#vmsnapshots)
 6. [Status and metrics](#statusmetrics)
 7. [Pool configuration](#poolconfig)
+8. [Ondemand VM provisioning](#ondemandvm)
 
 ### API <a name="API"></a>
 
 vmpooler provides a REST API for VM management.  The following examples use `curl` for communication.
 
+## Major change in V3 versus V2
+
+The api/v1 and api/v2 endpoints have been removed. Additionally, the generic api endpoint that reroutes to a versioned endpoint has been removed.
+
+The api/v3 endpoint removes the deprecated "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each pool instead of only a single domain for all pools, or a domain restricted to a particular provider.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will now now need to specify the versioned endpoint (api/v3).
+
+## Major change in V2 versus V1
+
+The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each provider instead of only one.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will still be redirected to v1, but this behavior is notw deprecated and will be changed to v2 in the next major version.
+
+### updavig clients from v1 to v2
+
+Clients need to update their paths to using api/v2 instead of api/v1. Please note the API responses that used to return a "domain" key, will no longer have a separate "domain" key and now return
+the FQDN (includes the domain in the hostname).
+
+One way to support both v1 and v2 in the client logic is to look for a "domain" and append it to the hostname if it exists (existing v1 behavior). If the "domain" key does not exist, you can use the hostname 
+as is since it is a FQDN (v2 behavor).
+
 #### Token operations <a name="token"></a>
 
 Token-based authentication can be used when requesting or modifying VMs.  The `/token` route can be used to create, query, or delete tokens.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations.
@@ -19,8 +44,13 @@ Token-based authentication can be used when requesting or modifying VMs.  The `/
 
 Get a list of issued tokens.
 
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found or other error
+
 ```
-$ curl -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -36,8 +66,13 @@ Enter host password for user 'jdoe':
 
 Generate a new authentication token.
 
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found
+
 ```
-$ curl -X POST -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -X POST -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -51,8 +86,12 @@ Enter host password for user 'jdoe':
 
 Get information about an existing token (including associated VMs).
 
+Return codes:
+* 200  OK
+* 404  when config:auth or token not found
+
 ```
-$ curl --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 ```
 ```json
 {
@@ -75,8 +114,13 @@ $ curl --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 
 Delete an authentication token.
 
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found
+
 ```
-$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -91,8 +135,11 @@ Enter host password for user 'jdoe':
 
 Retrieve a list of available VM pools.
 
+Return codes:
+* 200  OK
+
 ```
-$ curl --url vmpooler.example.com/api/v1/vm
+$ curl --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 [
@@ -103,26 +150,30 @@ $ curl --url vmpooler.example.com/api/v1/vm
 
 ##### POST /vm
 
-Useful for batch operations; post JSON (see format below), get back VMs.
+Useful for batch operations; post JSON (see format below), get back allocated VMs.
 
 If an authentication store is configured, an authentication token supplied via the `X-AUTH-TOKEN` HTTP header will modify a VM's default lifetime.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information.
 
+Return codes:
+* 200  OK
+* 404  when sending invalid JSON in the request body or requesting an invalid VM pool name
+* 503  when the vm failed to allocate a vm, or the pool is empty
+
 ```
-$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v1/vm
+$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "o41xtodlvnvu5cw",
-      "khirruvwfjlmx3y"
+      "o41xtodlvnvu5cw.example.com",
+      "khirruvwfjlmx3y.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "y91qbrpbfj6d13q"
-  },
-  "domain": "example.com"
+    "hostname": "y91qbrpbfj6d13q.example.com"
+  }
 }
 ```
 
@@ -132,23 +183,27 @@ $ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.c
 
 Check-out a VM or VMs.
 
+Return codes:
+* 200  OK
+* 404  when sending invalid JSON in the request body or requesting an invalid VM pool name
+* 503  when the vm failed to allocate a vm, or the pool is empty
+
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
-    "hostname": "fq6qlpjlsskycq6"
-  },
-  "domain": "example.com"
+    "hostname": "fq6qlpjlsskycq6.example.com"
+  }
 }
 ```
 
 Multiple VMs can be requested by using multiple query parameters in the URL:
 
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
 ```
 
 ```json
@@ -156,14 +211,13 @@ $ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debia
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "sc0o4xqtodlul5w",
-      "4m4dkhqiufnjmxy"
+      "sc0o4xqtodlul5w.example.com",
+      "4m4dkhqiufnjmxy.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "zb91y9qbrbf6d3q"
-  },
-  "domain": "example.com"
+    "hostname": "zb91y9qbrbf6d3q.example.com"
+  }
 }
 ```
 
@@ -171,10 +225,14 @@ $ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debia
 
 ##### GET /vm/&lt;hostname&gt;
 
-Query a checked-out VM.
+Query metadata information for a checked-out VM.
+
+Return codes:
+* 200  OK
+* 404  when requesting an invalid VM hostname
 
 ```
-$ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
+$ curl --url vmpooler.example.com/api/v3/vm/pxpmtoonx7fiqg6
 ```
 ```json
 {
@@ -182,13 +240,16 @@ $ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
   "pxpmtoonx7fiqg6": {
     "template": "centos-6-x86_64",
     "lifetime": 12,
-    "running": 3.1,
+    "running": 3,
+    "remaining": 9, 
     "state": "running",
     "tags": {
       "department": "engineering",
       "user": "jdoe"
     },
-    "domain": "example.com"
+    "ip": "192.168.0.1",
+    "host": "host1.example.com",
+    "migrated": "true"
   }
 }
 ```
@@ -208,8 +269,14 @@ Any modifications can be verified using the [GET /vm/&lt;hostname&gt;](#get-vmho
 
 If an authentication store is configured, an authentication token is required (via the `X-AUTH-TOKEN` HTTP header) to access this route.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information.
 
+Return codes:
+* 200  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+* 400  when supplied PUT parameters fail validation
+
 ```
-$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -218,7 +285,7 @@ $ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpj
 ```
 
 ```
-$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -230,8 +297,13 @@ $ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmp
 
 Schedule a checked-out VM for deletion.
 
+Return codes:
+* 200  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+
 ```
-$ curl -X DELETE --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X DELETE --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -245,8 +317,13 @@ $ curl -X DELETE --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
 
 Add an additional disk to a running VM.
 
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname or size is not an integer
+
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/disk/8
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/disk/8
 ````
 ````json
 {
@@ -260,7 +337,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -272,8 +349,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "disk": [
       "+8gb"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 
@@ -285,8 +361,13 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
 
 Create a snapshot of a running VM.
 
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot
 ````
 ````json
 {
@@ -300,7 +381,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -312,8 +393,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "snapshots": [
       "n4eb4kdtp7rwv4x158366vd9jhac8btq"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 ````
@@ -322,8 +402,13 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
 
 Revert a VM back to a snapshot.
 
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname or snapshot is not valid
+
 ````
-$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
+$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
 ````
 ````json
 {
@@ -338,7 +423,7 @@ $ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.ex
 A "live" status endpoint, representing the current state of the service.
 
 ```
-$ curl --url vmpooler.example.com/api/v1/status
+$ curl --url vmpooler.example.com/api/v3/status
 ```
 ```json
 {
@@ -386,6 +471,12 @@ If there are empty pools, the "status" section will convey this:
   }
 ```
 
+The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "status".
+If the query parameter 'view' is provided, it will be used to select which top level
+element to compute and return. Select them by specifying which one you want in a comma 
+separated list.
+For example `vmpooler.example.com/api/v3/status?view=capacity,boot`
+
 ##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]]
 
 Returns a summary, or report, for the timespan between `from` and `to` (inclusive)
@@ -395,8 +486,13 @@ metrics, such as boot and cloning durations.
 Any omitted query parameter will default to now/today. A request without any
 parameters will result in the current day's summary.
 
+Return codes:
+* 200  OK
+* 400  Invalid date format or range
+
+
 ```
-$ curl --url vmpooler.example.com/api/v1/summary
+$ curl --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -486,8 +582,9 @@ $ curl --url vmpooler.example.com/api/v1/summary
 }
 ```
 
+
 ```
-$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v1/summary
+$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -550,17 +647,69 @@ $ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api
 }
 ```
 
+You can also query only the specific top level section you want by including it after `summary/`.
+The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v3/summary/boot/`.
+You can further drill-down the data by specifying the second level parameter to query eg
+`vmpooler.example.com/api/v3/summary/tag/created_by`
+
+##### GET /poolstat?pool=FOO
+
+For parameter `pool`, containing a comma separated list of pool names to query, this endpoint returns
+each of the pool's ready, max and alias information. It can be used to get a fast response for
+the required pools instead of using the /status API endpoint
+
+Return codes
+* 200  OK 
+
+```
+$ curl https://vmpooler.example.com/api/v3/poolstat?pool=centos-6-x86_64
+```
+```json
+{
+  "pools": {
+    "centos-6-x86_64": {
+      "ready": 25,
+      "max": 25,
+      "alias": [
+        "centos-6-64",
+        "centos-6-amd64"
+      ]
+    }
+  }
+}
+```
+
+##### GET /totalrunning
+
+Fast endpoint to return the total number of VMs in a 'running' state
+
+Return codes
+* 200  OK 
+
+```
+$ curl https://vmpooler.example.com/api/v3/totalrunning
+```
+
+```json
+{
+
+  "running": 362
+
+}
+```
+
 #### Managing pool configuration via API <a name="poolconfig"></a>
 
 ##### GET /config
 
 Returns the running pool configuration
 
-Responses:
-* 200 - OK
-* 404 - No configuration found
+Return codes
+* 200  OK 
+* 400  No configuration found  
+
 ```
-$ curl https://vmpooler.example.com/api/v1/config
+$ curl https://vmpooler.example.com/api/v3/config
 ```
 ```json
 {
@@ -604,7 +753,7 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v1/config/poolsize
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v3/config/poolsize
 ```
 ```json
 {
@@ -612,6 +761,28 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","deb
 }
 ```
 
+##### DELETE /config/poolsize/&lt;pool&gt;
+
+Delete an overridden pool size. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - size reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/poolsize/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "pool_size_before_overrides": 2,
+    "pool_size_before_reset": 4
+}
+```
+
 ##### POST /config/pooltemplate
 
 Change the template configured for a pool, and replenish the pool with instances built from the new template.
@@ -636,7 +807,145 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v1/config/pooltemplate
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v3/config/pooltemplate
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### DELETE /config/pooltemplate/&lt;pool&gt;
+
+Delete an overridden pool template. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - template reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/pooltemplate/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2",
+    "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta"
+}
+```
+
+##### POST /poolreset
+
+Clear all pending and ready instances in a pool, and deploy replacements
+
+All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned.
+
+When a pool reset is requested a 201 status will be returned.
+
+A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances.
+
+For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured.
+
+Responses:
+* 201 - Pool reset requested received
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v3/poolreset
+```
+```json
+{
+  "ok": true
+}
+```
+
+#### Ondemand VM operations <a name="ondemandvm"></a>
+
+Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated.
+
+##### POST /ondemandvm
+
+All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned
+
+When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status.
+
+An authentication token is required in order to request instances on demand when authentication is configured.
+
+Responses:
+* 201 - Provisioning request accepted
+* 400 - Payload contains invalid JSON and cannot be parsed
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 403 - Request exceeds the configured per pool maximum
+* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred.
+* 409 - A request of the matching ID has already been created
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v3/ondemandvm
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a"
+}
+```
+
+##### GET /ondemandvm
+
+Get the status of an ondemandvm request that has already been posted.
+
+When the request is ready the ready status will change to 'true'.
+
+The number of instances pending vs ready will be reflected in the API response.
+
+Responses:
+* 200 - The API request was successful and the status is ok
+* 202 - The request is not ready yet
+* 404 - The request can not be found, or an unknown error occurred
+```
+$ curl https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": false,
+  "debian-7-i386": {
+    "ready": "3",
+    "pending": "1"
+  }
+}
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": true,
+  "debian-7-i386": {
+    "hostname": [
+      "vm1",
+      "vm2",
+      "vm3",
+      "vm4"
+    ]
+  }
+}
+```
+
+##### DELETE /ondemandvm
+
+Delete a ondemand request
+
+Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted.
+
+Responses:
+* 200 - The API request was sucessful. A message will indicate if the request has already been deleted.
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 404 - The request can not be found, or an unknown error occurred.
+```
+$ curl -X DELETE https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
 ```
 ```json
 {

docs/PROVIDER_API.md

@@ -0,0 +1,100 @@
+# Provider API
+
+Providers facilitate VMPooler interacting with some other system that can create virtual machines. A single VMPooler instance can utilize one or more providers and can have multiple instances of the same provider. An example of having multiple instances of the same provider is when you need to interact with multiple vCenters from the same VMPooler instance.
+
+## Known Providers
+
+- `vmpooler-provider-vsphere` provides the ability to use VMware as a source of VMs. Its code can be found in the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository.
+
+Know of others? Please submit a pull request to update this list or reach out to us on the Puppet community Slack.
+
+Want to create a new one? See below!
+
+## Create a new provider gem from scratch
+
+### Requirements
+
+1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
+2. the main provider code file should be named the same at the name of the provider. For example, the `vpshere` provider's main file is `lib/vmpooler/providers/vsphere.rb`.
+3. The gem must be installed on the same machine as VMPooler
+4. The provider name must be referenced in the VMPooler config file in order for it to be loaded.
+5. Your gem name and repository name should be `vmpooler-provider-<provider name>` so the community can easily search provider plugins.
+
+The resulting directory structure should resemble this:
+
+```bash
+lib/
+├── vmpooler/
+│   └── providers/
+│       └── <provider name>.rb
+└── vmpooler-provider-<provider name>/
+    └── version.rb
+```
+
+### 1. Use bundler to create the provider scaffolding
+
+```bash
+bundler gem --test=rspec --no-exe --no-ext vmpooler-provider-spoof
+cd vmpooler-providers-spoof/
+mkdir -p ./lib/vmpooler/providers
+touch ./lib/vmpooler/providers/spoof.rb
+mkdir ./lib/vmpooler-providers-spoof
+touch ./lib/vmpooler-providers-spoof/version.rb
+```
+
+There may be some boilerplate files generated, just delete those.
+
+### 2. Create the main provider file
+
+Ensure the main provider file uses the following code.
+
+```ruby
+# lib/vmpooler/providers/spoof.rb
+require 'yaml'
+require 'vmpooler/providers/base'
+
+module Vmpooler
+  class PoolManager
+    class Provider
+      class Spoof < Vmpooler::PoolManager::Provider::Base
+        # At this time it is not documented which methods should be implemented
+        # have a look at the https://github.com/puppetlabs/vmpooler-provider-vsphere
+        #for an example
+      end
+    end
+  end
+end
+```
+
+### 3. Create the version file
+
+Ensure you have a version file similar this:
+
+```ruby
+# frozen_string_literal: true
+# lib/vmpooler-provider-vsphere/version.rb 
+
+module VmpoolerProviderSpoof
+  VERSION = '1.0.0'
+end
+```
+
+### 4. Fill out your gemspec
+
+Ensure you fill out your gemspec file to your specifications.  If you need a dependency, please make sure you require it.
+
+`spec.add_dependency "foo", "~> 1.15"`.
+
+At a minimum you may want to add the `vmpooler` gem as a dev dependency so you can use it during testing.
+
+`spec.add_dev_dependency "vmpooler", "~> 2.0"`
+
+Also make sure this dependency can be loaded by JRuby.  If the dependency cannot be used by JRuby don't use it.
+
+### 5. Create some tests
+
+Your provider code should be tested before releasing. Copy and refactor some tests from the `vmpooler` gem under `spec/unit/providers/dummy_spec.rb`.
+
+### 6. Publish
+
+Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.

docs/README.md

@@ -1,5 +0,0 @@
-# Documentation for vmpooler
-
-
-* [Setting up a Development Environment](dev-setup.md)
-* [API Documentation](API.md)

docs/configuration.md

@@ -19,11 +19,6 @@ Provide the entire configuration as a blob of yaml. Individual parameters passed
 
 Path to a the file to use when loading the vmpooler configuration. This is only evaluated if `VMPOOLER_CONFIG` has not been specified.
 
-### DOMAIN\_NAME
-
-If set, returns a top-level 'domain' JSON key in POST requests
-(optional)
-
 ### REDIS\_SERVER
 
 The redis server to use for vmpooler.
@@ -74,6 +69,16 @@ The prefix to use while storing Graphite data.
 The TCP port to communicate with the graphite server.
 (optional; default: 2003)
 
+### MAX\_ONDEMAND\_INSTANCES\_PER\_REQUEST
+
+The maximum number of instances any individual ondemand request may contain per pool.
+(default: 10)
+
+### ONDEMAND\_REQUEST\_TTL
+
+The amount of time (in minutes) to give for a ondemand request to be fulfilled before considering it to have failed.
+(default: 5)
+
 ## Manager options <a name="manager"></a>
 
 ### TASK\_LIMIT
@@ -103,7 +108,7 @@ Same as `vm_lifetime`, but applied if a valid authentication token is
 included during the request.
 (required)
 
-### VM\_PREFIX
+### PREFIX
 
 If set, prefixes all created VMs with this string. This should include a separator.
 (optional; default: '')
@@ -123,6 +128,11 @@ The target cluster VMs are cloned into (host with least VMs chosen)
 How long (in minutes) before marking a clone as 'failed' and retrying.
 (optional; default: 15)
 
+### READY\_TTL
+
+How long (in minutes) a ready VM should stay in the ready queue.
+(default: 60)
+
 ### MAX\_TRIES
 
 Set the max number of times a connection should retry in VM providers. This optional setting allows a user to dial in retry limits to suit your environment.
@@ -130,7 +140,7 @@ Set the max number of times a connection should retry in VM providers. This opti
 
 ### RETRY\_FACTOR
 
-When retrying, each attempt sleeps for the try count * retry_factor.
+When retrying, each attempt sleeps for the try count * retry\_factor.
 Increase this number to lengthen the delay between retry attempts.
 This is particularly useful for instances with a large number of pools
 to prevent a thundering herd when retrying connections.
@@ -147,15 +157,67 @@ Note: this will only create the last folder when it does not exist. It will not
 Create backing delta disks for the specified templates to support creating linked clones.
 (optional; default: false)
 
+### CREATE\_LINKED\_CLONES
+
+Whether to create linked clone virtual machines when using the vsphere provider.
+This can also be set per pool.
+(optional; default: true)
+
 ### PURGE\_UNCONFIGURED\_FOLDERS
 
-Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+Deprecated, see PURGE\_UNCONFIGURED\_RESOURCES
+
+### PURGE\_UNCONFIGURED\_RESOURCES
+
+Enable purging of VMs (and other resources) detected within the provider that are not explicitly configured.
+Implementation is provider-dependent
 When enabled in the global configuration then purging is enabled for all providers
 Expects a boolean value
 (optional; default: false)
 
+### USAGE\_STATS
+
+Enable shipping of VM usage stats
+When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize
+shipped metrics if there is a jenkins\_build\_url tag set for the VM.
+Without the jenkins\_build\_url tag set the metric will be sent as "usage.$user.$pool\_name".
+When the jenkins\_build\_url tag is set the metric will be sent with additional data. Here is an example
+based off of the following URL;
+https://jenkins.example.com/job/platform\_puppet-agent-extra\_puppet-agent-integration-suite\_pr/RMM\_COMPONENT\_TO\_TEST\_NAME=puppet,SLAVE\_LABEL=beaker,TEST\_TARGET=redhat7-64a/824/
+"usage.$user.$instance.$value\_stream.$branch.$project.$job\_name.$component\_to\_test.$pool\_name", which translates to
+"usage.$user.jenkins\_example\_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool\_name"
+Expects a boolean value
+(optional; default: false)
+
+### REQUEST\_LOGGER
+
+Enable API request logging to the logger
+When enabled all requests to the API are written to the standard logger.
+Expects a boolean value
+(optional; default: false)
+
+### EXTRA\_CONFIG
+
+Specify additional application configuration files
+The argument can accept a full path to a file, or multiple files comma separated.
+Expects a string value
+(optional)
+
+### ONDEMAND\_CLONE\_LIMIT
+
+Maximum number of simultaneous clones to perform for ondemand provisioning requests.
+(default: 10)
+
+### REDIS\_CONNECTION\_POOL\_SIZE
+
+Maximum number of connections to utilize for the redis connection pool.
+(default: 10)
+
+### REDIS\_CONNECTION\_POOL\_TIMEOUT
+
+How long a task should wait (in seconds) for a redis connection when all connections are in use.
+(default: 5)
+
 ## API options <a name="API"></a>
 
 ### AUTH\_PROVIDER
@@ -184,6 +246,18 @@ This can be a string providing a single DN. For multiple DNs please specify the
 The LDAP object-type used to designate a user object.
 (optional)
 
+### LDAP\_SERVICE_ACCOUNT\_HASH
+
+A hash containing the following parameters for a service account to perform the
+initial bind. After the initial bind, then a search query is performed using the
+'base' and 'user_object', then re-binds as the returned user.
+
+- :user_dn: The full distinguished name (DN) of the service account used to bind.
+
+- :password: The password for the service account used to bind.
+
+(optional)
+
 ### SITE\_NAME
 
 The name of your deployment.
@@ -194,3 +268,8 @@ The name of your deployment.
 Enable experimental API capabilities such as changing pool template and size without application restart
 Expects a boolean value
 (optional; default: false)
+
+### MAX\_LIFETIME\_UPPER\_LIMIT
+
+Specify a maximum lifetime that a VM may be extended to in hours.
+(optional)

docs/dev-setup.md

@@ -1,235 +0,0 @@
-# Setting up a vmpooler development environment
-
-## Requirements
-
-* Supported on OSX, Windows and Linux
-
-* Ruby or JRuby
-
-  Note - Ruby 1.x support will be removed so it is best to use more modern ruby versions
-
-  Note - It is recommended to user Bundler instead of installing gems into the system repository
-
-* A local Redis server
-
-  Either a containerized instance of Redis or a local version is fine.
-
-## Setup source and ruby
-
-* Clone repository, either from your own fork or the original source
-
-* Perform a bundle install
-
-```
-~/ > git clone https://github.com/puppetlabs/vmpooler.git
-Cloning into 'vmpooler'...
-remote: Counting objects: 3411, done.
-...
-
-~/ > cd vmpooler
-
-~/vmpooler/ > bundle install
-Fetching gem metadata from https://rubygems.org/.........
-Fetching version metadata from https://rubygems.org/..
-Resolving dependencies...
-Installing rake 12.0.0
-...
-Bundle complete! 16 Gemfile dependencies, 37 gems now installed.
-```
-
-## Setup environment variables
-
-### `VMPOOLER_DEBUG`
-
-Setting the `VMPOOLER_DEBUG` environment variable will instruct vmpooler to:
-
-* Output log messages to STDOUT
-
-* Allow the use of the dummy authentication method
-
-* Add interrupt traps so you can stop vmpooler when run interactively
-
-Linux, OSX
-```bash
-~/vmpooler/ > export VMPOOLER_DEBUG=true
-```
-
-Windows (PowerShell)
-```powershell
-C:\vmpooler > $ENV:VMPOOLER_DEBUG = 'true'
-```
-
-
-### `VMPOOLER_CONFIG`
-
-When `VMPOOLER_CONFIG` is set, vmpooler will read its configuration from the content of the environment variable.
-
-Note that this variable does not point to a different configuration file, but stores the contents of a configuration file. You may use `VMPOOLER_CONFIG_FILE` instead to specify a filename.
-
-
-### `VMPOOLER_CONFIG_FILE`
-
-When `VMPOOLER_CONFIG_FILE` is set, vmpooler will read its configuration from the file specified in the environment variable.
-
-Note that this variable points to a different configuration file, unlike `VMPOOLER_CONFIG`.
-
-
-## Setup vmpooler Configuration
-
-You can create a `vmpooler.yaml` file, set the `VMPOOLER_CONFIG` environment variable with the equivalent content, or set the `VMPOOLER_CONFIG_FILE` environment variable with the name of another configuration file to use. `VMPOOLER_CONFIG` takes precedence over `VMPOOLER_CONFIG_FILE`.
-
-Example minimal configuration file:
-```yaml
-
----
-:providers:
-  :dummy:
-
-:redis:
-  server: 'localhost'
-
-:auth:
-   provider: dummy
-
-:tagfilter:
-  url: '(.*)\/'
-
-:config:
-  site_name: 'vmpooler'
-  # Need to change this on Windows
-  logfile: '/var/log/vmpooler.log'
-  task_limit: 10
-  timeout: 15
-  vm_checktime: 15
-  vm_lifetime: 12
-  vm_lifetime_auth: 24
-  allowed_tags:
-    - 'created_by'
-    - 'project'
-  domain: 'example.com'
-  prefix: 'poolvm-'
-
-# Uncomment the lines below to suppress metrics to STDOUT
-# :statsd:
-#   server: 'localhost'
-#   prefix: 'vmpooler'
-#   port: 8125
-
-:pools:
-  - name: 'pool01'
-    size: 5
-    provider: dummy
-  - name: 'pool02'
-    size: 5
-    provider: dummy
-```
-
-## Running vmpooler locally
-
-* Run `bundle exec ruby vmpooler`
-
-  If using JRuby, you may need to use `bundle exec jruby vmpooler`
-
-You should see output similar to:
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-[2017-06-16 14:50:31] starting vmpooler
-[2017-06-16 14:50:31] [!] Creating provider 'dummy'
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection pool of size 1 with timeout 10
-[2017-06-16 14:50:31] [*] [disk_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [snapshot_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [pool01] starting worker thread
-[2017-06-16 14:50:31] [*] [pool02] starting worker thread
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection object ID 1784
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-[2017-06-16 14:50:31] [!] [pool02] is empty
-[2017-06-16 14:50:31] [!] [pool01] is empty
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nexs1w50m4djap5'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-r543eibo4b6tjer'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-neqmu7wj7aukyjy'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nsdnrhhy22lnemo'
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-r543eibo4b6tjer' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-neqmu7wj7aukyjy' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool02] 'poolvm-nexs1w50m4djap5' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-edzlp954lyiozli'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nb0uci0yrwbxr6x'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-y2yxgnovaneymvy'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nur59d25s1y8jko'
-...
-```
-
-### Common Errors
-
-* Forget to set VMPOOLER_DEBUG environment variable
-
-vmpooler will fail to start with an error similar to below
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-
-~/vmpooler/lib/vmpooler.rb:44:in `config': Dummy authentication should not be used outside of debug mode; please set environment variable VMPOOLER_DEBUG to 'true' if you want to use dummy authentication (RuntimeError)
-        from vmpooler:8:in `<main>'
-...
-```
-
-* Error in vmpooler configuration
-
-If there is an error in the vmpooler configuration file, or any other fatal error in the Pool Manager, vmpooler will appear to be running but no log information is displayed.  This is due to the error not being displayed until you press `Ctrl-C` and then suddenly you can see the cause of the issue.
-
-For example, when running vmpooler on Windows, but with a unix style filename for the vmpooler log
-
-```powershell
-C:\vmpooler > bundle exec ruby vmpooler
-[2017-06-16 14:49:57] starting vmpooler
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-
-# [ NOTHING ELSE IS LOGGED ]
-```
-
-Once `Ctrl-C` is pressed the error is shown
-
-```powershell
-...
-== Sinatra has ended his set (crowd applauds)
-Shutting down.
-C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `initialize': No such file or directory @ rb_sysopen - /var/log/vmpooler.log (Errno::ENOENT)
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/vmpooler/lib/vmpooler/logger.rb:17:in `log'
-        from C:/vmpooler/lib/vmpooler/pool_manager.rb:709:in `execute!'
-        from vmpooler:26:in `block in <main>'
-```
-
-## Default vmpooler URLs
-
-| Endpoint  | URL                                                                  |
-|-----------|----------------------------------------------------------------------|
-| Dashboard | [http://localhost:4567/dashboard/](http://localhost:4567/dashboard/) |
-| API       | [http://localhost:4567/api/v1]([http://localhost:4567/api/v1)        |
-
-## Use the vmpooler API locally
-
-Once a local vmpooler instance is running you can use any tool you need to interact with the API.  The dummy authentication provider will allow a user to connect if the username and password are not the same:
-
-* Authentication is successful for username `Alice` with password `foo`
-
-* Authentication will fail for username `Alice` with password `Alice`
-
-Like normal vmpooler, tokens will be created for the user and can be used for regular vmpooler operations.

docs/vagrant.md

@@ -1,45 +0,0 @@
-A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer.
-To use it run:
-
-```
-vagrant up
-vagrant ssh
-docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler
-```
-
-To run vmpooler with the example dummy provider you can replace the above docker command with this:
-
-```
-docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler
-```
-
-Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/).
-
-### Running directly in Vagrant
-
-You can also run vmpooler directly in the Vagrant box. To do so run this:
-
-```
-vagrant up
-vagrant ssh
-cd /vagrant
-
-# Do this if using the dummy provider
-export VMPOOLER_DEBUG=true
-cp vmpooler.yaml.dummy-example vmpooler.yaml
-
-# vmpooler needs a redis server.
-sudo yum -y install redis
-sudo systemctl start redis
-
-# Optional: Choose your ruby version or use jruby
-# ruby 2.4.x is used by default
-rvm list
-rvm use jruby-9.1.7.0
-
-gem install bundler
-bundle install
-bundle exec ruby vmpooler
-```
-
-When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/).

examples/vmpooler.yaml.dummy-example.aliasedpools

@@ -17,15 +17,20 @@
   logfile: '/Users/samuel/workspace/vmpooler/vmpooler.log'
   task_limit: 10
   timeout: 15
-  vm_checktime: 15
+  timeout_notification: 5
+  vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 :pools:
   - name: 'debian-7-i386'
     alias: [ 'debian-7-32' ]
@@ -34,8 +39,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-stringalias'
     alias: 'debian-7-32-stringalias'
     template: 'Templates/debian-7-i386'
@@ -43,8 +50,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64'
     alias: [ 'debian-7-64', 'debian-7-amd64' ]
     template: 'Templates/debian-7-x86_64'
@@ -52,16 +61,20 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-noalias'
     template: 'Templates/debian-7-i386'
     folder: 'Pooled VMs/debian-7-i386'
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64-alias-otherpool-extended'
     alias: [ 'debian-7-x86_64' ]
     template: 'Templates/debian-7-x86_64'
@@ -69,6 +82,7 @@
     datastore: 'other-vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
-
+    dns_plugin: 'example'

lib/vmpooler.rb

@@ -1,10 +1,13 @@
+# frozen_string_literal: true
+
 module Vmpooler
+  require 'concurrent'
   require 'date'
+  require 'deep_merge'
   require 'json'
   require 'net/ldap'
   require 'open-uri'
   require 'pickup'
-  require 'rbvmomi'
   require 'redis'
   require 'set'
   require 'sinatra/base'
@@ -12,7 +15,16 @@ module Vmpooler
   require 'timeout'
   require 'yaml'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
+  # Dependencies for tracing
+  require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-http_client'
+  require 'opentelemetry-instrumentation-redis'
+  require 'opentelemetry-instrumentation-sinatra'
+  require 'opentelemetry-sdk'
+  require 'opentelemetry/exporter/jaeger'
+  require 'opentelemetry/resource/detectors'
+
+  %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
 
@@ -21,21 +33,28 @@ module Vmpooler
     if ENV['VMPOOLER_CONFIG']
       config_string = ENV['VMPOOLER_CONFIG']
       # Parse the YAML config into a Hash
-      # Whitelist the Symbol class
-      parsed_config = YAML.safe_load(config_string, [Symbol])
+      # Allow the Symbol class
+      parsed_config = YAML.safe_load(config_string, permitted_classes: [Symbol])
     else
       # Take the name of the config file either from an ENV variable or from the filepath argument
       config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
       parsed_config = YAML.load_file(config_file) if File.exist? config_file
+      parsed_config[:config]['extra_config'] = ENV['EXTRA_CONFIG'] if ENV['EXTRA_CONFIG']
+      if parsed_config[:config]['extra_config']
+        extra_configs = parsed_config[:config]['extra_config'].split(',')
+        extra_configs.each do |config|
+          puts "loading extra_config file #{config}"
+          extra_config = YAML.load_file(config)
+          parsed_config.deep_merge(extra_config)
+        end
+      end
     end
 
     parsed_config ||= { config: {} }
 
     # Bail out if someone attempts to start vmpooler with dummy authentication
     # without enbaling debug mode.
-    if parsed_config.has_key? :auth
-      if parsed_config[:auth]['provider'] == 'dummy'
-        unless ENV['VMPOOLER_DEBUG']
+    if parsed_config.key?(:auth) && parsed_config[:auth]['provider'] == 'dummy' && !ENV['VMPOOLER_DEBUG']
       warning = [
         'Dummy authentication should not be used outside of debug mode',
         'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
@@ -43,84 +62,120 @@ module Vmpooler
 
       raise warning.join(";\s")
     end
-      end
-    end
 
     # Set some configuration defaults
-    parsed_config[:config]['task_limit'] = ENV['TASK_LIMIT'] || parsed_config[:config]['task_limit'] || 10
-    parsed_config[:config]['migration_limit'] = ENV['MIGRATION_LIMIT'] if ENV['MIGRATION_LIMIT']
-    parsed_config[:config]['vm_checktime'] = ENV['VM_CHECKTIME'] || parsed_config[:config]['vm_checktime'] || 15
-    parsed_config[:config]['vm_lifetime'] = ENV['VM_LIFETIME'] || parsed_config[:config]['vm_lifetime']  || 24
-    parsed_config[:config]['prefix'] = ENV['VM_PREFIX'] || parsed_config[:config]['prefix'] || ''
-
+    parsed_config[:config]['task_limit']                         = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit']               = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
+    parsed_config[:config]['migration_limit']                    = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
+    parsed_config[:config]['vm_checktime']                       = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
+    parsed_config[:config]['vm_lifetime']                        = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit']           = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl']                          = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl']               = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
+    parsed_config[:config]['prefix']                             = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
     parsed_config[:config]['logfile']                            = ENV['LOGFILE'] if ENV['LOGFILE']
-
     parsed_config[:config]['site_name']                          = ENV['SITE_NAME'] if ENV['SITE_NAME']
-    parsed_config[:config]['domain'] = ENV['DOMAIN_NAME'] if ENV['DOMAIN_NAME']
+    if !parsed_config[:config]['domain'].nil? || !ENV['DOMAIN'].nil?
+      puts '[!] [error] The "domain" config setting has been removed in v3. Please see the docs for migrating the domain config to use a dns plugin at https://github.com/puppetlabs/vmpooler/blob/main/README.md#migrating-to-v3'
+      exit 1
+    end
     parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
-    parsed_config[:config]['timeout'] = ENV['TIMEOUT'] if ENV['TIMEOUT']
-    parsed_config[:config]['vm_lifetime_auth'] = ENV['VM_LIFETIME_AUTH'] if ENV['VM_LIFETIME_AUTH']
-    parsed_config[:config]['max_tries'] = ENV['MAX_TRIES'] if ENV['MAX_TRIES']
-    parsed_config[:config]['retry_factor'] = ENV['RETRY_FACTOR'] if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
-    parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['timeout_notification']               = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
+    parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
+    parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
+    parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
+    parsed_config[:config]['create_folders']                     = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
     parsed_config[:config]['experimental_features']              = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
-    parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    parsed_config[:config]['usage_stats']                        = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
+    parsed_config[:config]['request_logger']                     = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
+    parsed_config[:config]['create_template_delta_disks']        = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_RESOURCES'] if ENV['PURGE_UNCONFIGURED_RESOURCES']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    # ENV PURGE_UNCONFIGURED_FOLDERS deprecated, will be removed in version 3
+    puts '[!] [deprecation] rename ENV var \'PURGE_UNCONFIGURED_FOLDERS\' to \'PURGE_UNCONFIGURED_RESOURCES\'' if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    set_linked_clone(parsed_config)
 
     parsed_config[:redis]                            = parsed_config[:redis] || {}
     parsed_config[:redis]['server']                  = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
-    parsed_config[:redis]['port'] = ENV['REDIS_PORT'] if ENV['REDIS_PORT']
+    parsed_config[:redis]['port']                    = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
     parsed_config[:redis]['password']                = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
-    parsed_config[:redis]['data_ttl'] = ENV['REDIS_DATA_TTL'] || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['data_ttl']                = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size']    = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+    parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
+    parsed_config[:redis]['reconnect_attempts']      = string_array_to_array(ENV['REDIS_RECONNECT_ATTEMPTS']) || parsed_config[:redis]['reconnect_attempts'] || 10
 
     parsed_config[:statsd]           = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
     parsed_config[:statsd]['prefix'] = ENV['STATSD_PREFIX'] if ENV['STATSD_PREFIX']
-    parsed_config[:statsd]['port'] = ENV['STATSD_PORT'] if ENV['STATSD_PORT']
+    parsed_config[:statsd]['port']   = string_to_int(ENV['STATSD_PORT']) if ENV['STATSD_PORT']
 
     parsed_config[:graphite]           = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
-    parsed_config[:graphite]['port'] = ENV['GRAPHITE_PORT'] if ENV['GRAPHITE_PORT']
+    parsed_config[:graphite]['port']   = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
+
+    parsed_config[:tracing]                = parsed_config[:tracing] || {}
+    parsed_config[:tracing]['enabled']     = ENV['VMPOOLER_TRACING_ENABLED'] || parsed_config[:tracing]['enabled'] || 'false'
+    parsed_config[:tracing]['jaeger_host'] = ENV['VMPOOLER_TRACING_JAEGER_HOST'] || parsed_config[:tracing]['jaeger_host'] || 'http://localhost:14268/api/traces'
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
-    if parsed_config.has_key? :auth
+    if parsed_config.key? :auth
       parsed_config[:auth]['provider']           = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
       parsed_config[:auth][:ldap]                = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
       parsed_config[:auth][:ldap]['host']        = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
-      parsed_config[:auth][:ldap]['port'] = ENV['LDAP_PORT'] if ENV['LDAP_PORT']
+      parsed_config[:auth][:ldap]['port']        = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
       parsed_config[:auth][:ldap]['base']        = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
+      if parsed_config[:auth]['provider'] == 'ldap' && parsed_config[:auth][:ldap].key?('encryption')
+        parsed_config[:auth][:ldap]['encryption'] = parsed_config[:auth][:ldap]['encryption']
+      elsif parsed_config[:auth]['provider'] == 'ldap'
+        parsed_config[:auth][:ldap]['encryption'] = {}
+      end
     end
 
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
     unless parsed_config[:pools]
+      puts 'loading pools configuration from redis, since the config[:pools] is empty'
       redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
       parsed_config[:pools] = load_pools_from_redis(redis)
     end
 
+    # Marshal.dump is paired with Marshal.load to create a copy that has its own memory space
+    # so that each can be edited independently
+    # rubocop:disable Security/MarshalLoad
+
+    # retain a copy of the pools that were observed at startup
+    serialized_pools = Marshal.dump(parsed_config[:pools])
+    parsed_config[:pools_at_startup] = Marshal.load(serialized_pools)
+
+    # Create an index of pools by title
+    parsed_config[:pool_index] = pool_index(parsed_config[:pools])
+    # rubocop:enable Security/MarshalLoad
+
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
+      pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
-        if pool['alias'].is_a?(Array)
-          pool['alias'].each do |a|
+        if pool['alias'].instance_of?(Array)
+          pool['alias'].each do |pool_alias|
             parsed_config[:alias] ||= {}
-            parsed_config[:alias][a] = pool['name']
-            parsed_config[:pool_names] << a
+            parsed_config[:alias][pool_alias] = [pool['name']] unless parsed_config[:alias].key? pool_alias
+            parsed_config[:alias][pool_alias] << pool['name'] unless parsed_config[:alias][pool_alias].include? pool['name']
+            parsed_config[:pool_names] << pool_alias
           end
-        elsif pool['alias'].is_a?(String)
+        elsif pool['alias'].instance_of?(String)
           parsed_config[:alias][pool['alias']] = pool['name']
           parsed_config[:pool_names] << pool['alias']
         end
       end
     end
 
-    if parsed_config[:tagfilter]
-      parsed_config[:tagfilter].keys.each do |tag|
+    parsed_config[:tagfilter]&.keys&.each do |tag|
       parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
     end
-    end
 
     parsed_config[:uptime] = Time.now
 
@@ -140,25 +195,104 @@ module Vmpooler
     pools
   end
 
-  def self.new_redis(host = 'localhost', port = nil, password = nil)
-    Redis.new(host: host, port: port, password: password)
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics, redis_reconnect_attempts = 0)
+    Vmpooler::PoolManager::GenericConnectionPool.new(
+      metrics: metrics,
+      connpool_type: 'redis_connection_pool',
+      connpool_provider: 'manager',
+      size: size,
+      timeout: timeout
+    ) do
+      connection = Concurrent::Hash.new
+      redis = new_redis(host, port, password, redis_reconnect_attempts)
+      connection['connection'] = redis
+    end
   end
 
-  def self.new_logger(logfile)
-    Vmpooler::Logger.new logfile
-  end
-
-  def self.new_metrics(params)
-    if params[:statsd]
-      Vmpooler::Statsd.new(params[:statsd])
-    elsif params[:graphite]
-      Vmpooler::Graphite.new(params[:graphite])
-    else
-      Vmpooler::DummyStatsd.new
-    end
+  def self.new_redis(host = 'localhost', port = nil, password = nil, redis_reconnect_attempts = 10)
+    Redis.new(
+      host: host,
+      port: port,
+      password: password,
+      reconnect_attempts: redis_reconnect_attempts,
+      connect_timeout: 300
+    )
   end
 
   def self.pools(conf)
     conf[:pools]
   end
+
+  def self.pool_index(pools)
+    pools_hash = {}
+    index = 0
+    pools.each do |pool|
+      pools_hash[pool['name']] = index
+      index += 1
+    end
+    pools_hash
+  end
+
+  def self.string_to_int(s)
+    # Returns a integer if input is a string
+    return if s.nil?
+    return unless s =~ /\d/
+
+    Integer(s)
+  end
+
+  def self.string_array_to_array(s)
+    # Returns an array from an array like string
+    return if s.nil?
+
+    JSON.parse(s)
+  end
+
+  def self.true?(obj)
+    obj.to_s.downcase == 'true'
+  end
+
+  def self.set_linked_clone(parsed_config) # rubocop:disable Naming/AccessorMethodName
+    parsed_config[:config]['create_linked_clones'] = parsed_config[:config]['create_linked_clones'] || true
+    parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/
+    parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones']
+  end
+
+  def self.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+    if startup_args.length == 1 && startup_args.include?('api')
+      service_name = 'vmpooler-api'
+    elsif startup_args.length == 1 && startup_args.include?('manager')
+      service_name = 'vmpooler-manager'
+    else
+      service_name = 'vmpooler'
+    end
+
+    service_name += "-#{prefix}" unless prefix.empty?
+
+    if tracing_enabled.eql?('false')
+      puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
+      )
+    else
+      puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
+      )
+    end
+
+    OpenTelemetry::SDK.configure do |c|
+      c.use 'OpenTelemetry::Instrumentation::Sinatra'
+      c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::HttpClient'
+      c.use 'OpenTelemetry::Instrumentation::Redis'
+
+      c.add_span_processor(span_processor)
+
+      c.service_name = service_name
+      c.service_version = version
+
+      c.resource = OpenTelemetry::Resource::Detectors::AutoDetector.detect
+    end
+  end
 end

lib/vmpooler/api.rb

@@ -1,9 +1,25 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API < Sinatra::Base
-    def initialize
-      super
+    # Load API components
+    %w[helpers dashboard v3 request_logger healthcheck].each do |lib|
+      require "vmpooler/api/#{lib}"
     end
+    # Load dashboard components
+    require 'vmpooler/dashboard'
 
+    def self.execute(torun, config, redis, metrics, logger)
+      self.settings.set :config, config
+      self.settings.set :redis, redis unless redis.nil?
+      self.settings.set :metrics, metrics
+      self.settings.set :checkoutlock, Mutex.new
+
+      # Deflating in all situations
+      # https://www.schneems.com/2017/11/08/80-smaller-rails-footprint-with-rack-deflate/
+      use Rack::Deflater
+
+      # not_found clause placed here to fix rspec test issue.
       not_found do
         content_type :json
 
@@ -14,40 +30,33 @@ module Vmpooler
         JSON.pretty_generate(result)
       end
 
-    get '/' do
-      redirect to('/dashboard/')
+      if metrics.respond_to?(:setup_prometheus_metrics)
+        # Prometheus metrics are only setup if actually specified
+        # in the config file.
+        metrics.setup_prometheus_metrics(torun)
+
+        # Using customised collector that filters out hostnames on API paths
+        require 'vmpooler/metrics/promstats/collector_middleware'
+        require 'prometheus/middleware/exporter'
+        use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.prometheus_prefix}_http"
+        use Prometheus::Middleware::Exporter, path: metrics.prometheus_endpoint
+        # Note that a user may want to use this check without prometheus
+        # However, prometheus setup includes the web server which is required for this check
+        # At this time prometheus is a requirement of using the health check on manager
+        use Vmpooler::API::Healthcheck
       end
 
-    # Load dashboard components
-    begin
-      require 'dashboard'
-    rescue LoadError
-      require File.expand_path(File.join(File.dirname(__FILE__), 'dashboard'))
-    end
+      if torun.include? :api
+        # Enable API request logging only if required
+        use Vmpooler::API::RequestLogger, logger: logger if config[:config]['request_logger']
 
         use Vmpooler::Dashboard
-
-    # Load API components
-    %w[helpers dashboard reroute v1].each do |lib|
-      begin
-        require "api/#{lib}"
-      rescue LoadError
-        require File.expand_path(File.join(File.dirname(__FILE__), 'api', lib))
-      end
-    end
-
         use Vmpooler::API::Dashboard
-    use Vmpooler::API::Reroute
-    use Vmpooler::API::V1
-
-    def configure(config, redis, metrics)
-      self.settings.set :config, config
-      self.settings.set :redis, redis
-      self.settings.set :metrics, metrics
+        use Vmpooler::API::V3
       end
 
-    def execute!
-      self.settings.run!
+      # Get thee started O WebServer
+      self.run!
     end
   end
 end

lib/vmpooler/api/dashboard.rb

@@ -1,20 +1,32 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API
     class Dashboard < Sinatra::Base
+      helpers do
+        include Vmpooler::API::Helpers
+      end
+
       # handle to the App's configuration information
       def config
         @config ||= Vmpooler::API.settings.config
       end
 
+      def backend
+        Vmpooler::API.settings.redis
+      end
+
       # configuration setting for server hosting graph URLs to view
       def graph_server
         return @graph_server if @graph_server
 
         if config[:graphs]
           return false unless config[:graphs]['server']
+
           @graph_server = config[:graphs]['server']
         elsif config[:graphite]
           return false unless config[:graphite]['server']
+
           @graph_server = config[:graphite]['server']
         else
           false
@@ -27,9 +39,11 @@ module Vmpooler
 
         if config[:graphs]
           return 'vmpooler' unless config[:graphs]['prefix']
+
           @graph_prefix = config[:graphs]['prefix']
         elsif config[:graphite]
           return false unless config[:graphite]['prefix']
+
           @graph_prefix = config[:graphite]['prefix']
         else
           false
@@ -39,12 +53,14 @@ module Vmpooler
       # what is the base URL for viewable graphs?
       def graph_url
         return false unless graph_server && graph_prefix
+
         @graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
       end
 
       # return a full URL to a viewable graph for a given metrics target (graphite syntax)
       def graph_link(target = '')
         return '' unless graph_url
+
         graph_url + target
       end
 
@@ -53,10 +69,13 @@ module Vmpooler
         content_type :json
         result = {}
 
-        Vmpooler::API.settings.config[:pools].each do |pool|
+        pools = Vmpooler::API.settings.config[:pools]
+        ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
+        pools.each do |pool|
           result[pool['name']] ||= {}
           result[pool['name']]['size'] = pool['size']
-          result[pool['name']]['ready'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])
+          result[pool['name']]['ready'] = ready_hash[pool['name']]
         end
 
         if params[:history]
@@ -64,7 +83,7 @@ module Vmpooler
             history ||= {}
 
             begin
-              buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
+              buffer = URI.parse(graph_link('.ready.*&from=-1hour&format=json')).read
               history = JSON.parse(buffer)
 
               history.each do |pool|
@@ -88,12 +107,12 @@ module Vmpooler
                   end
                 end
               end
-            rescue
+            rescue StandardError
             end
           else
-            Vmpooler::API.settings.config[:pools].each do |pool|
+            pools.each do |pool|
               result[pool['name']] ||= {}
-              result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])]
+              result[pool['name']]['history'] = [ready_hash[pool['name']]]
             end
           end
         end
@@ -104,21 +123,23 @@ module Vmpooler
         content_type :json
         result = {}
 
-        Vmpooler::API.settings.config[:pools].each do |pool|
-          running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
-          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+        pools = Vmpooler::API.settings.config[:pools]
+        running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+
+        pools.each do |pool|
+          running = running_hash[pool['name']]
+          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
           result[pool['major']] ||= {}
           result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
         end
 
-        if params[:history]
-          if graph_url
+        if params[:history] && graph_url
           begin
-              buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
+            buffer = URI.parse(graph_link('.running.*&from=-1hour&format=json')).read
             JSON.parse(buffer).each do |pool|
               if pool['target'] =~ /.*\.(.*)$/
                 pool['name'] = Regexp.last_match[1]
-                  pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+                pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
                 result[pool['major']]['history'] ||= []
 
                 for i in 0..pool['datapoints'].length
@@ -132,8 +153,7 @@ module Vmpooler
                 end
               end
             end
-            rescue
-            end
+          rescue StandardError
           end
         end
         JSON.pretty_generate(result)

lib/vmpooler/api/healthcheck.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    class Healthcheck < Sinatra::Base
+      get '/healthcheck/?' do
+        content_type :json
+
+        status 200
+        JSON.pretty_generate({ 'ok' => true })
+      end
+    end
+  end
+end

lib/vmpooler/api/helpers.rb

@@ -1,22 +1,34 @@
+# frozen_string_literal: true
+
+require 'vmpooler/api/input_validator'
+
 module Vmpooler
 
   class API
 
     module Helpers
+      include InputValidator
+
+      def tracer
+        @tracer ||= OpenTelemetry.tracer_provider.tracer('api', Vmpooler::VERSION)
+      end
 
       def has_token?
         request.env['HTTP_X_AUTH_TOKEN'].nil? ? false : true
       end
 
       def valid_token?(backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           return false unless has_token?
 
-        backend.exists('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
+          backend.exists?("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}") ? true : false
+        end
       end
 
       def validate_token(backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           if valid_token?(backend)
-          backend.hset('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'last', Time.now)
+            backend.hset("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'last', Time.now.to_s)
 
             return true
           end
@@ -28,8 +40,10 @@ module Vmpooler
           headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
           halt 401, JSON.pretty_generate(result)
         end
+      end
 
       def validate_auth(backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           return if authorized?
 
           content_type :json
@@ -39,8 +53,10 @@ module Vmpooler
           headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
           halt 401, JSON.pretty_generate(result)
         end
+      end
 
       def authorized?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           @auth ||= Rack::Auth::Basic::Request.new(request.env)
 
           if @auth.provided? and @auth.basic? and @auth.credentials
@@ -53,82 +69,131 @@ module Vmpooler
 
           return false
         end
+      end
+
+      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.name' => host,
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp',
+            'enduser.id' => username_str
+          },
+          kind: :client
+        ) do
+          if service_account_hash
+            username = service_account_hash[:user_dn]
+            password = service_account_hash[:password]
+          else
+            username = "#{user_object}=#{username_str},#{base}"
+            password = password_str
+          end
 
-      def authenticate_ldap(port, host, user_object, base, username_str, password_str)
           ldap = Net::LDAP.new(
             :host => host,
             :port => port,
-          :encryption => {
-            :method => :start_tls,
-            :tls_options => { :ssl_version => 'TLSv1' }
-          },
+            :encryption => encryption_hash,
             :base => base,
             :auth => {
               :method => :simple,
-            :username => "#{user_object}=#{username_str},#{base}",
-            :password => password_str
+              :username => username,
+              :password => password
             }
           )
 
-        return true if ldap.bind
+          if service_account_hash
+            return true if ldap.bind_as(
+              :base => base,
+              :filter => "(#{user_object}=#{username_str})",
+              :password => password_str
+            )
+          elsif ldap.bind
+            return true
+          else
             return false
           end
 
+          return false
+        end
+      end
+
       def authenticate(auth, username_str, password_str)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'enduser.id' => username_str
+          }
+        ) do
           case auth['provider']
           when 'dummy'
             return (username_str != password_str)
           when 'ldap'
             ldap_base = auth[:ldap]['base']
             ldap_port = auth[:ldap]['port'] || 389
+            ldap_user_obj = auth[:ldap]['user_object']
+            ldap_host = auth[:ldap]['host']
+            ldap_encryption_hash = auth[:ldap]['encryption'] || {
+              :method => :start_tls,
+              :tls_options => { :ssl_version => 'TLSv1' }
+            }
+            service_account_hash = auth[:ldap]['service_account_hash']
+
+            unless ldap_base.is_a? Array
+              ldap_base = ldap_base.split
+            end
+
+            unless ldap_user_obj.is_a? Array
+              ldap_user_obj = ldap_user_obj.split
+            end
 
-            if ldap_base.is_a? Array
             ldap_base.each do |search_base|
+              ldap_user_obj.each do |search_user_obj|
                 result = authenticate_ldap(
                   ldap_port,
-                  auth[:ldap]['host'],
-                  auth[:ldap]['user_object'],
+                  ldap_host,
+                  ldap_encryption_hash,
+                  search_user_obj,
                   search_base,
                   username_str,
                   password_str,
+                  service_account_hash
                 )
-                return true if result == true
+                return true if result
               end
-            else
-              result = authenticate_ldap(
-                ldap_port,
-                auth[:ldap]['host'],
-                auth[:ldap]['user_object'],
-                ldap_base,
-                username_str,
-                password_str,
-              )
-              return result
             end
 
             return false
           end
         end
+      end
 
       def export_tags(backend, hostname, tags)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          backend.pipelined do |pipeline|
             tags.each_pair do |tag, value|
               next if value.nil? or value.empty?
 
-          backend.hset('vmpooler__vm__' + hostname, 'tag:' + tag, value)
-          backend.hset('vmpooler__tag__' + Date.today.to_s, hostname + ':' + tag, value)
+              pipeline.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value)
+              pipeline.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value)
+            end
+          end
         end
       end
 
       def filter_tags(tags)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           return unless Vmpooler::API.settings.config[:tagfilter]
 
           tags.each_pair do |tag, value|
             next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
+
             tags[tag] = value.match(filter).captures.join if value.match(filter)
           end
 
           tags
         end
+      end
 
       def mean(list)
         s = list.map(&:to_f).reduce(:+).to_f
@@ -139,19 +204,71 @@ module Vmpooler
         /^\d{4}-\d{2}-\d{2}$/ === date_str
       end
 
-      def hostname_shorten(hostname, domain=nil)
-        if domain && hostname =~ /^\w+\.#{domain}$/
-          hostname = hostname[/[^\.]+/]
-        end
-
-        hostname
+      def hostname_shorten(hostname)
+        hostname[/[^.]+/]
       end
 
       def get_task_times(backend, task, date_str)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
         end
+      end
+
+      # Takes the pools and a key to run scard on
+      # returns an integer for the total count
+      def get_total_across_pools_redis_scard(pools, key, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
+          end
+          res.inject(0) { |m, x| m + x }.to_i
+        end
+      end
+
+      # Takes the pools and a key to run scard on
+      # returns a hash with each pool name as key and the value being the count as integer
+      def get_list_across_pools_redis_scard(pools, key, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
+          end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_i
+          end
+          temp_hash
+        end
+      end
+
+      # Takes the pools and a key to run hget on
+      # returns a hash with each pool name as key and the value as string
+      def get_list_across_pools_redis_hget(pools, key, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.hget(key, pool['name'])
+            end
+          end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_s
+          end
+          temp_hash
+        end
+      end
 
       def get_capacity_metrics(pools, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           capacity = {
               current: 0,
               total: 0,
@@ -159,21 +276,23 @@ module Vmpooler
           }
 
           pools.each do |pool|
-          pool['capacity'] = backend.scard('vmpooler__ready__' + pool['name']).to_i
-
-          capacity[:current] += pool['capacity']
             capacity[:total] += pool['size'].to_i
           end
 
+          capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
           if capacity[:total] > 0
-          capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1)
+            capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1)
           end
 
           capacity
         end
+      end
 
       def get_queue_metrics(pools, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           queue = {
+              requested: 0,
               pending: 0,
               cloning: 0,
               booting: 0,
@@ -183,37 +302,55 @@ module Vmpooler
               total: 0
           }
 
+          # Use a single pipeline to fetch all queue counts at once for better performance
+          results = backend.pipelined do |pipeline|
+            # Order matters - we'll use indices to extract values
             pools.each do |pool|
-          queue[:pending]   += backend.scard('vmpooler__pending__' + pool['name']).to_i
-          queue[:ready]     += backend.scard('vmpooler__ready__' + pool['name']).to_i
-          queue[:running]   += backend.scard('vmpooler__running__' + pool['name']).to_i
-          queue[:completed] += backend.scard('vmpooler__completed__' + pool['name']).to_i
+              pipeline.scard("vmpooler__provisioning__request#{pool['name']}")    # 0..n-1
+              pipeline.scard("vmpooler__provisioning__processing#{pool['name']}") # n..2n-1
+              pipeline.scard("vmpooler__odcreate__task#{pool['name']}")           # 2n..3n-1
+              pipeline.scard("vmpooler__pending__#{pool['name']}")                # 3n..4n-1
+              pipeline.scard("vmpooler__ready__#{pool['name']}")                  # 4n..5n-1
+              pipeline.scard("vmpooler__running__#{pool['name']}")                # 5n..6n-1
+              pipeline.scard("vmpooler__completed__#{pool['name']}")              # 6n..7n-1
+            end
+            pipeline.get('vmpooler__tasks__clone')           # 7n
+            pipeline.get('vmpooler__tasks__ondemandclone')   # 7n+1
           end
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+          n = pools.length
+          # Safely extract results with default to empty array if slice returns nil
+          queue[:requested] = (results[0...n] || []).sum(&:to_i) +
+                              (results[n...(2 * n)] || []).sum(&:to_i) +
+                              (results[(2 * n)...(3 * n)] || []).sum(&:to_i)
+          queue[:pending]   = (results[(3 * n)...(4 * n)] || []).sum(&:to_i)
+          queue[:ready]     = (results[(4 * n)...(5 * n)] || []).sum(&:to_i)
+          queue[:running]   = (results[(5 * n)...(6 * n)] || []).sum(&:to_i)
+          queue[:completed] = (results[(6 * n)...(7 * n)] || []).sum(&:to_i)
+          queue[:cloning]   = (results[7 * n] || 0).to_i + (results[7 * n + 1] || 0).to_i
           queue[:booting]   = queue[:pending].to_i - queue[:cloning].to_i
           queue[:booting]   = 0 if queue[:booting] < 0
-        queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
+          queue[:total]     = queue[:requested] + queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
 
           queue
         end
+      end
 
       def get_tag_metrics(backend, date_str, opts = {})
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           opts = {:only => false}.merge(opts)
 
           tags = {}
 
-        backend.hgetall('vmpooler__tag__' + date_str).each do |key, value|
+          backend.hgetall("vmpooler__tag__#{date_str}").each do |key, value|
             hostname = 'unknown'
             tag = 'unknown'
 
-          if key =~ /\:/
+            if key =~ /:/
               hostname, tag = key.split(':', 2)
             end
 
-          if opts[:only]
-            next unless tag == opts[:only]
-          end
+            next if opts[:only] && tag != opts[:only]
 
             tags[tag] ||= {}
             tags[tag][value] ||= 0
@@ -225,8 +362,10 @@ module Vmpooler
 
           tags
         end
+      end
 
       def get_tag_summary(backend, from_date, to_date, opts = {})
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           opts = {:only => false}.merge(opts)
 
           result = {
@@ -255,8 +394,10 @@ module Vmpooler
 
           result
         end
+      end
 
       def get_task_metrics(backend, task_str, date_str, opts = {})
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           opts = {:bypool => false, :only => false}.merge(opts)
 
           task = {
@@ -271,7 +412,7 @@ module Vmpooler
               }
           }
 
-        task[:count][:total] = backend.hlen('vmpooler__' + task_str + '__' + date_str).to_i
+          task[:count][:total] = backend.hlen("vmpooler__#{task_str}__#{date_str}").to_i
 
           if task[:count][:total] > 0
             if opts[:bypool] == true
@@ -280,11 +421,11 @@ module Vmpooler
               task[:count][:pool]    = {}
               task[:duration][:pool] = {}
 
-            backend.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
+              backend.hgetall("vmpooler__#{task_str}__#{date_str}").each do |key, value|
                 pool     = 'unknown'
                 hostname = 'unknown'
 
-              if key =~ /\:/
+                if key =~ /:/
                   pool, hostname = key.split(':')
                 else
                   hostname = key
@@ -321,8 +462,10 @@ module Vmpooler
 
           task
         end
+      end
 
       def get_task_summary(backend, task_str, from_date, to_date, opts = {})
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           opts = {:bypool => false, :only => false}.merge(opts)
 
           task_sym = task_str.to_sym
@@ -402,11 +545,12 @@ module Vmpooler
 
           result
         end
+      end
 
       def pool_index(pools)
         pools_hash = {}
         index = 0
-        for pool in pools
+        pools.each do |pool|
           pools_hash[pool['name']] = index
           index += 1
         end
@@ -414,18 +558,43 @@ module Vmpooler
       end
 
       def template_ready?(pool, backend)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
           prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
           return false if prepared_template.nil?
           return true if pool['template'] == prepared_template
+
           return false
         end
+      end
 
       def is_integer?(x)
         Integer(x)
         true
-      rescue
+      rescue StandardError
         false
       end
+
+      def open_socket(host, domain = nil, timeout = 1, port = 22, &_block)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp'
+          },
+          kind: :client
+        ) do
+          target_host = host
+          target_host = "#{host}.#{domain}" if domain
+          span = OpenTelemetry::Trace.current_span
+          span.set_attribute('net.peer.name', target_host)
+          sock = TCPSocket.new(target_host, port, connect_timeout: timeout)
+          begin
+            yield sock if block_given?
+          ensure
+            sock.close
+          end
+        end
+      end
     end
   end
 end

lib/vmpooler/api/input_validator.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Input validation helpers to enhance security
+    module InputValidator
+      # Maximum lengths to prevent abuse
+      MAX_HOSTNAME_LENGTH = 253
+      MAX_TAG_KEY_LENGTH = 50
+      MAX_TAG_VALUE_LENGTH = 255
+      MAX_REASON_LENGTH = 500
+      MAX_POOL_NAME_LENGTH = 100
+      MAX_TOKEN_LENGTH = 64
+
+      # Valid patterns
+      HOSTNAME_PATTERN = /\A[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)* \z/ix.freeze
+      POOL_NAME_PATTERN = /\A[a-zA-Z0-9_-]+\z/.freeze
+      TAG_KEY_PATTERN = /\A[a-zA-Z0-9_\-.]+\z/.freeze
+      TOKEN_PATTERN = /\A[a-zA-Z0-9\-_]+\z/.freeze
+      INTEGER_PATTERN = /\A\d+\z/.freeze
+
+      class ValidationError < StandardError; end
+
+      # Validate hostname format and length
+      def validate_hostname(hostname)
+        return error_response('Hostname is required') if hostname.nil? || hostname.empty?
+        return error_response('Hostname too long') if hostname.length > MAX_HOSTNAME_LENGTH
+        return error_response('Invalid hostname format') unless hostname.match?(HOSTNAME_PATTERN)
+
+        true
+      end
+
+      # Validate pool/template name
+      def validate_pool_name(pool_name)
+        return error_response('Pool name is required') if pool_name.nil? || pool_name.empty?
+        return error_response('Pool name too long') if pool_name.length > MAX_POOL_NAME_LENGTH
+        return error_response('Invalid pool name format') unless pool_name.match?(POOL_NAME_PATTERN)
+
+        true
+      end
+
+      # Validate tag key and value
+      def validate_tag(key, value)
+        return error_response('Tag key is required') if key.nil? || key.empty?
+        return error_response('Tag key too long') if key.length > MAX_TAG_KEY_LENGTH
+        return error_response('Invalid tag key format') unless key.match?(TAG_KEY_PATTERN)
+
+        if value
+          return error_response('Tag value too long') if value.length > MAX_TAG_VALUE_LENGTH
+
+          # Sanitize value to prevent injection attacks
+          sanitized_value = value.gsub(/[^\w\s\-.@:\/]/, '')
+          return error_response('Tag value contains invalid characters') if sanitized_value != value
+        end
+
+        true
+      end
+
+      # Validate token format
+      def validate_token_format(token)
+        return error_response('Token is required') if token.nil? || token.empty?
+        return error_response('Token too long') if token.length > MAX_TOKEN_LENGTH
+        return error_response('Invalid token format') unless token.match?(TOKEN_PATTERN)
+
+        true
+      end
+
+      # Validate integer parameter
+      def validate_integer(value, name = 'value', min: nil, max: nil)
+        return error_response("#{name} is required") if value.nil?
+
+        value_str = value.to_s
+        return error_response("#{name} must be a valid integer") unless value_str.match?(INTEGER_PATTERN)
+
+        int_value = value.to_i
+        return error_response("#{name} must be at least #{min}") if min && int_value < min
+        return error_response("#{name} must be at most #{max}") if max && int_value > max
+
+        int_value
+      end
+
+      # Validate VM request count
+      def validate_vm_count(count)
+        validated = validate_integer(count, 'VM count', min: 1, max: 100)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate disk size
+      def validate_disk_size(size)
+        validated = validate_integer(size, 'Disk size', min: 1, max: 2048)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate lifetime (TTL) in hours
+      def validate_lifetime(lifetime)
+        validated = validate_integer(lifetime, 'Lifetime', min: 1, max: 168) # max 1 week
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate reason text
+      def validate_reason(reason)
+        return true if reason.nil? || reason.empty?
+        return error_response('Reason too long') if reason.length > MAX_REASON_LENGTH
+
+        # Sanitize to prevent XSS/injection
+        sanitized = reason.gsub(/[<>"']/, '')
+        return error_response('Reason contains invalid characters') if sanitized != reason
+
+        true
+      end
+
+      # Sanitize JSON body to prevent injection
+      def sanitize_json_body(body)
+        return {} if body.nil? || body.empty?
+
+        begin
+          parsed = JSON.parse(body)
+          return error_response('Request body must be a JSON object') unless parsed.is_a?(Hash)
+
+          # Limit depth and size to prevent DoS
+          return error_response('Request body too complex') if json_depth(parsed) > 5
+          return error_response('Request body too large') if body.length > 10_240 # 10KB max
+
+          parsed
+        rescue JSON::ParserError => e
+          error_response("Invalid JSON: #{e.message}")
+        end
+      end
+
+      # Check if validation result is an error
+      def validation_error?(result)
+        result.is_a?(Hash) && result['ok'] == false
+      end
+
+      private
+
+      def error_response(message)
+        { 'ok' => false, 'error' => message }
+      end
+
+      def json_depth(obj, depth = 0)
+        return depth unless obj.is_a?(Hash) || obj.is_a?(Array)
+        return depth + 1 if obj.empty?
+
+        if obj.is_a?(Hash)
+          depth + 1 + obj.values.map { |v| json_depth(v, 0) }.max
+        else
+          depth + 1 + obj.map { |v| json_depth(v, 0) }.max
+        end
+      end
+    end
+  end
+end

lib/vmpooler/api/rate_limiter.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Rate limiter middleware to protect against abuse
+    # Uses Redis to track request counts per IP and token
+    class RateLimiter
+      DEFAULT_LIMITS = {
+        global_per_ip: { limit: 100, period: 60 }, # 100 requests per minute per IP
+        authenticated: { limit: 500, period: 60 }, # 500 requests per minute with token
+        vm_creation: { limit: 20, period: 60 },    # 20 VM creations per minute
+        vm_deletion: { limit: 50, period: 60 }     # 50 VM deletions per minute
+      }.freeze
+
+      def initialize(app, redis, config = {})
+        @app = app
+        @redis = redis
+        @config = DEFAULT_LIMITS.merge(config[:rate_limits] || {})
+        @enabled = config.fetch(:rate_limiting_enabled, true)
+      end
+
+      def call(env)
+        return @app.call(env) unless @enabled
+
+        request = Rack::Request.new(env)
+        client_id = identify_client(request)
+        endpoint_type = classify_endpoint(request)
+
+        # Check rate limits
+        return rate_limit_response(client_id, endpoint_type) if rate_limit_exceeded?(client_id, endpoint_type, request)
+
+        # Track the request
+        increment_request_count(client_id, endpoint_type)
+
+        @app.call(env)
+      end
+
+      private
+
+      def identify_client(request)
+        # Prioritize token-based identification for authenticated requests
+        token = request.env['HTTP_X_AUTH_TOKEN']
+        return "token:#{token}" if token && !token.empty?
+
+        # Fall back to IP address
+        ip = request.ip || request.env['REMOTE_ADDR'] || 'unknown'
+        "ip:#{ip}"
+      end
+
+      def classify_endpoint(request)
+        path = request.path
+        method = request.request_method
+
+        return :vm_creation if method == 'POST' && path.include?('/vm')
+        return :vm_deletion if method == 'DELETE' && path.include?('/vm')
+        return :authenticated if request.env['HTTP_X_AUTH_TOKEN']
+
+        :global_per_ip
+      end
+
+      def rate_limit_exceeded?(client_id, endpoint_type, _request)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        current_count = @redis.get(key).to_i
+        current_count >= limit_config[:limit]
+      rescue StandardError => e
+        # If Redis fails, allow the request through (fail open)
+        warn "Rate limiter Redis error: #{e.message}"
+        false
+      end
+
+      def increment_request_count(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        @redis.pipelined do |pipeline|
+          pipeline.incr(key)
+          pipeline.expire(key, limit_config[:period])
+        end
+      rescue StandardError => e
+        # Log error but don't fail the request
+        warn "Rate limiter increment error: #{e.message}"
+      end
+
+      def rate_limit_response(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        begin
+          ttl = @redis.ttl(key)
+        rescue StandardError
+          ttl = limit_config[:period]
+        end
+
+        headers = {
+          'Content-Type' => 'application/json',
+          'X-RateLimit-Limit' => limit_config[:limit].to_s,
+          'X-RateLimit-Remaining' => '0',
+          'X-RateLimit-Reset' => (Time.now.to_i + ttl).to_s,
+          'Retry-After' => ttl.to_s
+        }
+
+        body = JSON.pretty_generate({
+                                      'ok' => false,
+                                      'error' => 'Rate limit exceeded',
+                                      'limit' => limit_config[:limit],
+                                      'period' => limit_config[:period],
+                                      'retry_after' => ttl
+                                    })
+
+        [429, headers, [body]]
+      end
+    end
+  end
+end

lib/vmpooler/api/request_logger.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    class RequestLogger
+      attr_reader :app
+
+      def initialize(app, options = {})
+        @app = app
+        @logger = options[:logger]
+      end
+
+      def call(env)
+        status, headers, body = @app.call(env)
+        @logger.log('s', "[ ] API: Method: #{env['REQUEST_METHOD']}, Status: #{status}, Path: #{env['PATH_INFO']}, Body: #{body}")
+        [status, headers, body]
+      end
+    end
+  end
+end

lib/vmpooler/api/reroute.rb

@@ -1,71 +0,0 @@
-module Vmpooler
-  class API
-    class Reroute < Sinatra::Base
-      api_version = '1'
-
-      get '/status/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/status")
-      end
-
-      get '/summary/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary")
-      end
-
-      get '/summary/:route/?:key?/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}")
-      end
-
-      get '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      post '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      get '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      delete '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      get '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/:template/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}")
-      end
-
-      get '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      delete '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      put '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      post '/vm/:hostname/snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot")
-      end
-
-      post '/vm/:hostname/snapshot/:snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}")
-      end
-
-      put '/vm/:hostname/disk/:size/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}")
-      end
-    end
-  end
-end

lib/vmpooler/api/v1.rb

@@ -1,961 +0,0 @@
-module Vmpooler
-  class API
-  class V1 < Sinatra::Base
-    api_version = '1'
-    api_prefix  = "/api/v#{api_version}"
-
-    helpers do
-      include Vmpooler::API::Helpers
-    end
-
-    def backend
-      Vmpooler::API.settings.redis
-    end
-
-    def metrics
-      Vmpooler::API.settings.metrics
-    end
-
-    def config
-      Vmpooler::API.settings.config[:config]
-    end
-
-    def pools
-      Vmpooler::API.settings.config[:pools]
-    end
-
-    def pool_exists?(template)
-      Vmpooler::API.settings.config[:pool_names].include?(template)
-    end
-
-    def need_auth!
-      validate_auth(backend)
-    end
-
-    def need_token!
-      validate_token(backend)
-    end
-
-    def fetch_single_vm(template)
-      template_backends = [template]
-      aliases = Vmpooler::API.settings.config[:alias]
-      if aliases
-        template_backends << aliases[template] if aliases[template]
-
-        pool_index = pool_index(pools)
-        weighted_pools = {}
-        template_backends.each do |t|
-          next unless pool_index.key? t
-          index = pool_index[t]
-          clone_target = pools[index]['clone_target'] || config['clone_target']
-          next unless config.key?('backend_weight')
-          weight = config['backend_weight'][clone_target]
-          if weight
-            weighted_pools[t] = weight
-          end
-        end
-
-        if weighted_pools.count == template_backends.count
-          pickup = Pickup.new(weighted_pools)
-          selection = pickup.pick
-          template_backends.delete(selection)
-          template_backends.unshift(selection)
-        else
-          template_backends = template_backends.sample(template_backends.count)
-        end
-      end
-
-      template_backends.each do |t|
-        vm = backend.spop('vmpooler__ready__' + t)
-        return [vm, t] if vm
-      end
-      [nil, nil]
-    end
-
-    def return_vm_to_ready_state(template, vm)
-      backend.sadd('vmpooler__ready__' + template, vm)
-    end
-
-    def account_for_starting_vm(template, vm)
-      backend.sadd('vmpooler__running__' + template, vm)
-      backend.sadd('vmpooler__migrating__' + template, vm)
-      backend.hset('vmpooler__active__' + template, vm, Time.now)
-      backend.hset('vmpooler__vm__' + vm, 'checkout', Time.now)
-
-      if Vmpooler::API.settings.config[:auth] and has_token?
-        validate_token(backend)
-
-        backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
-        backend.hset('vmpooler__vm__' + vm, 'token:user',
-          backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')
-        )
-
-        if config['vm_lifetime_auth'].to_i > 0
-          backend.hset('vmpooler__vm__' + vm, 'lifetime', config['vm_lifetime_auth'].to_i)
-        end
-      end
-    end
-
-    def update_result_hosts(result, template, vm)
-      result[template] ||= {}
-      if result[template]['hostname']
-        result[template]['hostname'] = Array(result[template]['hostname'])
-        result[template]['hostname'].push(vm)
-      else
-        result[template]['hostname'] = vm
-      end
-    end
-
-    def atomically_allocate_vms(payload)
-      result = { 'ok' => false }
-      failed = false
-      vms = []
-
-      payload.each do |requested, count|
-        count.to_i.times do |_i|
-          vm, name = fetch_single_vm(requested)
-          if !vm
-            failed = true
-            metrics.increment('checkout.empty.' + requested)
-            break
-          else
-            vms << [ name, vm ]
-            metrics.increment('checkout.success.' + name)
-          end
-        end
-      end
-
-      if failed
-        vms.each do |(name, vm)|
-          return_vm_to_ready_state(name, vm)
-        end
-        status 503
-      else
-        vms.each do |(name, vm)|
-          account_for_starting_vm(name, vm)
-          update_result_hosts(result, name, vm)
-        end
-
-        result['ok'] = true
-        result['domain'] = config['domain'] if config['domain']
-      end
-
-      result
-    end
-
-    def update_pool_size(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_sizes
-
-      payload.each do |poolname, size|
-        unless pools[pool_index[poolname]]['size'] == size.to_i
-          pools[pool_index[poolname]]['size'] = size.to_i
-          backend.hset('vmpooler__config__poolsize', poolname, size)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def update_pool_template(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_templates
-
-      payload.each do |poolname, template|
-        unless pools[pool_index[poolname]]['template'] == template
-          pools[pool_index[poolname]]['template'] = template
-          backend.hset('vmpooler__config__template', poolname, template)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def sync_pool_templates
-      pool_index = pool_index(pools)
-      template_configs = backend.hgetall('vmpooler__config__template')
-      unless template_configs.nil?
-        template_configs.each do |poolname, template|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['template'] == template
-              pools[pool_index[poolname]]['template'] = template
-            end
-          end
-        end
-      end
-    end
-
-    def sync_pool_sizes
-      pool_index = pool_index(pools)
-      poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
-      unless poolsize_configs.nil?
-        poolsize_configs.each do |poolname, size|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['size'] == size.to_i
-              pools[pool_index[poolname]]['size'] == size.to_i
-            end
-          end
-        end
-      end
-    end
-
-    # Provide run-time statistics
-    #
-    # Example:
-    #
-    # {
-    #   "boot": {
-    #     "duration": {
-    #       "average": 163.6,
-    #       "min": 65.49,
-    #       "max": 830.07,
-    #       "total": 247744.71000000002
-    #     },
-    #     "count": {
-    #       "total": 1514
-    #     }
-    #   },
-    #   "capacity": {
-    #     "current": 968,
-    #     "total": 975,
-    #     "percent": 99.3
-    #   },
-    #   "clone": {
-    #     "duration": {
-    #       "average": 17.0,
-    #       "min": 4.66,
-    #       "max": 637.96,
-    #       "total": 25634.15
-    #     },
-    #     "count": {
-    #       "total": 1507
-    #     }
-    #   },
-    #   "queue": {
-    #     "pending": 12,
-    #     "cloning": 0,
-    #     "booting": 12,
-    #     "ready": 968,
-    #     "running": 367,
-    #     "completed": 0,
-    #     "total": 1347
-    #   },
-    #   "pools": {
-    #     "ready": 100,
-    #     "running": 120,
-    #     "pending": 5,
-    #     "max": 250,
-    #   }
-    #   "status": {
-    #     "ok": true,
-    #     "message": "Battle station fully armed and operational.",
-    #     "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools
-    #       "redhat-7-x86_64",
-    #       "ubuntu-1404-i386"
-    #     ],
-    #     "uptime": 179585.9
-    # }
-    #
-    # If the query parameter 'view' is provided, it will be used to select which top level
-    # element to compute and return. Select them by specifying them in a comma separated list.
-    # For example /status?view=capacity,boot
-    # would return only the "capacity" and "boot" statistics. "status" is always returned
-
-    get "#{api_prefix}/status/?" do
-      content_type :json
-
-      if params[:view]
-        views = params[:view].split(",")
-      end
-
-      result = {
-        status: {
-          ok: true,
-          message: 'Battle station fully armed and operational.'
-        }
-      }
-
-      sync_pool_sizes
-
-      result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
-      result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
-      result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
-      result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot")
-
-      # Check for empty pools
-      result[:pools] = {} unless views and not views.include?("pools")
-      pools.each do |pool|
-        # REMIND: move this out of the API and into the back-end
-        ready    = backend.scard('vmpooler__ready__' + pool['name']).to_i
-        running  = backend.scard('vmpooler__running__' + pool['name']).to_i
-        pending  = backend.scard('vmpooler__pending__' + pool['name']).to_i
-        max      = pool['size']
-        lastBoot = backend.hget('vmpooler__lastboot',pool['name']).to_s
-        aka      = pool['alias']
-
-        result[:pools][pool['name']] = {
-          ready:    ready,
-          running:  running,
-          pending:  pending,
-          max:      max,
-          lastBoot: lastBoot
-        }
-
-        if aka
-          result[:pools][pool['name']][:alias] = aka
-        end
-
-        # for backwards compatibility, include separate "empty" stats in "status" block
-        if ready == 0
-          result[:status][:empty] ||= []
-          result[:status][:empty].push(pool['name'])
-
-          result[:status][:ok] = false
-          result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
-        end
-      end unless views and not views.include?("pools")
-
-      result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
-
-      JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
-    end
-
-    get "#{api_prefix}/summary/?" do
-      content_type :json
-
-      result = {
-        daily: []
-      }
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true)
-      clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true)
-      tag = get_tag_summary(backend, from_date, to_date)
-
-      result[:boot] = boot[:boot]
-      result[:clone] = clone[:clone]
-      result[:tag] = tag[:tag]
-
-      daily = {}
-
-      boot[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:boot] = day[:boot]
-      end
-
-      clone[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:clone] = day[:clone]
-      end
-
-      tag[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:tag] = day[:tag]
-      end
-
-      daily.each_key do |day|
-        result[:daily].push({
-          date: day,
-          boot: daily[day][:boot],
-          clone: daily[day][:clone],
-          tag: daily[day][:tag]
-        })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/summary/:route/?:key?/?" do
-      content_type :json
-
-      result = {}
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      case params[:route]
-        when 'boot'
-          result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'clone'
-          result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'tag'
-          result = get_tag_summary(backend, from_date, to_date, :only => params[:key])
-        else
-          halt 404, JSON.pretty_generate({ 'ok' => false })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        backend.keys('vmpooler__token__*').each do |key|
-          data = backend.hgetall(key)
-
-          if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
-            token = key.split('__').last
-
-            result[token] ||= {}
-
-            result[token]['created'] = data['created']
-            result[token]['last'] = data['last'] || 'never'
-
-            result['ok'] = true
-          end
-        end
-
-        if result['ok']
-          status 200
-        else
-          status 404
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        token = backend.hgetall('vmpooler__token__' + params[:token])
-
-        if not token.nil? and not token.empty?
-          status 200
-
-          pools.each do |pool|
-            backend.smembers('vmpooler__running__' + pool['name']).each do |vm|
-              if backend.hget('vmpooler__vm__' + vm, 'token:token') == params[:token]
-                token['vms'] ||= {}
-                token['vms']['running'] ||= []
-                token['vms']['running'].push(vm)
-              end
-            end
-          end
-
-          result = { 'ok' => true, params[:token] => token }
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        if backend.del('vmpooler__token__' + params[:token]).to_i > 0
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/token" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.hset('vmpooler__token__' + result['token'], 'user', @auth.username)
-        backend.hset('vmpooler__token__' + result['token'], 'created', Time.now)
-
-        status 200
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/?" do
-      content_type :json
-
-      result = []
-
-      pools.each do |pool|
-        result.push(pool['name'])
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      payload = JSON.parse(request.body.read)
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    def extract_templates_from_query_params(params)
-      payload = {}
-
-      params.split('+').each do |template|
-        payload[template] ||= 0
-        payload[template] += 1
-      end
-
-      payload
-    end
-
-    def invalid_templates(payload)
-      invalid = []
-      payload.keys.each do |template|
-        invalid << template unless pool_exists?(template)
-      end
-      invalid
-    end
-
-    def invalid_template_or_size(payload)
-      invalid = []
-      payload.each do |pool, size|
-        invalid << pool unless pool_exists?(pool)
-        unless is_integer?(size)
-          invalid << pool
-          next
-        end
-        invalid << pool unless Integer(size) >= 0
-      end
-      invalid
-    end
-
-    def invalid_template_or_path(payload)
-      invalid = []
-      payload.each do |pool, template|
-        invalid << pool unless pool_exists?(pool)
-        invalid << pool unless template.include? '/'
-        invalid << pool if template[0] == '/'
-        invalid << pool if template[-1] == '/'
-      end
-      invalid
-    end
-
-    post "#{api_prefix}/vm/:template/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      payload = extract_templates_from_query_params(params[:template])
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        status 200
-        result['ok'] = true
-
-        result[params[:hostname]] = {}
-
-        result[params[:hostname]]['template'] = rdata['template']
-        result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
-
-        if rdata['destroy']
-          result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['state'] = 'destroyed'
-        elsif rdata['checkout']
-          result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
-          result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
-          result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
-          result[params[:hostname]]['state'] = 'running'
-        elsif rdata['check']
-          result[params[:hostname]]['state'] = 'ready'
-        else
-          result[params[:hostname]]['state'] = 'pending'
-        end
-
-        rdata.keys.each do |key|
-          if key.match('^tag\:(.+?)$')
-            result[params[:hostname]]['tags'] ||= {}
-            result[params[:hostname]]['tags'][$1] = rdata[key]
-          end
-
-          if key.match('^snapshot\:(.+?)$')
-            result[params[:hostname]]['snapshots'] ||= []
-            result[params[:hostname]]['snapshots'].push($1)
-          end
-        end
-
-        if rdata['disk']
-          result[params[:hostname]]['disk'] = rdata['disk'].split(':')
-        end
-
-        # Look up IP address of the hostname
-        begin
-          ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
-        rescue
-          ipAddress = ""
-        end
-
-        result[params[:hostname]]['ip'] = ipAddress
-
-        if config['domain']
-          result[params[:hostname]]['domain'] = config['domain']
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        need_token! if rdata['token:token']
-
-        if backend.srem('vmpooler__running__' + rdata['template'], params[:hostname])
-          backend.sadd('vmpooler__completed__' + rdata['template'], params[:hostname])
-
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    put "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      failure = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists('vmpooler__vm__' + params[:hostname])
-        begin
-          jdata = JSON.parse(request.body.read)
-        rescue
-          halt 400, JSON.pretty_generate(result)
-        end
-
-        # Validate data payload
-        jdata.each do |param, arg|
-          case param
-            when 'lifetime'
-              need_token! if Vmpooler::API.settings.config[:auth]
-
-              unless arg.to_i > 0
-                failure = true
-              end
-            when 'tags'
-              unless arg.is_a?(Hash)
-                failure = true
-              end
-
-              if config['allowed_tags']
-                failure = true if not (arg.keys - config['allowed_tags']).empty?
-              end
-            else
-              failure = true
-          end
-        end
-
-        if failure
-          status 400
-        else
-          jdata.each do |param, arg|
-            case param
-              when 'lifetime'
-                need_token! if Vmpooler::API.settings.config[:auth]
-
-                arg = arg.to_i
-
-                backend.hset('vmpooler__vm__' + params[:hostname], param, arg)
-              when 'tags'
-                filter_tags(arg)
-                export_tags(backend, params[:hostname], arg)
-            end
-          end
-
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/disk/:size/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if ((params[:size].to_i > 0 )and (backend.exists('vmpooler__vm__' + params[:hostname])))
-        result[params[:hostname]] = {}
-        result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
-
-        backend.sadd('vmpooler__tasks__disk', params[:hostname] + ':' + params[:size])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists('vmpooler__vm__' + params[:hostname])
-        result[params[:hostname]] = {}
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.sadd('vmpooler__tasks__snapshot', params[:hostname] + ':' + result[params[:hostname]]['snapshot'])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      unless backend.hget('vmpooler__vm__' + params[:hostname], 'snapshot:' + params[:snapshot]).to_i.zero?
-        backend.sadd('vmpooler__tasks__snapshot-revert', params[:hostname] + ':' + params[:snapshot])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/poolsize/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_size(payload)
-          if invalid.empty?
-            result = update_pool_size(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/pooltemplate/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_path(payload)
-          if invalid.empty?
-            result = update_pool_template(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/config/?" do
-      content_type :json
-      result = { 'ok' => false }
-      status 404
-
-      if pools
-        sync_pool_sizes
-        sync_pool_templates
-
-        pool_configuration = []
-        pools.each do |pool|
-          pool['template_ready'] = template_ready?(pool, backend)
-          pool_configuration << pool
-        end
-
-        result = {
-          pool_configuration: pool_configuration,
-          status: {
-            ok: true
-          }
-        }
-
-        status 200
-      end
-      JSON.pretty_generate(result)
-    end
-  end
-  end
-end

lib/vmpooler/dashboard.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class Dashboard < Sinatra::Base
-
     def config
       Vmpooler.config
     end

lib/vmpooler/dns.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Vmpooler
+  class Dns
+    # Load one or more VMPooler DNS plugin gems by name
+    #
+    # @param names [Array<String>] The list of gem names to load
+    def self.load_by_name(names)
+      names = Array(names)
+      instance = new
+      names.map { |name| instance.load_from_gems(name) }.flatten
+    end
+
+    # Returns the plugin class for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns class
+    # @return [String] The plugin class for the specifid dns config
+    def self.get_dns_plugin_class_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      plugin_class = ''
+
+      dns_configs.map do |dns_config_name|
+        plugin_class = config[:dns_configs][dns_config_name]['dns_class'] if dns_config_name.to_s == name
+      end
+
+      plugin_class
+    end
+
+    # Returns the domain for the specified pool
+    #
+    # @param config [String] - the full config structure
+    # @param pool_name [String] - the name of the pool
+    # @return [String] - domain name for pool, which is set via reference to the dns_configs block
+    def self.get_domain_for_pool(config, pool_name)
+      pool = config[:pools].find { |p| p['name'] == pool_name }
+      pool_dns_config = pool['dns_plugin']
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == pool_dns_config
+      end
+    end
+
+    # Returns the plugin domain for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns domain
+    # @return [String] The domain for the specifid dns config
+    def self.get_dns_plugin_domain_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == name
+      end
+    end
+
+    # Returns a list of DNS plugin classes specified in the vmpooler configuration
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @return nil || [Array<String>] A list of DNS plugin classes
+    def self.get_dns_plugin_config_classes(config)
+      return nil unless config[:dns_configs]
+
+      dns_configs = config[:dns_configs].keys
+      dns_plugins = dns_configs.map do |dns_config_name|
+        if config[:dns_configs][dns_config_name] && config[:dns_configs][dns_config_name]['dns_class']
+          config[:dns_configs][dns_config_name]['dns_class'].to_s
+        else
+          dns_config_name.to_s
+        end
+      end.compact.uniq
+
+      # dynamic-dns is not actually a class, it's just used as a value to denote
+      # that dynamic dns is used so no loading or record management is needed
+      dns_plugins.delete('dynamic-dns')
+
+      dns_plugins
+    end
+
+    # Load a single DNS plugin gem by name
+    #
+    # @param name [String] The name of the DNS plugin gem to load
+    # @return [String] The full require path to the specified gem
+    def load_from_gems(name = nil)
+      require_path = "vmpooler/dns/#{name.gsub('-', '/')}"
+      require require_path
+      require_path
+    end
+  end
+end

lib/vmpooler/dns/base.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class PoolManager
+    class Dns
+      class Base
+        # These defs must be overidden in child classes
+
+        # Helper Methods
+        # Global Logger object
+        attr_reader :logger
+        # Global Metrics object
+        attr_reader :metrics
+        # Provider options passed in during initialization
+        attr_reader :dns_options
+
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          @config = config
+          @logger = logger
+          @metrics = metrics
+          @redis = redis_connection_pool
+          @dns_plugin_name = name
+
+          @dns_options = options
+
+          logger.log('s', "[!] Creating dns plugin '#{name}'")
+        end
+
+        def pool_config(pool_name)
+          # Get the configuration of a specific pool
+          @config[:pools].each do |pool|
+            return pool if pool['name'] == pool_name
+          end
+
+          nil
+        end
+
+        # Returns this dns plugin's configuration
+        #
+        # @returns [Hashtable] This dns plugins's configuration from the config file.  Returns nil if the dns plugin config does not exist
+        def dns_config
+          @config[:dns_configs].each do |dns|
+            # Convert the symbol from the config into a string for comparison
+            return (dns[1].nil? ? {} : dns[1]) if dns[0].to_s == @dns_plugin_name
+          end
+
+          nil
+        end
+
+        def global_config
+          # This entire VM Pooler config
+          @config
+        end
+
+        def name
+          @dns_plugin_name
+        end
+
+        def get_ip(vm_name)
+          @redis.with_metrics do |redis|
+            redis.hget("vmpooler__vm__#{vm_name}", 'ip')
+          end
+        end
+
+        # returns
+        #   Array[String] : Array of pool names this provider services
+        def provided_pools
+          @config[:pools].select { |pool| pool['dns_config'] == name }.map { |pool| pool['name'] }
+        end
+
+        def create_or_replace_record(hostname)
+          raise("#{self.class.name} does not implement create_or_replace_record #{hostname}")
+        end
+
+        def delete_record(hostname)
+          raise("#{self.class.name} does not implement delete_record for #{hostname}")
+        end
+      end
+    end
+  end
+end

lib/vmpooler/dummy_statsd.rb

@@ -1,20 +0,0 @@
-module Vmpooler
-  class DummyStatsd
-    attr_reader :server, :port, :prefix
-
-    def initialize(*)
-    end
-
-    def increment(*)
-      true
-    end
-
-    def gauge(*)
-      true
-    end
-
-    def timing(*)
-      true
-    end
-  end
-end

lib/vmpooler/generic_connection_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'connection_pool'
 
 module Vmpooler
@@ -9,42 +11,26 @@ module Vmpooler
       def initialize(options = {}, &block)
         super(options, &block)
         @metrics = options[:metrics]
-        @metric_prefix = options[:metric_prefix]
-        @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
+        @connpool_type = options[:connpool_type]
+        @connpool_type = 'connectionpool' if @connpool_type.nil? || @connpool_type == ''
+        @connpool_provider = options[:connpool_provider]
+        @connpool_provider = 'unknown' if @connpool_provider.nil? || @connpool_provider == ''
       end
 
-      if Thread.respond_to?(:handle_interrupt)
-        # MRI
       def with_metrics(options = {})
         Thread.handle_interrupt(Exception => :never) do
           start = Time.now
           conn = checkout(options)
           timespan_ms = ((Time.now - start) * 1000).to_i
-            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-            @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
+          @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
+          @metrics&.timing("connection_waited.#{@connpool_type}.#{@connpool_provider}", timespan_ms)
           begin
             Thread.handle_interrupt(Exception => :immediate) do
               yield conn
             end
           ensure
             checkin
-              @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-            end
-          end
-        end
-      else
-        # jruby 1.7.x
-        def with_metrics(options = {})
-          start = Time.now
-          conn = checkout(options)
-          timespan_ms = ((Time.now - start) * 1000).to_i
-          @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-          @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
-          begin
-            yield conn
-          ensure
-            checkin
-            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+            @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
           end
         end
       end

lib/vmpooler/graphite.rb

@@ -1,42 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-module Vmpooler
-  class Graphite
-    attr_reader :server, :port, :prefix
-
-    def initialize(params = {})
-      if params['server'].nil? || params['server'].empty?
-        raise ArgumentError, "Graphite server is required. Config: #{params.inspect}"
-      end
-
-      @server = params['server']
-      @port   = params['port'] || 2003
-      @prefix = params['prefix'] || 'vmpooler'
-    end
-
-    def increment(label)
-      log label, 1
-    end
-
-    def gauge(label, value)
-      log label, value
-    end
-
-    def timing(label, duration)
-      log label, duration
-    end
-
-    def log(path, value)
-      Thread.new do
-        socket = TCPSocket.new(server, port)
-        begin
-          socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
-        ensure
-          socket.close
-        end
-      end
-    rescue => err
-      $stderr.puts "Failure logging #{path} to graphite server [#{server}:#{port}]: #{err}"
-    end
-  end
-end

lib/vmpooler/logger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems' unless defined?(Gem)
 
 module Vmpooler
@@ -14,7 +16,7 @@ module Vmpooler
 
       puts "[#{stamp}] #{string}" if ENV['VMPOOLER_DEBUG']
 
-      open(@file, 'a') do |f|
+      File.open(@file, 'a') do |f|
         f.puts "[#{stamp}] #{string}"
       end
     end

lib/vmpooler/metrics.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'vmpooler/metrics/statsd'
+require 'vmpooler/metrics/graphite'
+require 'vmpooler/metrics/promstats'
+require 'vmpooler/metrics/dummy_statsd'
+
+module Vmpooler
+  class Metrics
+    # static class instantiate appropriate metrics object.
+    def self.init(logger, params)
+      if params[:statsd]
+        metrics = Vmpooler::Metrics::Statsd.new(logger, params[:statsd])
+      elsif params[:graphite]
+        metrics = Vmpooler::Metrics::Graphite.new(logger, params[:graphite])
+      elsif params[:prometheus]
+        metrics = Vmpooler::Metrics::Promstats.new(logger, params[:prometheus])
+      else
+        metrics = Vmpooler::Metrics::DummyStatsd.new
+      end
+      metrics
+    end
+  end
+end

lib/vmpooler/metrics/dummy_statsd.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class Metrics
+    class DummyStatsd < Metrics
+      attr_reader :server, :port, :prefix
+
+      def increment(*)
+        true
+      end
+
+      def gauge(*)
+        true
+      end
+
+      def timing(*)
+        true
+      end
+    end
+  end
+end

lib/vmpooler/metrics/graphite.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Metrics
+    class Graphite < Metrics
+      attr_reader :server, :port, :prefix
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
+
+        @server = params['server']
+        @port   = params['port'] || 2003
+        @prefix = params['prefix'] || 'vmpooler'
+        @logger = logger
+      end
+      # rubocop:enable Lint/MissingSuper
+
+      def increment(label)
+        log label, 1
+      end
+
+      def gauge(label, value)
+        log label, value
+      end
+
+      def timing(label, duration)
+        log label, duration
+      end
+
+      def log(path, value)
+        Thread.new do
+          socket = TCPSocket.new(server, port)
+          begin
+            socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
+          ensure
+            socket.close
+          end
+        end
+      rescue Errno::EADDRNOTAVAIL => e
+        warn "Could not assign address to graphite server #{server}: #{e}"
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}")
+      end
+    end
+  end
+end

lib/vmpooler/metrics/promstats.rb

@@ -0,0 +1,501 @@
+# frozen_string_literal: true
+
+require 'prometheus/client'
+
+module Vmpooler
+  class Metrics
+    class Promstats < Metrics
+      attr_reader :prefix, :prometheus_endpoint, :prometheus_prefix
+
+      # Constants for Metric Types
+      M_COUNTER   = 1
+      M_GAUGE     = 2
+      M_SUMMARY   = 3
+      M_HISTOGRAM = 4
+
+      # Customised Bucket set to use for the Pooler clone times set to more appropriate intervals.
+      POOLER_CLONE_TIME_BUCKETS = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 120.0, 180.0, 240.0, 300.0, 600.0].freeze
+      POOLER_READY_TIME_BUCKETS = [30.0, 60.0, 120.0, 180.0, 240.0, 300.0, 500.0, 800.0, 1200.0, 1600.0].freeze
+      # Same for redis connection times - this is the same as the current Prometheus Default.
+      # https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14
+      REDIS_CONNECT_BUCKETS = [1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 18.0, 23.0].freeze
+
+      @p_metrics = {}
+      @torun = []
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        @prefix = params['prefix'] || 'vmpooler'
+        @prometheus_prefix = params['prometheus_prefix'] || 'vmpooler'
+        @prometheus_endpoint = params['prometheus_endpoint'] || '/prometheus'
+        @logger = logger
+
+        # Setup up prometheus registry and data structures
+        @prometheus = Prometheus::Client.registry
+      end
+# rubocop:enable Lint/MissingSuper
+
+=begin # rubocop:disable Style/BlockComments
+      The Metrics table is used to register metrics and translate/interpret the incoming metrics.
+
+      This table describes all of the prometheus metrics that are recognised by the application.
+      The background documentation for defining metrics is at: https://prometheus.io/docs/introduction/
+      In particular, the naming practices should be adhered to: https://prometheus.io/docs/practices/naming/
+      The Ruby Client docs are also useful: https://github.com/prometheus/client_ruby
+
+      The table here allows the currently used stats definitions to be translated correctly for Prometheus.
+      The current format is of the form A.B.C, where the final fields may be actual values (e.g. poolname).
+      Prometheus metrics cannot use the '.' as a character, so this is either translated into '_' or
+      variable parameters are expressed as labels accompanying the metric.
+
+      Sample statistics are:
+          # Example showing hostnames (FQDN)
+          migrate_from.pix-jj26-chassis1-2.ops.puppetlabs.net
+          migrate_to.pix-jj26-chassis1-8.ops.puppetlabs.net
+
+          # Example showing poolname as a parameter
+          poolreset.invalid.centos-8-x86_64
+
+          # Examples showing similar sub-typed checkout stats
+          checkout.empty.centos-8-x86_64
+          checkout.invalid.centos-8-x86_64
+          checkout.invalid.unknown
+          checkout.success.centos-8-x86_64
+
+          # Stats without any final parameter.
+          connect.fail
+          connect.open
+          delete.failed
+          delete.success
+
+          # Stats with multiple param_labels
+          vmpooler_user.debian-8-x86_64-pixa4.john
+
+        The metrics implementation here preserves the existing framework which will continue to support
+        graphite and statsd (since vmpooler is used outside of puppet). Some rationalisation and renaming
+        of the actual metrics was done to get a more usable model to fit within the prometheus framework.
+        This particularly applies to the user stats collected once individual machines are terminated as
+        this would have challenged prometheus' ability due to multiple (8) parameters being collected
+        in a single measure (which has a very high cardinality).
+
+        Prometheus requires all metrics to be pre-registered (which is the primary reason for this
+        table) and also uses labels to differentiate the characteristics of the measurement. This
+        is used throughout to capture information such as poolnames. So for example, this is a sample
+        of the prometheus metrics generated for the "vmpooler_ready" measurement:
+
+          # TYPE vmpooler_ready gauge
+          # HELP vmpooler_ready vmpooler number of machines in ready State
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="win-10-ent-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="debian-8-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="centos-8-x86_64-pixa4"} 2.0
+
+        Prometheus supports the following metric types:
+        (see https://prometheus.io/docs/concepts/metric_types/)
+
+          Counter (increment):
+            A counter is a cumulative metric that represents a single monotonically increasing counter whose
+            value can only increase or be reset to zero on restart
+
+          Gauge:
+            A gauge is a metric that represents a single numerical value that can arbitrarily go up and down.
+
+          Histogram:
+            A histogram samples observations (usually things like request durations or response sizes) and
+            counts them in configurable buckets. It also provides a sum of all observed values.
+            This replaces the timer metric supported by statsd
+
+          Summary :
+            Summary provides a total count of observations and a sum of all observed values, it calculates
+            configurable quantiles over a sliding time window.
+            (Summary is not used in vmpooler)
+
+        vmpooler_metrics_table is a table of hashes, where the hash key represents the first part of the
+        metric name, e.g. for the metric 'delete.*' (see above) the key would be 'delete:'. "Sub-metrics",
+        are supported, again for the 'delete.*' example, this can be subbed into '.failed' and '.success'
+
+        The entries within the hash as are follows:
+
+          mtype:
+            Metric type, which is one of the following constants:
+              M_COUNTER   = 1
+              M_GAUGE     = 2
+              M_SUMMARY   = 3
+              M_HISTOGRAM = 4
+
+          torun:
+            Indicates which process the metric is for - within vmpooler this is either ':api' or ':manager'
+            (there is a suggestion that we change this to two separate tables).
+
+          docstring:
+            Documentation string for the metric - this is displayed as HELP text by the endpoint.
+
+          metric_suffixes:
+            Array of sub-metrics of the form 'sub-metric: "doc-string for sub-metric"'. This supports
+            the generation of individual sub-metrics for all elements in the array.
+
+          param_labels:
+            This is an optional array of symbols for the final labels in a metric. It should not be
+            specified if there are no additional parameters.
+
+            If it specified, it can either be a single symbol, or two or more symbols. The treatment
+            differs if there is only one symbol given as all of the remainder of the metric string
+            supplied is collected into a label with the symbol name. This allows the handling of
+            node names (FQDN).
+
+            To illustrate:
+            1. In the 'connect.*' or 'delete.*' example above, it should not be specified.
+            2. For the 'migrate_from.*' example above, the remainder of the measure is collected
+               as the 'host_name' label.
+            3. For the 'vmpooler_user' example above, the first parameter is treated as the pool
+               name, and the second as the username.
+
+=end
+      def vmpooler_metrics_table
+        {
+          errors: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'Count of errors for pool',
+            metric_suffixes: {
+              markedasfailed: 'timeout waiting for instance to initialise',
+              duplicatehostname: 'unable to create instance due to duplicate hostname',
+              staledns: 'unable to create instance due to duplicate DNS record'
+            },
+            param_labels: %i[template_name]
+          },
+          user: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by a user',
+            param_labels: %i[user operation poolname]
+          },
+          usage_litmus: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Litmus jobs',
+            param_labels: %i[user operation poolname]
+          },
+          usage_jenkins_instance: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins instances',
+            param_labels: %i[jenkins_instance value_stream operation poolname]
+          },
+          usage_branch_project: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins branch/project',
+            param_labels: %i[branch project operation poolname]
+          },
+          usage_job_component: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins job/component',
+            param_labels: %i[job_name component_to_test operation poolname]
+          },
+          checkout: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Pool checkout counts',
+            metric_suffixes: {
+              nonresponsive: 'checkout failed - non responsive machine',
+              empty: 'checkout failed - no machine',
+              success: 'successful checkout',
+              invalid: 'checkout failed - invalid template'
+            },
+            param_labels: %i[poolname]
+          },
+          delete: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Delete machine',
+            metric_suffixes: {
+              success: 'succeeded',
+              failed: 'failed'
+            },
+            param_labels: []
+          },
+          ondemandrequest_generate: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Ondemand request',
+            metric_suffixes: {
+              duplicaterequests: 'failed duplicate request',
+              success: 'succeeded'
+            },
+            param_labels: []
+          },
+          ondemandrequest_fail: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Ondemand request failure',
+            metric_suffixes: {
+              toomanyrequests: 'too many requests',
+              invalid: 'invalid poolname'
+            },
+            param_labels: %i[poolname]
+          },
+          config: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'vmpooler pool configuration request',
+            metric_suffixes: { invalid: 'Invalid' },
+            param_labels: %i[poolname]
+          },
+          poolreset: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Pool reset counter',
+            metric_suffixes: { invalid: 'Invalid Pool' },
+            param_labels: %i[poolname]
+          },
+          connect: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler connect (to vSphere)',
+            metric_suffixes: {
+              open: 'Connect Succeeded',
+              fail: 'Connect Failed'
+            },
+            param_labels: []
+          },
+          migrate_from: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler machine migrated from',
+            param_labels: %i[host_name]
+          },
+          migrate_to: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler machine migrated to',
+            param_labels: %i[host_name]
+          },
+          http_requests_vm_total: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Total number of HTTP request/sub-operations handled by the Rack application under the /vm endpoint',
+            param_labels: %i[method subpath operation]
+          },
+          ready: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler number of machines in ready State',
+            param_labels: %i[poolname]
+          },
+          running: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler number of machines running',
+            param_labels: %i[poolname]
+          },
+          connection_available: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler redis connections available',
+            param_labels: %i[type provider]
+          },
+          time_to_ready_state: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_READY_TIME_BUCKETS,
+            docstring: 'Time taken for machine to read ready state for pool',
+            param_labels: %i[poolname]
+          },
+          migrate: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to migrate machine for pool',
+            param_labels: %i[poolname]
+          },
+          clone: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to clone machine',
+            param_labels: %i[poolname]
+          },
+          destroy: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to destroy machine',
+            param_labels: %i[poolname]
+          },
+          connection_waited: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: REDIS_CONNECT_BUCKETS,
+            docstring: 'vmpooler redis connection wait time',
+            param_labels: %i[type provider]
+          },
+          vmpooler_health: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler health check metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_purge: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler purge metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_destroy: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler destroy metrics',
+            param_labels: %i[poolname]
+          },
+          vmpooler_clone: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler clone metrics',
+            param_labels: %i[poolname]
+          }
+        }
+      end
+
+      # Helper to add individual prom metric.
+      # Allow Histograms to specify the bucket size.
+      def add_prometheus_metric(metric_spec, name, docstring)
+        case metric_spec[:mtype]
+        when M_COUNTER
+          metric_class = Prometheus::Client::Counter
+        when M_GAUGE
+          metric_class = Prometheus::Client::Gauge
+        when M_SUMMARY
+          metric_class = Prometheus::Client::Summary
+        when M_HISTOGRAM
+          metric_class = Prometheus::Client::Histogram
+        else
+          raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}")
+        end
+
+        if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets)
+          prom_metric = metric_class.new(
+            name.to_sym,
+            docstring: docstring,
+            labels: metric_spec[:param_labels] + [:vmpooler_instance],
+            buckets: metric_spec[:buckets],
+            preset_labels: { vmpooler_instance: @prefix }
+          )
+        else
+          prom_metric = metric_class.new(
+            name.to_sym,
+            docstring: docstring,
+            labels: metric_spec[:param_labels] + [:vmpooler_instance],
+            preset_labels: { vmpooler_instance: @prefix }
+          )
+        end
+        @prometheus.register(prom_metric)
+      end
+
+      # Top level method to register all the prometheus metrics.
+
+      def setup_prometheus_metrics(torun)
+        @torun = torun
+        @p_metrics = vmpooler_metrics_table
+        @p_metrics.each do |name, metric_spec|
+          # Only register metrics appropriate to api or manager
+          next if (torun & metric_spec[:torun]).empty?
+
+          if metric_spec.key? :metric_suffixes
+            # Iterate thru the suffixes if provided to register multiple counters here.
+            metric_spec[:metric_suffixes].each do |metric_suffix|
+              add_prometheus_metric(
+                metric_spec,
+                "#{@prometheus_prefix}_#{name}_#{metric_suffix[0]}",
+                "#{metric_spec[:docstring]} #{metric_suffix[1]}"
+              )
+            end
+          else
+            # No Additional counter suffixes so register this as metric.
+            add_prometheus_metric(
+              metric_spec,
+              "#{@prometheus_prefix}_#{name}",
+              metric_spec[:docstring]
+            )
+          end
+        end
+      end
+
+      # locate a metric and check/interpet the sub-fields.
+      def find_metric(label)
+        sublabels = label.split('.')
+        metric_key = sublabels.shift.to_sym
+        raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key
+
+        metric_spec = @p_metrics[metric_key]
+        raise("Invalid Component #{component} for #{metric_key}") if (metric_spec[:torun] & @torun).nil?
+
+        metric = metric_spec.clone
+
+        if metric.key? :metric_suffixes
+          metric_subkey = sublabels.shift.to_sym
+          raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym
+
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}_#{metric_subkey}"
+        else
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}"
+        end
+
+        # Check if we are looking for a parameter value at last element.
+        if metric.key? :param_labels
+          metric[:labels] = {}
+          # Special case processing here - if there is only one parameter label then make sure
+          # we append all of the remaining contents of the metric with "." separators to ensure
+          # we get full nodenames (e.g. for Migration to node operations)
+          if metric[:param_labels].length == 1
+            metric[:labels][metric[:param_labels].first] = sublabels.join('.')
+          else
+            metric[:param_labels].reverse_each do |param_label|
+              metric[:labels][param_label] = sublabels.pop(1).first
+            end
+          end
+        end
+        metric
+      end
+
+      # Helper to get lab metrics.
+      def get(label)
+        metric = find_metric(label)
+        [metric, @prometheus.get(metric[:metric_name])]
+      end
+
+      # Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation.
+      def increment(label)
+        begin
+          counter_metric, c = get(label)
+          c.increment(labels: counter_metric[:labels])
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}")
+        end
+      end
+
+      def gauge(label, value)
+        begin
+          unless value.nil?
+            gauge_metric, g = get(label)
+            g.set(value.to_i, labels: gauge_metric[:labels])
+          end
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}")
+        end
+      end
+
+      def timing(label, duration)
+        begin
+          # https://prometheus.io/docs/practices/histograms/
+          unless duration.nil?
+            histogram_metric, hm = get(label)
+            hm.observe(duration.to_f, labels: histogram_metric[:labels])
+          end
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}")
+        end
+      end
+    end
+  end
+end

lib/vmpooler/metrics/promstats/collector_middleware.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# This is an adapted Collector module for vmpooler based on the sample implementation
+# available in the prometheus client_ruby library
+# https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/middleware/collector.rb
+#
+# The code was also failing Rubocop on PR check, so have addressed all the offenses.
+#
+# The method strip_hostnames_from_path (originally strip_ids_from_path) has been adapted
+# to add a match for hostnames in paths # to replace with a single ":hostname" string to
+# avoid # proliferation of stat lines for # each new vm hostname deleted, modified or
+# otherwise queried.
+
+require 'benchmark'
+require 'prometheus/client'
+require 'vmpooler/logger'
+
+module Vmpooler
+  class Metrics
+    class Promstats
+      # CollectorMiddleware is an implementation of Rack Middleware customised
+      # for vmpooler use.
+      #
+      # By default metrics are registered on the global registry. Set the
+      # `:registry` option to use a custom registry.
+      #
+      # By default metrics all have the prefix "http_server". Set to something
+      # else if you like.
+      #
+      # The request counter metric is broken down by code, method and path by
+      # default. Set the `:counter_label_builder` option to use a custom label
+      # builder.
+      #
+      # The request duration metric is broken down by method and path by default.
+      # Set the `:duration_label_builder` option to use a custom label builder.
+      #
+      # Label Builder functions will receive a Rack env and a status code, and must
+      # return a hash with the labels for that request. They must also accept an empty
+      # env, and return a hash with the correct keys. This is necessary to initialize
+      # the metrics with the correct set of labels.
+      class CollectorMiddleware
+        attr_reader :app, :registry
+
+        def initialize(app, options = {})
+          @app = app
+          @registry = options[:registry] || Prometheus::Client.registry
+          @metrics_prefix = options[:metrics_prefix] || 'http_server'
+
+          init_request_metrics
+          init_exception_metrics
+        end
+
+        def call(env) # :nodoc:
+          trace(env) { @app.call(env) }
+        end
+
+        protected
+
+        def init_request_metrics
+          @requests = @registry.counter(
+            :"#{@metrics_prefix}_requests_total",
+            docstring:
+              'The total number of HTTP requests handled by the Rack application.',
+            labels: %i[code method path]
+          )
+          @durations = @registry.histogram(
+            :"#{@metrics_prefix}_request_duration_seconds",
+            docstring: 'The HTTP response duration of the Rack application.',
+            labels: %i[method path]
+          )
+        end
+
+        def init_exception_metrics
+          @exceptions = @registry.counter(
+            :"#{@metrics_prefix}_exceptions_total",
+            docstring: 'The total number of exceptions raised by the Rack application.',
+            labels: [:exception]
+          )
+        end
+
+        def trace(env)
+          response = nil
+          duration = Benchmark.realtime { response = yield }
+          record(env, response.first.to_s, duration)
+          response
+        rescue StandardError => e
+          @exceptions.increment(labels: { exception: e.class.name })
+          raise
+        end
+
+        def record(env, code, duration)
+          counter_labels = {
+            code: code,
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          duration_labels = {
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          @requests.increment(labels: counter_labels)
+          @durations.observe(duration, labels: duration_labels)
+        rescue # rubocop:disable Style/RescueStandardError
+          nil
+        end
+
+        def strip_hostnames_from_path(path)
+          # Custom for /vm path - so we just collect aggrate stats for all usage along this one
+          # path. Custom counters are then added more specific endpoints in v1.rb
+          # Since we aren't parsing UID/GIDs as in the original example, these are removed.
+          # Similarly, request IDs are also stripped from the /ondemand path.
+          path
+            .gsub(%r{/vm/.+$}, '/vm')
+            .gsub(%r{/ondemandvm/.+$}, '/ondemandvm')
+            .gsub(%r{/token/.+$}, '/token')
+            .gsub(%r{/lib/.+$}, '/lib')
+            .gsub(%r{/img/.+$}, '/img')
+        end
+      end
+    end
+  end
+end

lib/vmpooler/metrics/statsd.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'rubygems' unless defined?(Gem)
+require 'statsd'
+
+module Vmpooler
+  class Metrics
+    class Statsd < Metrics
+      attr_reader :server, :port, :prefix
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
+
+        host    = params['server']
+        @port   = params['port'] || 8125
+        @prefix = params['prefix'] || 'vmpooler'
+        @server = ::Statsd.new(host, @port)
+        @logger = logger
+      end
+      # rubocop:enable Lint/MissingSuper
+
+      def increment(label)
+        server.increment("#{prefix}.#{label}")
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def gauge(label, value)
+        server.gauge("#{prefix}.#{label}", value)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def timing(label, duration)
+        server.timing("#{prefix}.#{label}", duration)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+    end
+  end
+end

lib/vmpooler/providers.rb

@@ -1,27 +1,28 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Vmpooler
   class Providers
-
     # @param names [Array] - an array of names or string name of a provider
     # @return [Array] - list of provider files loaded
     # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
     def self.load_by_name(names)
       names = Array(names)
-      instance = self.new
-      names.map {|name| instance.load_from_gems(name)}.flatten
+      instance = new
+      names.map { |name| instance.load_from_gems(name) }.flatten
     end
 
     # @return [Array] - array of provider files
     # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
     # although these files can come from any gem
     def self.load_all_providers
-      self.new.load_from_gems
+      new.load_from_gems
     end
 
     # @return [Array] - returns an array of gem names that contain a provider
     def self.installed_providers
-      self.new.vmpooler_provider_gem_list.map(&:name)
+      new.vmpooler_provider_gem_list.map(&:name)
     end
 
     # @return [Array] returns a list of vmpooler providers gem plugin specs
@@ -38,7 +39,7 @@ module Vmpooler
         # we don't exactly know if the provider name matches the main file name that should be loaded
         # so we use globs to get everything like the name
         # this could mean that vsphere5 and vsphere6 are loaded when only vsphere5 is used
-        Dir.glob(File.join(gem_path, "*#{name}*.rb")).each do |file|
+        Dir.glob(File.join(gem_path, "*#{name}*.rb")).sort.each do |file|
           require file
         end
       end
@@ -50,7 +51,7 @@ module Vmpooler
     # @return [String] - the relative path to the vmpooler provider dir
     # this is used when searching gems for this path
     def provider_path
-      File.join('lib','vmpooler','providers')
+      File.join('lib', 'vmpooler', 'providers')
     end
 
     # Add constants to array to skip over classes, ie. Vmpooler::PoolManager::Provider::Dummy
@@ -81,26 +82,24 @@ module Vmpooler
       @plugin_map ||= Hash[plugin_classes.map { |gem| [gem.send(:name), gem] }]
     end
 
-
-
     # Internal: Retrieve a list of available gem paths from RubyGems.
     #
     # Returns an Array of Pathname objects.
     def gem_directories
       dirs = []
-      if has_rubygems?
+      if rubygems?
         dirs = gemspecs.map do |spec|
-          lib_path = File.expand_path(File.join(spec.full_gem_path,provider_path))
-          lib_path if File.exists? lib_path
+          lib_path = File.expand_path(File.join(spec.full_gem_path, provider_path))
+          lib_path if File.exist? lib_path
         end + included_lib_dirs
       end
-      dirs.reject { |dir| dir.nil? }.uniq
+      dirs.reject(&:nil?).uniq
     end
 
     # Internal: Check if RubyGems is loaded and available.
     #
     # Returns true if RubyGems is available, false if not.
-    def has_rubygems?
+    def rubygems?
       defined? ::Gem
     end
 
@@ -114,6 +113,5 @@ module Vmpooler
                       Gem.searcher.init_gemspecs
                     end
     end
-
   end
 end

lib/vmpooler/providers/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class PoolManager
     class Provider
@@ -12,10 +14,11 @@ module Vmpooler
         # Provider options passed in during initialization
         attr_reader :provider_options
 
-        def initialize(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
           @config = config
           @logger = logger
           @metrics = metrics
+          @redis = redis_connection_pool
           @provider_name = name
 
           # Ensure that there is not a nil provider configuration
@@ -55,6 +58,10 @@ module Vmpooler
           nil
         end
 
+        def dns_config(dns_config_name)
+          Vmpooler::Dns.get_dns_plugin_domain_by_name(@config, dns_config_name)
+        end
+
         # returns
         #   [Hashtable] : The entire VMPooler configuration
         def global_config
@@ -209,6 +216,22 @@ module Vmpooler
           raise("#{self.class.name} does not implement vm_ready?")
         end
 
+        # tag_vm_user This method is called once we know who is using the VM (it is running). This method enables seeing
+        # who is using what in the provider pools.
+        # This method should be implemented in the providers, if it is not implemented, this base method will be called
+        # and should be a noop. The implementation should check if the vm has a user (as per redis) and add a new tag
+        # with the information.
+        # inputs
+        #   [String] pool_name : Name of the pool
+        #   [String] vm_name   : Name of the VM to check if ready
+        # returns
+        #   [Boolean] : true if successful, false if an error occurred and it should retry
+        def tag_vm_user(_pool_name, _vm_name)
+          # noop by design. If the provider does not implement this method, this base method is called (because inherited)
+          # and should basically do nothing.
+          true
+        end
+
         # inputs
         #   [String] pool_name : Name of the pool
         #   [String] vm_name   : Name of the VM to check if it exists
@@ -222,20 +245,31 @@ module Vmpooler
         #   [Hash] pool : Configuration for the pool
         # returns
         #   nil when successful. Raises error when encountered
-        def create_template_delta_disks(pool)
-          raise("#{self.class.name} does not implement create_template_delta_disks")
+        def create_template_delta_disks(_pool)
+          puts("#{self.class.name} does not implement create_template_delta_disks")
         end
 
         # inputs
         #   [String] provider_name : Name of the provider
         # returns
         #   Hash of folders
-        def get_target_datacenter_from_config(provider_name)
+        def get_target_datacenter_from_config(_provider_name)
           raise("#{self.class.name} does not implement get_target_datacenter_from_config")
         end
 
-        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        def purge_unconfigured_resources(_allowlist)
+          raise("#{self.class.name} does not implement purge_unconfigured_resources")
+        end
+
+        def get_vm_ip_address(vm_name, pool_name)
+          raise("#{self.class.name} does not implement get_vm_ip_address for vm #{vm_name} in pool #{pool_name}")
+        end
+
+        # DEPRECATED if a provider does not implement the new method, it will hit this base class method
+        # and return a deprecation message
+        def purge_unconfigured_folders(_deprecated, _deprecated2, allowlist)
+          logger.log('s', '[!] purge_unconfigured_folders was renamed to purge_unconfigured_resources, please update your provider implementation')
+          purge_unconfigured_resources(allowlist)
         end
       end
     end

lib/vmpooler/providers/dummy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'vmpooler/providers/base'
 
@@ -7,8 +9,8 @@ module Vmpooler
       class Dummy < Vmpooler::PoolManager::Provider::Base
         # Fake VM Provider for testing
 
-        def initialize(config, logger, metrics, name, options)
-          super(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          super(config, logger, metrics, redis_connection_pool, name, options)
           dummyfilename = provider_config['filename']
 
           # This initial_state option is only intended to be used by spec tests
@@ -27,7 +29,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do
@@ -73,10 +76,10 @@ module Vmpooler
           return current_vm['vm_host'] if provider_config['migratevm_couldmove_percent'].nil?
 
           # Only migrate if migratevm_couldmove_percent is met
-          return current_vm['vm_host'] if 1 + rand(100) > provider_config['migratevm_couldmove_percent']
+          return current_vm['vm_host'] if rand(1..100) > provider_config['migratevm_couldmove_percent']
 
           # Simulate a 10 node cluster and randomly pick a different one
-          new_host = 'HOST' + (1 + rand(10)).to_s while new_host == current_vm['vm_host']
+          new_host = "HOST#{rand(1..10)}" while new_host == current_vm['vm_host']
 
           new_host
         end
@@ -92,9 +95,7 @@ module Vmpooler
             end
 
             # Inject clone failure
-            unless provider_config['migratevm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['migratevm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['migratevm_fail_percent'].nil? && rand(1..100) <= provider_config['migratevm_fail_percent']
 
             @write_lock.synchronize do
               current_vm = get_dummy_vm(pool_name, vm_name)
@@ -113,8 +114,7 @@ module Vmpooler
             return nil if dummy.nil?
 
             # Randomly power off the VM
-            unless dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?
-              if 1 + rand(100) <= provider_config['getvm_poweroff_percent']
+            if !(dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?) && rand(1..100) <= provider_config['getvm_poweroff_percent']
               @write_lock.synchronize do
                 dummy = get_dummy_vm(pool_name, vm_name)
                 dummy['powerstate'] = 'PoweredOff'
@@ -122,19 +122,16 @@ module Vmpooler
               end
               logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
             end
-            end
 
             # Randomly rename the host
-            unless dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?
-              if 1 + rand(100) <= provider_config['getvm_rename_percent']
+            if !(dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?) && rand(1..100) <= provider_config['getvm_rename_percent']
               @write_lock.synchronize do
                 dummy = get_dummy_vm(pool_name, vm_name)
-                  dummy['hostname'] = 'DUMMY' + dummy['name']
+                dummy['hostname'] = "DUMMY#{dummy['name']}"
                 write_backing_file
               end
               logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
             end
-            end
 
             obj['name'] = dummy['name']
             obj['hostname'] = dummy['hostname']
@@ -193,16 +190,14 @@ module Vmpooler
 
             begin
               # Inject clone failure
-              unless provider_config['createvm_fail_percent'].nil?
-                raise('Dummy Failure for createvm_fail_percent') if 1 + rand(100) <= provider_config['createvm_fail_percent']
-              end
+              raise('Dummy Failure for createvm_fail_percent') if !provider_config['createvm_fail_percent'].nil? && rand(1..100) <= provider_config['createvm_fail_percent']
 
               # Assert the VM is ready for use
               @write_lock.synchronize do
                 vm['dummy_state'] = 'RUNNING'
                 write_backing_file
               end
-            rescue => _err
+            rescue StandardError => _e
               @write_lock.synchronize do
                 remove_dummy_vm(pool_name, dummy_hostname)
                 write_backing_file
@@ -226,9 +221,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createdisk_fail_percent'].nil?
-              raise('Dummy Failure for createdisk_fail_percent') if 1 + rand(100) <= provider_config['createdisk_fail_percent']
-            end
+            raise('Dummy Failure for createdisk_fail_percent') if !provider_config['createdisk_fail_percent'].nil? && rand(1..100) <= provider_config['createdisk_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -252,9 +245,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for createsnapshot_fail_percent') if 1 + rand(100) <= provider_config['createsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for createsnapshot_fail_percent') if !provider_config['createsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['createsnapshot_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -279,9 +270,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['revertsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for revertsnapshot_fail_percent') if 1 + rand(100) <= provider_config['revertsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for revertsnapshot_fail_percent') if !provider_config['revertsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['revertsnapshot_fail_percent']
           end
 
           vm_object['snapshots'].include?(snapshot_name)
@@ -317,9 +306,7 @@ module Vmpooler
             end
 
             # Inject destroy VM failure
-            unless provider_config['destroyvm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['destroyvm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['destroyvm_fail_percent'].nil? && rand(1..100) <= provider_config['destroyvm_fail_percent']
 
             # 'Destroy' the VM
             @write_lock.synchronize do
@@ -351,9 +338,7 @@ module Vmpooler
             # it's ready to receive a connection
             sleep(2)
 
-            unless provider_config['vmready_fail_percent'].nil?
-              raise('Dummy Failure for vmready_fail_percent') if 1 + rand(100) <= provider_config['vmready_fail_percent']
-            end
+            raise('Dummy Failure for vmready_fail_percent') if !provider_config['vmready_fail_percent'].nil? && rand(1..100) <= provider_config['vmready_fail_percent']
 
             @write_lock.synchronize do
               vm_object['ready'] = true
@@ -370,6 +355,7 @@ module Vmpooler
 
         def remove_dummy_vm(pool_name, vm_name)
           return if @dummylist['pool'][pool_name].nil?
+
           new_poollist = @dummylist['pool'][pool_name].delete_if { |vm| vm['name'] == vm_name }
           @dummylist['pool'][pool_name] = new_poollist
         end
@@ -395,6 +381,7 @@ module Vmpooler
         def write_backing_file
           dummyfilename = provider_config['filename']
           return if dummyfilename.nil?
+
           File.open(dummyfilename, 'w') { |file| file.write(YAML.dump(@dummylist)) }
         end
       end

lib/vmpooler/public/lib/dashboard.js

@@ -29,10 +29,10 @@ Date.prototype.yyyymmdd = function() {
 
 var data_url = {
   'capacity': '/dashboard/stats/vmpooler/pool',
-  'pools'   : '/api/v1/vm',
+  'pools'   : '/api/v3/vm',
   'running' : '/dashboard/stats/vmpooler/running',
-  'status'  : '/api/v1/status',
-  'summary' : '/api/v1/summary'
+  'status'  : '/api/v3/status',
+  'summary' : '/api/v3/summary'
 };
 
 

lib/vmpooler/statsd.rb

@@ -1,37 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-require 'statsd'
-
-module Vmpooler
-  class Statsd
-    attr_reader :server, :port, :prefix
-
-    def initialize(params = {})
-      if params['server'].nil? || params['server'].empty?
-        raise ArgumentError, "Statsd server is required. Config: #{params.inspect}"
-      end
-
-      host    = params['server']
-      @port   = params['port'] || 8125
-      @prefix = params['prefix'] || 'vmpooler'
-      @server = ::Statsd.new(host, @port)
-    end
-
-    def increment(label)
-      server.increment(prefix + '.' + label)
-    rescue => err
-      $stderr.puts "Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-
-    def gauge(label, value)
-      server.gauge(prefix + '.' + label, value)
-    rescue => err
-      $stderr.puts "Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-
-    def timing(label, duration)
-      server.timing(prefix + '.' + label, duration)
-    rescue => err
-      $stderr.puts "Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-  end
-end

lib/vmpooler/util/parsing.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# utility class shared between apps api and pool_manager
+module Vmpooler
+  class Parsing
+    def self.get_platform_pool_count(requested, &_block)
+      requested_platforms = requested.split(',')
+      requested_platforms.each do |platform|
+        platform_alias, pool, count = platform.split(':')
+        raise ArgumentError if platform_alias.nil? || pool.nil? || count.nil?
+
+        yield platform_alias, pool, count
+      end
+    end
+  end
+end

lib/vmpooler/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
-  VERSION = '0.2.1'.freeze
+  VERSION = '3.8.1'
 end

release-notes.md

@@ -0,0 +1,15 @@
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
+
+**Implemented enhancements:**
+
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
+
+**Fixed bugs:**
+
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))

release-prep

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+#
+# Update Gemfile.lock
+docker run -t --rm \
+  -v $(pwd):/app \
+  jruby:9.4.12.1-jdk11 \
+  /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase && cd /app && gem install bundler && bundle install --jobs 3; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"'
+
+# Update Changelog
+docker run -t --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \
+  githubchangeloggenerator/github-changelog-generator:1.16.4 \
+  github_changelog_generator --future-release $(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev) \
+  --token $CHANGELOG_GITHUB_TOKEN --release-branch main
+

scripts/create_template_deltas.rb

@@ -1,98 +0,0 @@
-#!/usr/bin/ruby
-
-require 'rubygems'
-require 'rbvmomi'
-require 'yaml'
-
-def load_configuration( file_array )
-  file_array.each do |file|
-    file = File.expand_path( file )
-
-    if File.exists?( file )
-      return YAML.load_file( file )
-    end
-  end
-
-  return false
-end
-
-def create_template_deltas( folder )
-  config = load_configuration( [ 'vmpooler.yaml', '~/.vmpooler' ] ) || nil
-
-  abort 'No config file (./vmpooler.yaml or ~/.vmpooler) found!' unless config
-
-  vim = RbVmomi::VIM.connect(
-    :host     => config[ :providers ][ :vsphere ][ "server" ],
-    :user     => config[ :providers ][ :vsphere ][ "username" ],
-    :password => config[ :providers ][ :vsphere ][ "password" ],
-    :ssl      => true,
-    :insecure => true,
-  ) or abort "Unable to connect to #{config[ :vsphere ][ "server" ]}!"
-
-  containerView = vim.serviceContent.viewManager.CreateContainerView( {
-    :container => vim.serviceContent.rootFolder,
-    :recursive => true,
-    :type      => [ 'VirtualMachine' ]
-  } )
-
-  datacenter = vim.serviceInstance.find_datacenter
-  base = datacenter.vmFolder
-
-  case base
-    when RbVmomi::VIM::Folder
-      base = base.childEntity.find { |f| f.name == folder }
-    else
-      abort "Unexpected object type encountered (#{base.class}) while finding folder!"
-  end
-
-  unless base
-    abort "Folder #{ARGV[0]} not found!"
-  end
-
-  base.childEntity.each do |vm|
-    print vm.name
-
-    begin
-      disks = vm.config.hardware.device.grep( RbVmomi::VIM::VirtualDisk )
-    rescue
-      puts ' !'
-      next
-    end
-
-    begin
-      disks.select { |d| d.backing.parent == nil }.each do |disk|
-        linkSpec = {
-          :deviceChange => [
-            {
-              :operation => :remove,
-              :device => disk
-            },
-            {
-              :operation => :add,
-              :fileOperation => :create,
-              :device => disk.dup.tap { |x|
-                x.backing = x.backing.dup
-                x.backing.fileName = "[#{disk.backing.datastore.name}]"
-                x.backing.parent = disk.backing
-              }
-            }
-          ]
-        }
-
-        vm.ReconfigVM_Task( :spec => linkSpec ).wait_for_completion
-      end
-
-      puts " \u2713"
-    rescue
-      puts ' !'
-    end
-  end
-
-  vim.close
-end
-
-if ARGV[0]
-  create_template_deltas( ARGV[0] )
-else
-  puts "Usage: #{$0} <folder>"
-end

spec/fixtures/extra_config1.yaml

@@ -0,0 +1,4 @@
+---
+:providers:
+  :alice:
+    foo: "foo"

spec/fixtures/extra_config2.yaml

@@ -0,0 +1,12 @@
+---
+:providers:
+  :bob:
+    foo: "foo_bob"
+    bar: "bar"
+
+:pools:
+  - name: 'pool05'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5

spec/fixtures/vmpooler.yaml

@@ -17,15 +17,19 @@
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
-  vm_checktime: 15
+  vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,6 +40,10 @@
   - name: 'pool01'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
   - name: 'pool02'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5

spec/fixtures/vmpooler2.yaml

@@ -17,15 +17,19 @@
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
-  vm_checktime: 15
+  vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,6 +40,10 @@
   - name: 'pool03'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
   - name: 'pool04'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5

spec/fixtures/vmpooler_domain.yaml

@@ -0,0 +1,45 @@
+---
+:providers:
+  :dummy:
+
+:redis:
+  server: 'localhost'
+
+:auth:
+   provider: dummy
+
+:tagfilter:
+  url: '(.*)\/'
+
+:config:
+  site_name: 'vmpooler'
+  # Need to change this on Windows
+  logfile: '/var/log/vmpooler.log'
+  task_limit: 10
+  timeout: 15
+  vm_checktime: 1
+  vm_lifetime: 12
+  vm_lifetime_auth: 24
+  allowed_tags:
+    - 'created_by'
+    - 'project'
+  domain: 'example.com'
+  prefix: 'poolvm-'
+
+# Uncomment the lines below to suppress metrics to STDOUT
+# :statsd:
+#   server: 'localhost'
+#   prefix: 'vmpooler'
+#   port: 8125
+
+:pools:
+  - name: 'pool01'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
+  - name: 'pool02'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5

spec/helpers.rb

@@ -1,7 +1,8 @@
+require 'mock_redis'
+
 def redis
   unless @redis
-    @redis = Redis.new
-    @redis.select(15) # let's use the highest numbered database available in a default install
+    @redis = MockRedis.new
   end
   @redis
 end
@@ -13,6 +14,16 @@ class MockLogger
   end
 end
 
+class MockPoolManagerDnsBase
+  def delete_record(hostname)
+
+  end
+
+  def create_or_replace_record(hostname)
+
+  end
+end
+
 def expect_json(ok = true, http = 200)
   expect(last_response.header['Content-Type']).to eq('application/json')
 
@@ -39,85 +50,115 @@ def token_exists?(token)
   result && !result.empty?
 end
 
-def create_ready_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_ready_vm(template, name, redis, token = nil)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__ready__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_running_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_running_vm(template, name, redis, token = nil, user = nil)
+  create_vm(name, redis, token, user)
   redis.sadd("vmpooler__running__#{template}", name)
-  redis.hset("vmpooler__vm__#{name}", "template", template)
+  redis.hset("vmpooler__vm__#{name}", 'template', template)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'host', 'host1')
 end
 
-def create_pending_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_pending_vm(template, name, redis, token = nil)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__pending__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_vm(name, token = nil, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'token:token', token) if token
+def create_vm(name, redis, token = nil, user = nil)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'clone', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'token:token', token) if token
+  redis.hset("vmpooler__vm__#{name}", 'token:user', user) if user
 end
 
-def create_completed_vm(name, pool, active = false, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd("vmpooler__completed__#{pool}", name)
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__active__#{pool}", name, Time.now) if active
+def create_completed_vm(name, pool, redis, active = false)
+  redis.sadd("vmpooler__completed__#{pool}", name)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__active__#{pool}", name, Time.now) if active
 end
 
-def create_discovered_vm(name, pool, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd("vmpooler__discovered__#{pool}", name)
+def create_discovered_vm(name, pool, redis)
+  redis.sadd("vmpooler__discovered__#{pool}", name)
 end
 
-def create_migrating_vm(name, pool, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.sadd("vmpooler__migrating__#{pool}", name)
+def create_migrating_vm(name, pool, redis)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.sadd("vmpooler__migrating__#{pool}", name)
 end
 
-def add_vm_to_migration_set(name, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd('vmpooler__migration', name)
+def create_tag(vm, tag_name, tag_value, redis)
+  redis.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
+end
+
+def add_vm_to_migration_set(name, redis)
+  redis.sadd('vmpooler__migration', name)
 end
 
 def fetch_vm(vm)
   redis.hgetall("vmpooler__vm__#{vm}")
 end
 
-def snapshot_revert_vm(vm, snapshot = '12345678901234567890123456789012')
+def set_vm_data(vm, key, value, redis)
+  redis.hset("vmpooler__vm__#{vm}", key, value)
+end
+
+def snapshot_revert_vm(vm, snapshot = '12345678901234567890123456789012', redis)
   redis.sadd('vmpooler__tasks__snapshot-revert', "#{vm}:#{snapshot}")
   redis.hset("vmpooler__vm__#{vm}", "snapshot:#{snapshot}", "1")
 end
 
-def snapshot_vm(vm, snapshot = '12345678901234567890123456789012')
+def snapshot_vm(vm, snapshot = '12345678901234567890123456789012', redis)
   redis.sadd('vmpooler__tasks__snapshot', "#{vm}:#{snapshot}")
   redis.hset("vmpooler__vm__#{vm}", "snapshot:#{snapshot}", "1")
 end
 
-def disk_task_vm(vm, disk_size = '10')
+def disk_task_vm(vm, disk_size = '10', redis)
   redis.sadd('vmpooler__tasks__disk', "#{vm}:#{disk_size}")
 end
 
-def has_vm_snapshot?(vm)
+def has_vm_snapshot?(vm, redis)
   redis.smembers('vmpooler__tasks__snapshot').any? do |snapshot|
-    instance, sha = snapshot.split(':')
+    instance, _sha = snapshot.split(':')
     vm == instance
   end
 end
 
-def vm_reverted_to_snapshot?(vm, snapshot = nil)
+def vm_reverted_to_snapshot?(vm, redis, snapshot = nil)
   redis.smembers('vmpooler__tasks__snapshot-revert').any? do |action|
     instance, sha = action.split(':')
     instance == vm and (snapshot ? (sha == snapshot) : true)
   end
 end
 
-def pool_has_ready_vm?(pool, vm)
+def pool_has_ready_vm?(pool, vm, redis)
   !!redis.sismember('vmpooler__ready__' + pool, vm)
 end
+
+def create_ondemand_request_for_test(request_id, score, platforms_string, redis, user = nil, token = nil)
+  redis.zadd('vmpooler__provisioning__request', score, request_id)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'token:token', token) if token
+  redis.hset("vmpooler__odrequest__#{request_id}", 'token:user', user) if user
+end
+
+def set_ondemand_request_status(request_id, status, redis)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'status', status)
+end
+
+def create_ondemand_vm(vmname, request_id, pool, pool_alias, redis)
+  redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vmname)
+end
+
+def create_ondemand_creationtask(request_string, score, redis)
+  redis.zadd('vmpooler__odcreate__task', score, request_string)
+end
+
+def create_ondemand_processing(request_id, score, redis)
+  redis.zadd('vmpooler__provisioning__processing', score, request_id)
+end

spec/integration/api/healthcheck_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::Healthcheck do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method
+  before(:each) do
+    app.reset!
+  end
+
+  let(:config) {
+    {
+        config: {
+            'site_name' => 'test pooler',
+            'vm_lifetime_auth' => 2,
+        },
+        pools: [
+                    {'name' => 'pool1', 'size' => 5, 'alias' => ['poolone', 'poolun']},
+                    {'name' => 'pool2', 'size' => 10},
+                    {'name' => 'pool3', 'size' => 10, 'alias' => 'NotArray'}
+                ]
+    }
+  }
+
+  let(:current_time) { Time.now }
+
+  let(:metrics) {
+    double("metrics")
+  }
+
+  before(:each) do
+    expect(app).to receive(:run!).once
+    expect(metrics).to receive(:setup_prometheus_metrics)
+    expect(metrics).to receive(:prometheus_prefix)
+    expect(metrics).to receive(:prometheus_endpoint)
+    app.execute([:api], config, redis, metrics, nil)
+    app.settings.set :config, auth: false
+  end
+
+  describe '/healthcheck' do
+    it 'returns OK' do
+      get "/healthcheck"
+      expect(last_response.header['Content-Type']).to eq('application/json')
+      expect(last_response.status).to eq(200)
+      result = JSON.parse(last_response.body)
+      expect(result).to eq({'ok' => true})
+    end
+  end
+end

spec/integration/api/v3/config_spec.rb

@@ -1,13 +1,20 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   let(:config) {
     {
       config: {
@@ -16,7 +23,11 @@ describe Vmpooler::API::V1 do
         'experimental_features' => true
       },
       pools: [
-        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1'},
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      pools_at_startup: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
         {'name' => 'pool2', 'size' => 10}
       ],
       statsd: { 'prefix' => 'stats_prefix'},
@@ -26,21 +37,59 @@ describe Vmpooler::API::V1 do
   }
 
   describe '/config/pooltemplate' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:current_time) { Time.now }
 
     before(:each) do
-      redis.flushdb
-
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
+    describe 'DELETE /config/pooltemplate/:pool' do
+      it 'resets a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          template_before_reset: 'templates/new_template',
+          template_before_overrides: 'templates/pool1'
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when the pool has not been overridden' do
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails on nonexistent pools' do
+        delete "#{prefix}/config/pooltemplate/poolpoolpool"
+        expect_json(ok = false, http = 404)
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/pooltemplate/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+    end
+
     describe 'POST /config/pooltemplate' do
       it 'updates a pool template' do
         post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
@@ -138,6 +187,56 @@ describe Vmpooler::API::V1 do
 
     end
 
+    describe 'DELETE /config/poolsize' do
+      it 'resets a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          pool_size_before_reset: 2,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        delete "#{prefix}/config/poolsize/pool10"
+        expect_json(ok = false, http = 404)
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when a pool has not been overridden' do
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 200)
+        expected = {
+          ok: true,
+          pool_size_before_reset: 5,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/poolsize/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
     describe 'POST /config/poolsize' do
       it 'changes a pool size' do
         post "#{prefix}/config/poolsize", '{"pool1":"2"}'
@@ -162,7 +261,7 @@ describe Vmpooler::API::V1 do
         expect_json(ok = false, http = 400)
         expected = {
           ok: false,
-          bad_templates: ['pool10']
+          not_configured: ['pool10']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -192,7 +291,7 @@ describe Vmpooler::API::V1 do
 
         expected = {
           ok: false,
-          bad_templates: ['pool1']
+          not_configured: ['pool1']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -204,7 +303,7 @@ describe Vmpooler::API::V1 do
 
         expected = {
           ok: false,
-          bad_templates: ['pool1']
+          not_configured: ['pool1']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -225,8 +324,71 @@ describe Vmpooler::API::V1 do
       end
     end
 
+    describe 'POST /config/clonetarget' do
+      it 'changes the clone target' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'changes a pool size for multiple pools' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        post "#{prefix}/config/clonetarget", '{"pool10":"cluster1"}'
+        expect_json(ok = false, http = 400)
+        expected = {
+          ok: false,
+          bad_templates: ['pool10']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 200 when no change is required' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 201 when at least one pool changes' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
     describe 'GET /config' do
-      let(:prefix) { '/api/v1' }
+      let(:prefix) { '/api/v3' }
 
       it 'returns pool configuration when set' do
         get "#{prefix}/config"

spec/integration/api/v3/ondemandvm_spec.rb

@@ -0,0 +1,355 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V3 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/ondemandvm' do
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+          'max_ondemand_instances_per_request' => 50,
+          'backend_weight' => {
+            'compute1' => 5,
+            'compute2' => 0
+          }
+        },
+        dns_configs: {
+          :mock => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'},
+          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2', 'dns_plugin' => 'mock'},
+          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'}
+        ],
+        alias: {
+          'poolone' => ['pool1'],
+          'pool2' => ['pool1']
+        },
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ],
+        providers: {
+          :dummy => {},
+        }
+      }
+    }
+    let(:current_time) { Time.now }
+    let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
+    let(:uuid) { SecureRandom.uuid }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+      config[:pools].each do |pool|
+        redis.sadd('vmpooler__pools', pool['name'])
+      end
+    end
+
+    describe 'POST /ondemandvm' do
+
+      context 'with a configured pool' do
+
+        context 'with no request_id provided in payload' do
+          before(:each) do
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          end
+
+          it 'generates a request_id when none is provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'uses a configured platform to fulfill a ondemand request' do
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+            expect_json(true, 201)
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'creates a provisioning request in redis' do
+            expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1)
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          it 'sets a platform string in redis for the request to indicate selected platforms' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1')
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          context 'with a backend of 0 weight' do
+            before(:each) do
+              config[:config]['backend_weight']['compute1'] = 0
+            end
+
+            it 'sets the platform string in redis for the request to indicate the selected platforms' do
+              expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool1:pool1:1')
+              post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            end
+          end
+
+          it 'sets the platform string in redis for the request to indicate the selected platforms using weight' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool2:pool1:1')
+            post "#{prefix}/ondemandvm", '{"pool2":"1"}'
+          end
+        end
+
+        context 'with a resource request that exceeds the specified limit' do
+          let(:max_instances) { 50 }
+          before(:each) do
+            config[:config]['max_ondemand_instances_per_request'] = max_instances
+          end
+
+          it 'should reject the request with a message' do
+            post "#{prefix}/ondemandvm", '{"pool1":"51"}'
+            expect_json(false, 403)
+            expected = {
+              "ok": false,
+              "message": "requested amount of instances exceeds the maximum #{max_instances}"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with request_id provided in the payload' do
+          it 'uses the given request_id when provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": "1234"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'returns 409 conflict error when the request_id has been used' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(false, 409)
+
+            expected = {
+              "ok": false,
+              "request_id": "1234",
+              "message": "request_id '1234' has already been created"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with auth configured' do
+
+          it 'sets the token and user' do
+            app.settings.set :config, auth: true
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+            allow(redis).to receive(:hset)
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345')
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe')
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}', {
+              'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+            }
+          end
+        end
+      end
+
+      context 'with a pool that is not configured' do
+        let(:badpool) { 'pool4' }
+        it 'returns the bad template' do
+          post "#{prefix}/ondemandvm", '{"pool4":"1"}'
+          expect_json(false, 404)
+
+          expected = {
+            "ok": false,
+            "bad_templates": [ badpool ]
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'returns 400 and a message when JSON is invalid' do
+        post "#{prefix}/ondemandvm", '{"pool1":"1}'
+        expect_json(false, 400)
+        expected = {
+          "ok": false,
+          "message": "JSON payload could not be parsed"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+
+    describe 'GET /ondemandvm' do
+      it 'returns 404 with message when request is not found' do
+        get "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:score) { current_time }
+        let(:platforms_string) { 'pool1:pool1:1' }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 202 while the request is waiting' do
+          get "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 202)
+          expected = {
+            "ok": true,
+            "request_id": uuid,
+            "ready": false,
+            "pool1": {
+              "ready": "0",
+              "pending": "1"
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        context 'with ready instances' do
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis)
+            set_ondemand_request_status(uuid, 'ready', redis)
+          end
+
+          it 'returns 200 with hostnames when the request is ready' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": true,
+              "pool1": {
+                "hostname": [
+                  "#{vmname}.example.com"
+                ]
+              }
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with a deleted request' do
+          before(:each) do
+            set_ondemand_request_status(uuid, 'deleted', redis)
+          end
+
+          it 'returns a message that the request has been deleted' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request has been deleted"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with a failed request' do
+          let(:ondemand_request_ttl) { 5 }
+          before(:each) do
+            config[:config]['ondemand_request_ttl'] = ondemand_request_ttl
+            set_ondemand_request_status(uuid, 'failed', redis)
+          end
+
+          it 'returns a message that the request has failed' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request failed to provision instances within the configured ondemand_request_ttl '#{ondemand_request_ttl}'"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+      end
+    end
+
+    describe 'DELETE /ondemandvm' do
+      let(:expiration) { 129_600_0 }
+      it 'returns 404 with message when request is not found' do
+        delete "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:platforms_string) { 'pool1:pool1:1' }
+        let(:score) { current_time.to_i }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 200 for a deleted request' do
+          delete "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 200)
+          expected = { 'ok': true }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        it 'marks the request hash for expiration in two weeks' do
+          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{uuid}", expiration)
+          delete "#{prefix}/ondemandvm/#{uuid}"
+        end
+
+        context 'with running instances' do
+          let(:pool) { 'pool1' }
+          let(:pool_alias) { pool }
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, pool, pool_alias, redis)
+          end
+
+          it 'moves allocated instances to the completed queue' do
+            expect(redis).to receive(:smove).with("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vmname)
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+
+          it 'deletes the set tracking instances allocated for the request' do
+            expect(redis).to receive(:del).with("vmpooler__#{uuid}__#{pool_alias}__#{pool}")
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+        end
+      end
+    end
+  end
+end

spec/integration/api/v3/poolreset.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V3 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  after(:each) do
+    Vmpooler::API.reset!
+  end
+
+  let(:config) {
+    {
+      config: {
+        'site_name' => 'test pooler',
+        'vm_lifetime_auth' => 2,
+        'experimental_features' => true
+      },
+      pools: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      statsd: { 'prefix' => 'stats_prefix'},
+      alias: { 'poolone' => 'pool1' },
+      pool_names: [ 'pool1', 'pool2', 'poolone' ]
+    }
+  }
+
+  describe '/poolreset' do
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /poolreset' do
+      it 'refreshes ready and pending instances from a pool' do
+        post "#{prefix}/poolreset", '{"pool1":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/poolreset", '{"poolpoolpool":"1"}'
+        expect_json(ok = false, http = 400)
+      end
+
+      it 'resets multiple pools' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_pools: ['pool3']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/poolreset", '{"pool1":"1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'should return 400 for invalid json' do
+        post "#{prefix}/poolreset", '{"pool1":"1}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 400 with a bad pool name' do
+        post "#{prefix}/poolreset", '{"pool11":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 404 when there is no payload' do
+        post "#{prefix}/poolreset"
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+  end
+end

spec/integration/api/v3/status_spec.rb

@@ -5,15 +5,24 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
-  describe '/status' do
-    let(:prefix) { '/api/v1' }
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+    # Clear status cache to prevent test interference
+    Vmpooler::API::V3.clear_status_cache
+  end
+
+  describe 'status and metrics endpoints' do
+    let(:prefix) { '/api/v3' }
 
     let(:config) {
       {
@@ -32,10 +41,8 @@ describe Vmpooler::API::V1 do
     let(:current_time) { Time.now }
 
     before(:each) do
-      redis.flushdb
-
-      app.settings.set :config, config
-      app.settings.set :redis, redis
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
       app.settings.set :config, auth: false
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
@@ -52,7 +59,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of ready vms for each pool' do
-        3.times {|i| create_ready_vm("pool1", "vm-#{i}") }
+        3.times {|i| create_ready_vm("pool1", "vm-#{i}", redis) }
         get "#{prefix}/status/"
 
         # of course /status doesn't conform to the weird standard everything else uses...
@@ -63,8 +70,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of running vms for each pool' do
-        3.times {|i| create_running_vm("pool1", "vm-#{i}") }
-        4.times {|i| create_running_vm("pool2", "vm-#{i}") }
+        3.times {|i| create_running_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_running_vm("pool2", "vm-#{i}", redis) }
 
         get "#{prefix}/status/"
 
@@ -76,8 +83,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of pending vms for each pool' do
-        3.times {|i| create_pending_vm("pool1", "vm-#{i}") }
-        4.times {|i| create_pending_vm("pool2", "vm-#{i}") }
+        3.times {|i| create_pending_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_pending_vm("pool2", "vm-#{i}", redis) }
 
         get "#{prefix}/status/"
 
@@ -188,5 +195,55 @@ describe Vmpooler::API::V1 do
         expect(result["status"]).to_not be(nil)
       end
     end
+
+    describe 'GET /poolstat' do
+      it 'returns empty list when pool is not set' do
+        get "#{prefix}/poolstat"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result  == {})
+      end
+      it 'returns empty list when pool is not found' do
+        get "#{prefix}/poolstat?pool=unknownpool"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result  == {})
+      end
+      it 'returns one pool when requesting one with alias' do
+        get "#{prefix}/poolstat?pool=pool1"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 1)
+        expect(result["pools"]["pool1"]["ready"]).to eq(0)
+        expect(result["pools"]["pool1"]["max"]).to eq(5)
+        expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun'])
+      end
+      it 'returns one pool when requesting one without alias' do
+        get "#{prefix}/poolstat?pool=pool2"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 1)
+        expect(result["pools"]["pool2"]["ready"]).to eq(0)
+        expect(result["pools"]["pool2"]["max"]).to eq(10)
+        expect(result["pools"]["pool2"]["alias"]).to be(nil)
+      end
+      it 'returns multiple pools when requesting csv' do
+        get "#{prefix}/poolstat?pool=pool1,pool2"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 2)
+      end
+    end
+
+    describe 'GET /totalrunning' do
+      it 'returns the number of running VMs' do
+        get "#{prefix}/totalrunning"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        5.times {|i| create_running_vm("pool1", "vm-#{i}", redis, redis) }
+        5.times {|i| create_running_vm("pool3", "vm-#{i}", redis, redis) }
+        result = JSON.parse(last_response.body)
+        expect(result["running"] == 10)
+      end
+    end
   end
 end

spec/integration/api/v3/token_spec.rb

@@ -1,26 +1,38 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
-  describe '/token' do
-    let(:prefix) { '/api/v1' }
-    let(:current_time) { Time.now }
-    let(:config) { { } }
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
 
-    before do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
+  describe '/token' do
+    let(:prefix) { '/api/v3' }
+    let(:current_time) { Time.now }
+    let(:config) { { 
+      config: {}
+    } }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
     end
 
     describe 'GET /token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
 
         it 'returns a 404' do
           get "#{prefix}/token"
@@ -31,6 +43,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }
@@ -58,7 +71,10 @@ describe Vmpooler::API::V1 do
 
     describe 'POST /token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {}, 
+          auth: false 
+        } }
 
         it 'returns a 404' do
           post "#{prefix}/token"
@@ -69,6 +85,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }
@@ -94,10 +111,12 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/token/:token' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:current_time) { Time.now }
 
-    before do
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
       app.settings.set :config, config
       app.settings.set :redis, redis
     end
@@ -109,7 +128,10 @@ describe Vmpooler::API::V1 do
 
     describe 'GET /token/:token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
 
         it 'returns a 404' do
           get "#{prefix}/token/this"
@@ -119,6 +141,7 @@ describe Vmpooler::API::V1 do
 
       context '(auth configured)' do
         let(:config) { {
+          config: {},
           auth: true,
           pools: [
             {'name' => 'pool1', 'size' => 5}
@@ -141,7 +164,10 @@ describe Vmpooler::API::V1 do
 
     describe 'DELETE /token/:token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false
+        } }
 
         it 'returns a 404' do
           delete "#{prefix}/token/this"
@@ -152,6 +178,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }

spec/integration/api/v3/vm_hostname_spec.rb

@@ -5,44 +5,51 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm/:hostname' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
+
         },
         pools: [
           {'name' => 'pool1', 'size' => 5},
           {'name' => 'pool2', 'size' => 10}
         ],
         alias: { 'poolone' => 'pool1' },
+        auth: false
       }
     }
 
     let(:current_time) { Time.now }
 
     before(:each) do
-      redis.flushdb
-
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :config, auth: false
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'PUT /vm/:hostname' do
       it 'allows tags to be set' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested_by":"rspec"}}'
         expect_json(ok = true, http = 200)
 
@@ -50,7 +57,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'skips empty tags' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested_by":""}}'
         expect_json(ok = true, http = 200)
 
@@ -58,7 +65,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'does not set tags if request body format is invalid' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested"}}'
         expect_json(ok = false, http = 400)
 
@@ -70,7 +77,7 @@ describe Vmpooler::API::V1 do
           app.settings.set :config,
              { :config => { 'allowed_tags' => ['created_by', 'project', 'url'] } }
 
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"tags":{"created_by":"rspec","tested_by":"rspec"}}'
           expect_json(ok = false, http = 400)
@@ -80,12 +87,12 @@ describe Vmpooler::API::V1 do
       end
 
       context '(tagfilter configured)' do
-        let(:config) { {
-          tagfilter: { 'url' => '(.*)\/' },
-        } }
+        before(:each) do
+          app.settings.set :config, tagfilter: { 'url' => '(.*)\/' }
+        end
 
         it 'correctly filters tags' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com/something.html"}}'
           expect_json(ok = true, http = 200)
@@ -94,7 +101,7 @@ describe Vmpooler::API::V1 do
         end
 
         it "doesn't eat tags not matching filter" do
-          create_vm('testhost')
+          create_vm('testhost', redis)
           put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com"}}'
           expect_json(ok = true, http = 200)
 
@@ -103,10 +110,12 @@ describe Vmpooler::API::V1 do
       end
 
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        before(:each) do
+          app.settings.set :config, auth: false
+        end
 
         it 'allows VM lifetime to be modified without a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
           expect_json(ok = true, http = 200)
@@ -116,7 +125,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'does not allow a lifetime to be 0' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"0"}'
           expect_json(ok = false, http = 400)
@@ -124,6 +133,42 @@ describe Vmpooler::API::V1 do
           vm = fetch_vm('testhost')
           expect(vm['lifetime']).to be_nil
         end
+
+        it 'does not enforce a lifetime' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"20000"}'
+          expect_json(ok = true, http = 200)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to eq("20000")
+        end
+
+        it 'does not allow a lifetime to be initially past config max_lifetime_upper_limit' do
+          app.settings.set :config,
+                           { :config => { 'max_lifetime_upper_limit' => 168 } }
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"200"}'
+          expect_json(ok = false, http = 400)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to be_nil
+        end
+
+#       it 'does not allow a lifetime to be extended past config 168' do
+#         app.settings.set :config,
+#                          { :config => { 'max_lifetime_upper_limit' => 168 } }
+#         create_vm('testhost', redis)
+#
+#         set_vm_data('testhost', "checkout", (Time.now - (69*60*60)), redis)
+#         puts redis.hget("vmpooler__vm__testhost", 'checkout')
+#         put "#{prefix}/vm/testhost", '{"lifetime":"100"}'
+#         expect_json(ok = false, http = 400)
+#
+#         vm = fetch_vm('testhost')
+#         expect(vm['lifetime']).to be_nil
+#       end
       end
 
       context '(auth configured)' do
@@ -132,7 +177,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'allows VM lifetime to be modified with a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -144,7 +189,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'does not allows VM lifetime to be modified without a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
           expect_json(ok = false, http = 401)
@@ -160,7 +205,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'deletes an existing VM' do
-          create_running_vm('pool1', 'testhost')
+          create_running_vm('pool1', 'testhost', redis)
           expect fetch_vm('testhost')
 
           delete "#{prefix}/vm/testhost"
@@ -176,7 +221,7 @@ describe Vmpooler::API::V1 do
 
         context '(checked-out without token)' do
           it 'deletes a VM without supplying a token' do
-            create_running_vm('pool1', 'testhost')
+            create_running_vm('pool1', 'testhost', redis)
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost"
@@ -187,7 +232,7 @@ describe Vmpooler::API::V1 do
 
         context '(checked-out with token)' do
           it 'fails to delete a VM without supplying a token' do
-            create_running_vm('pool1', 'testhost', 'abcdefghijklmnopqrstuvwxyz012345')
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost"
@@ -196,7 +241,7 @@ describe Vmpooler::API::V1 do
           end
 
           it 'deletes a VM when token is supplied' do
-            create_running_vm('pool1', 'testhost', 'abcdefghijklmnopqrstuvwxyz012345')
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost", "", {
@@ -213,7 +258,7 @@ describe Vmpooler::API::V1 do
     describe 'POST /vm/:hostname/snapshot' do
       context '(auth not configured)' do
         it 'creates a snapshot' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
           post "#{prefix}/vm/testhost/snapshot"
           expect_json(ok = true, http = 202)
           expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
@@ -228,19 +273,19 @@ describe Vmpooler::API::V1 do
         it 'returns a 401 if not authed' do
           post "#{prefix}/vm/testhost/snapshot"
           expect_json(ok = false, http = 401)
-          expect !has_vm_snapshot?('testhost')
+          expect !has_vm_snapshot?('testhost', redis)
         end
 
         it 'creates a snapshot if authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = true, http = 202)
           expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
-          expect has_vm_snapshot?('testhost')
+          expect has_vm_snapshot?('testhost', redis)
         end
       end
     end
@@ -248,22 +293,22 @@ describe Vmpooler::API::V1 do
     describe 'POST /vm/:hostname/snapshot/:snapshot' do
       context '(auth not configured)' do
         it 'reverts to a snapshot' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot"
           expect_json(ok = true, http = 202)
-          expect vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'fails if the specified snapshot does not exist' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = false, http = 404)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
       end
 
@@ -273,33 +318,33 @@ describe Vmpooler::API::V1 do
         end
 
         it 'returns a 401 if not authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot"
           expect_json(ok = false, http = 401)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'fails if authed and the specified snapshot does not exist' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = false, http = 404)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'reverts to a snapshot if authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = true, http = 202)
-          expect vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
       end
     end

spec/integration/api/v3/vm_spec.rb

@@ -1,50 +1,77 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
-          'vm_lifetime_auth' => 2,
+          'vm_lifetime_auth' => 2
+        },
+        providers: {
+          vsphere: {},
+          gce: {},
+          foo: {}
+        },
+        dns_configs: {
+          :one => {
+            'dns_class' => 'mock',
+            'domain' => 'one.example.com'
+          },
+          :two => {
+            'dns_class' => 'mock',
+            'domain' => 'two.example.com'
+          },
+          :three => {
+            'dns_class' => 'mock',
+            'domain' => 'three.example.com'
+          }
         },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'one'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'gce', 'dns_plugin' => 'two'},
+          {'name' => 'pool3', 'size' => 10, 'provider' => 'foo', 'dns_plugin' => 'three'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
-        alias: { 'poolone' => 'pool1' },
-        pool_names: [ 'pool1', 'pool2', 'poolone' ]
+        alias: { 'poolone' => ['pool1'] },
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone', 'genericpool' ]
       }
     }
-
     let(:current_time) { Time.now }
+    let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
 
     before(:each) do
-      redis.flushdb
-
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'GET /vm/:hostname' do
       it 'returns correct information on a running vm' do
-        create_running_vm 'pool1', 'abcdefghijklmnop'
-        get "#{prefix}/vm/abcdefghijklmnop"
+        create_running_vm 'pool1', vmname, redis
+        expect(TCPSocket).to receive(:gethostbyname).and_raise(RuntimeError)
+        get "#{prefix}/vm/#{vmname}"
         expect_json(ok = true, http = 200)
-        response_body = (JSON.parse(last_response.body)["abcdefghijklmnop"])
+        response_body = (JSON.parse(last_response.body)[vmname])
 
         expect(response_body["template"]).to eq("pool1")
         expect(response_body["lifetime"]).to eq(0)
@@ -54,12 +81,16 @@ describe Vmpooler::API::V1 do
         expect(response_body["end_time"]).to eq(current_time.to_datetime.rfc3339)
         expect(response_body["state"]).to eq("running")
         expect(response_body["ip"]).to eq("")
+        expect(response_body["host"]).to eq("host1")
       end
     end
 
     describe 'POST /vm' do
+
+      let(:socket) { double('socket') }
       it 'returns a single VM' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', vmname, redis
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"1"}'
         expect_json(ok = true, http = 200)
@@ -67,7 +98,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -75,15 +106,17 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns a single VM for an alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', vmname, redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"poolone":"1"}'
         expect_json(ok = true, http = 200)
 
         expected = {
           ok: true,
-          pool1: {
-            hostname: 'abcdefghijklmnop'
+          poolone: {
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -96,10 +129,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool when aliases are not defined' do
-        Vmpooler::API.settings.config.delete(:alias)
-        Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2']
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
 
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', vmname, redis
         post "#{prefix}/vm/pool1"
         post "#{prefix}/vm/pool1"
 
@@ -110,8 +143,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        post "#{prefix}/vm/poolone"
+        create_ready_vm 'pool2', vmname, redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -121,8 +153,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', vmname, redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
         expect_json(ok = true, http = 200)
@@ -130,10 +164,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: "#{vmname}.one.example.com"
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.two.example.com'
           }
         }
 
@@ -141,9 +175,11 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs even when multiple instances from the same pool are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
 
@@ -159,18 +195,20 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
 
       it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
-        create_ready_vm 'pool2', '1qrstuvwxyz012345'
-        create_ready_vm 'pool2', '2qrstuvwxyz012345'
-        create_ready_vm 'pool2', '3qrstuvwxyz012345'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
 
@@ -186,14 +224,59 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.two.example.com', '2qrstuvwxyz012345.two.example.com', '3qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
 
+      it 'returns VMs from multiple backend pools requested by an alias' do
+        Vmpooler::API.settings.config[:alias]['genericpool'] = ['pool1', 'pool2', 'pool3']
+
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool3', '1qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"genericpool":"3"}'
+
+        expected = {
+          ok: true,
+          genericpool: {
+            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop', '1qrstuvwxyz012345' ]
+          }
+        }
+
+        result = JSON.parse(last_response.body)
+        expect(result['ok']).to eq(true)
+        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.two.example.com', '1qrstuvwxyz012345.three.example.com')
+
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'returns the first VM that was moved to the ready state when checking out a VM' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '3abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1"}'
+
+        expected = {
+          ok: true,
+          "pool1": {
+            "hostname": "1abcdefghijklmnop.one.example.com"
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = true, http = 200)
+      end
+
       it 'fails when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
 
@@ -204,7 +287,9 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
 
@@ -213,11 +298,11 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
 
@@ -228,7 +313,9 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
 
@@ -237,11 +324,11 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
 
@@ -252,8 +339,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
 
@@ -262,15 +351,39 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop', redis)).to eq(true)
+      end
+
+      it 'returns the second VM when the first fails to respond' do
+        create_running_vm 'pool1', vmname, redis
+        create_ready_vm 'pool1', "2#{vmname}", redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with(vmname, nil).and_raise('mockerror')
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with("2#{vmname}", nil).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: "2#{vmname}.one.example.com"
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+        expect(pool_has_ready_vm?('pool1', vmname, redis)).to be false
       end
 
       context '(auth not configured)' do
         it 'does not extend VM lifetime if auth token is provided' do
           app.settings.set :config, auth: false
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm", '{"pool1":"1"}', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -280,7 +393,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -294,7 +407,9 @@ describe Vmpooler::API::V1 do
         it 'extends VM lifetime if auth token is provided' do
           app.settings.set :config, auth: true
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm", '{"pool1":"1"}', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -304,7 +419,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -315,7 +430,9 @@ describe Vmpooler::API::V1 do
 
         it 'does not extend VM lifetime if auth token is not provided' do
           app.settings.set :config, auth: true
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm", '{"pool1":"1"}'
           expect_json(ok = true, http = 200)
@@ -323,7 +440,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))

spec/integration/api/v3/vm_template_spec.rb

@@ -1,25 +1,40 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm/:template' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
         },
+        dns_configs: {
+          :example => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
+        providers: { vsphere: {} },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'poolone', 'size' => 1, 'provider' => 'vsphere', 'dns_plugin' => 'example'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
         alias: { 'poolone' => 'pool1' },
@@ -28,20 +43,22 @@ describe Vmpooler::API::V1 do
     }
 
     let(:current_time) { Time.now }
+    let(:socket) { double('socket') }
+    let(:checkoutlock) { Mutex.new }
 
     before(:each) do
-      redis.flushdb
-
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'POST /vm/:template' do
       it 'returns a single VM' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1", ''
         expect_json(ok = true, http = 200)
@@ -49,7 +66,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
 
@@ -57,14 +74,16 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns a single VM for an alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/poolone", ''
 
         expected = {
           ok: true,
-          pool1: {
-            hostname: 'abcdefghijklmnop'
+          poolone: {
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
         expect_json(ok = true, http = 200)
@@ -78,10 +97,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool when aliases are not defined' do
-        Vmpooler::API.settings.config.delete(:alias)
-        Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2']
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
 
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
         post "#{prefix}/vm/pool1"
         post "#{prefix}/vm/pool1"
 
@@ -92,8 +111,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        post "#{prefix}/vm/poolone"
+        create_ready_vm 'pool2', 'abcdefghijklmnop', redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -103,8 +121,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1+pool2", ''
         expect_json(ok = true, http = 200)
@@ -112,10 +132,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.example.com'
           }
         }
 
@@ -123,34 +143,36 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
 
-        create_ready_vm 'pool2', '1qrstuvwxyz012345'
-        create_ready_vm 'pool2', '2qrstuvwxyz012345'
-        create_ready_vm 'pool2', '3qrstuvwxyz012345'
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
 
         expected = {
           ok: true,
           pool1: {
-            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+            hostname: [ '1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com' ]
           },
           pool2: {
-            hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ]
+            hostname: [ '1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com' ]
           }
         }
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com')
         expect_json(ok = true, http = 200)
       end
 
       it 'fails when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
         post "#{prefix}/vm/pool1+pool2", ''
 
@@ -161,7 +183,9 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1+pool2", ''
 
@@ -170,12 +194,12 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool1', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
 
         post "#{prefix}/vm/pool1+pool1+pool2", ''
 
@@ -186,8 +210,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool1', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1+pool1+pool2", ''
 
@@ -196,13 +222,13 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool1', '0123456789012345')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '0123456789012345', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
 
         post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
 
@@ -213,8 +239,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
 
@@ -223,15 +251,17 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool2', '0123456789012345')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool2', '0123456789012345', redis)).to eq(true)
       end
 
       context '(auth not configured)' do
         it 'does not extend VM lifetime if auth token is provided' do
           app.settings.set :config, auth: false
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm/pool1", '', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -241,7 +271,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
 
@@ -255,7 +285,9 @@ describe Vmpooler::API::V1 do
         it 'extends VM lifetime if auth token is provided' do
           app.settings.set :config, auth: true
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm/pool1", '', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -265,7 +297,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -276,14 +308,16 @@ describe Vmpooler::API::V1 do
 
         it 'does not extend VM lifetime if auth token is not provided' do
           app.settings.set :config, auth: true
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
           post "#{prefix}/vm/pool1", ''
 
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect_json(ok = true, http = 200)

spec/integration/dashboard_spec.rb

@@ -5,17 +5,36 @@ describe Vmpooler::API do
   include Rack::Test::Methods
 
   def app()
-    described_class
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
   end
 
   describe 'Dashboard' do
 
+    let(:config) { {
+      config: {},
+      pools: [
+          {'name' => 'pool1', 'size' => 5}
+      ],
+      graphite: {}
+    } }
+
     before(:each) do
-      redis.flushdb
+      expect(app).to receive(:run!)
+      app.execute([:api], config, redis, nil, nil)
+      app.settings.set :config, auth: false
     end
 
     context '/' do
-      before { get '/' }
+      before(:each) do
+        get '/'
+      end
 
       it { expect(last_response.status).to eq(302) }
       it { expect(last_response.location).to eq('http://example.org/dashboard/') }
@@ -25,7 +44,7 @@ describe Vmpooler::API do
       ENV['SITE_NAME'] = 'test pooler'
       ENV['VMPOOLER_CONFIG'] = 'thing'
 
-      before do
+      before(:each) do
         get '/dashboard/'
       end
 
@@ -35,15 +54,18 @@ describe Vmpooler::API do
     end
 
     context 'unknown route' do
-      before { get '/doesnotexist' }
+      before(:each) do
+        get '/doesnotexist'
+      end
 
-      it { expect(last_response.status).to eq(404) }
       it { expect(last_response.header['Content-Type']).to eq('application/json') }
+      it { expect(last_response.status).to eq(404) }
       it { expect(last_response.body).to eq(JSON.pretty_generate({ok: false})) }
     end
 
     describe '/dashboard/stats/vmpooler/pool' do
       let(:config) { {
+          config: {},
           pools: [
               {'name' => 'pool1', 'size' => 5},
               {'name' => 'pool2', 'size' => 1}
@@ -51,20 +73,18 @@ describe Vmpooler::API do
           graphite: {}
       } }
 
-      before do
-        $config = config
+      before(:each) do
         app.settings.set :config, config
-        app.settings.set :redis, redis
         app.settings.set :environment, :test
       end
 
       context 'without history param' do
         it 'returns basic JSON' do
-          create_ready_vm('pool1', 'vm1')
-          create_ready_vm('pool1', 'vm2')
-          create_ready_vm('pool1', 'vm3')
-          create_ready_vm('pool2', 'vm4')
-          create_ready_vm('pool2', 'vm5')
+          create_ready_vm('pool1', 'vm1', redis)
+          create_ready_vm('pool1', 'vm2', redis)
+          create_ready_vm('pool1', 'vm3', redis)
+          create_ready_vm('pool2', 'vm4', redis)
+          create_ready_vm('pool2', 'vm5', redis)
 
           get '/dashboard/stats/vmpooler/pool'
 
@@ -94,11 +114,11 @@ describe Vmpooler::API do
         end
 
         it 'returns JSON with history when redis has values' do
-          create_ready_vm('pool1', 'vm1')
-          create_ready_vm('pool1', 'vm2')
-          create_ready_vm('pool1', 'vm3')
-          create_ready_vm('pool2', 'vm4')
-          create_ready_vm('pool2', 'vm5')
+          create_ready_vm('pool1', 'vm1', redis)
+          create_ready_vm('pool1', 'vm2', redis)
+          create_ready_vm('pool1', 'vm3', redis)
+          create_ready_vm('pool2', 'vm4', redis)
+          create_ready_vm('pool2', 'vm5', redis)
 
           get '/dashboard/stats/vmpooler/pool', :history => true
 
@@ -116,6 +136,7 @@ describe Vmpooler::API do
 
     describe '/dashboard/stats/vmpooler/running' do
       let(:config) { {
+          config: {},
           pools: [
               {'name' => 'pool-1', 'size' => 5},
               {'name' => 'pool-2', 'size' => 1},
@@ -124,10 +145,8 @@ describe Vmpooler::API do
           graphite: {}
       } }
 
-      before do
-        $config = config
+      before(:each) do
         app.settings.set :config, config
-        app.settings.set :redis, redis
         app.settings.set :environment, :test
       end
 
@@ -144,18 +163,18 @@ describe Vmpooler::API do
         end
 
         it 'adds major correctly' do
-          create_running_vm('pool-1', 'vm1')
-          create_running_vm('pool-1', 'vm2')
-          create_running_vm('pool-1', 'vm3')
+          create_running_vm('pool-1', 'vm1', redis)
+          create_running_vm('pool-1', 'vm2', redis)
+          create_running_vm('pool-1', 'vm3', redis)
 
-          create_running_vm('pool-2', 'vm4')
-          create_running_vm('pool-2', 'vm5')
-          create_running_vm('pool-2', 'vm6')
-          create_running_vm('pool-2', 'vm7')
-          create_running_vm('pool-2', 'vm8')
+          create_running_vm('pool-2', 'vm4', redis)
+          create_running_vm('pool-2', 'vm5', redis)
+          create_running_vm('pool-2', 'vm6', redis)
+          create_running_vm('pool-2', 'vm7', redis)
+          create_running_vm('pool-2', 'vm8', redis)
 
-          create_running_vm('diffpool-1', 'vm9')
-          create_running_vm('diffpool-1', 'vm10')
+          create_running_vm('diffpool-1', 'vm9', redis)
+          create_running_vm('diffpool-1', 'vm10', redis)
 
           get '/dashboard/stats/vmpooler/running'
 

spec/rbvmomi_helper.rb

@@ -1,781 +0,0 @@
-# -----------------------------------------------------------------------------------------------------------------
-# Managed Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-mo_types.html)
-# -----------------------------------------------------------------------------------------------------------------
-
-MockClusterComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-  # From MockClusterComputeResource
-  :actionHistory, :configuration, :drsFault, :drsRecommendation, :migrationHistory, :recommendation,
-  # From ComputeResource
-  :resourcePool,
-  # From ManagedEntity
-  :name
-)
-
-MockComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-  # From ComputeResource
-  :configurationEx, :datastore, :host, :network, :resourcePool, :summary,
-  # From ManagedEntity
-  :name
-)
-
-MockContainerView = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-  # From ContainerView
-  :container, :recursive, :type
-)
-
-MockDatacenter = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datacenter.html
-  # From Datacenter
-  :datastore, :datastoreFolder, :hostFolder, :network, :networkFolder, :vmFolder,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Datacenter https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Datacenter.rb
-
-  # Find the Datastore with the given +name+.
-  def find_datastore name
-    datastore.find { |x| x.name == name }
-  end
-end
-
-MockDatastore = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-  # From Datastore
-  :browser, :capability, :host, :info, :iormConfiguration, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockFolder = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Folder.html
-  # From Folder
-  :childEntity, :childType,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Folder https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L107-L110
-  def children
-    childEntity
-  end
-
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L9-L12
-  def find(name, type=Object)
-    # Fake the searchIndex
-    childEntity.each do |child|
-      if child.name == name
-        if child.kind_of?(type)
-          return child
-        else
-          return nil
-        end
-      end
-    end
-
-    nil
-  end
-end
-
-MockHostSystem = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-  # From HostSystem
-  :capability, :config, :configManager, :datastore, :datastoreBrowser, :hardware, :network, :runtime, :summary, :systemResources, :vm,
-  # From ManagedEntity
-  :overallStatus, :name, :parent,
-  # From ManagedObject
-  :configIssue
-)
-
-MockPropertyCollector = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-  # PropertyCollector
-  :filter
-)
-
-MockResourcePool = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-  # From ResourcePool
-  :childConfiguration, :config, :owner, :resourcePool, :runtime, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockSearchIndex = Object
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.SearchIndex.html
-
-MockServiceInstance = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-  # From ServiceInstance
-  :capability, :content, :serverClock
-) do
-  # From ServiceInstance 
-  # Mock the CurrentTime method so that it appears the ServiceInstance is valid.
-  def CurrentTime
-    Time.now
-  end
-
-  # From RBVMOMI::VIM::ServiceInstance https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ServiceInstance.rb
-  def find_datacenter(path=nil)
-    # In our mocked instance, DataCenters are always in the root Folder.
-    # If path is nil the first DC is returned otherwise match by name
-    content.rootFolder.childEntity.each do |child|
-      if child.is_a?(RbVmomi::VIM::Datacenter)
-        return child if path.nil? || child.name == path
-      end
-    end
-    nil
-  end
-end
-
-MockTask = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-  # From Task
-  :info,
-) do
-  # From RBVMOMI https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Task.rb
-  # Mock the with 'Not Implemented'
-  def wait_for_completion
-    raise(RuntimeError,'Not Implemented')
-  end
-end
-
-MockViewManager = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ViewManager.html
-  # From ViewManager
-  :viewList,
-) do
-  # From ViewManager 
-  def CreateContainerView(options)
-    mock_RbVmomi_VIM_ContainerView({
-      :container => options[:container],
-      :recursive => options[:recursive],
-      :type => options[:type],
-    })
-  end
-end
-
-MockVirtualDiskManager = Object
-# https://pubs.vmware.com/vsphere-55/index.jsp#com.vmware.wssdk.apiref.doc/vim.VirtualDiskManager.html
-
-MockVirtualMachine = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-  # From VirtualMachine
-  :config, :runtime, :snapshot, :summary,
-  # From ManagedEntity
-  :name,
-  # From RbVmomi::VIM::ManagedEntity
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ManagedEntity.rb
-  :path
-)
-
-MockVirtualMachineSnapshot = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-  # From VirtualMachineSnapshot
-  :config
-)
-
-# -------------------------------------------------------------------------------------------------------------
-# Data Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-do_types.html)
-# -------------------------------------------------------------------------------------------------------------
-
-MockDescription = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Description.html
-  # From Description
-  :label, :summary
-)
-
-MockDynamicProperty = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.DynamicProperty.html
-  # From DynamicProperty
-  :name, :val
-)
-
-MockHostCpuPackage = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.CpuPackage.html
-  # From HostCpuPackage
-  :busHz, :cpuFeature, :description, :hz, :index, :threadId, :vendor
-)
-
-MockHostHardwareSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.HardwareSummary.html
-  # From HostHardwareSummary
-  :cpuMhz, :numCpuCores, :numCpuPkgs, :memorySize
-)
-
-MockHostListSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.html
-  # From HostListSummary
-  :quickStats, :hardware
-)
-
-MockHostListSummaryQuickStats = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.QuickStats.html
-  # From HostListSummaryQuickStats
-  :overallCpuUsage, :overallMemoryUsage
-)
-
-MockHostRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.RuntimeInfo.html
-  # From HostRuntimeInfo
-  :bootTime, :connectionState, :healthSystemRuntime, :inMaintenanceMode, :powerState, :tpmPcrValues
-)
-
-MockHostSystemHostHardwareInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.HardwareInfo.html
-  # From HostHardwareInfo
-  :biosInfo, :cpuFeature, :cpuInfo, :cpuPkg, :cpuPowerManagementInfo, :memorySize, :numaInfo, :pciDevice, :systemInfo
-)
-
-MockObjectContent = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.ObjectContent.html
-  # From ObjectContent
-  :missingSet, :obj, :propSet
-)
-
-MockRetrieveResult = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-  # From RetrieveResult
-  :objects, :token
-)
-
-MockServiceContent = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-  # From ServiceContent
-  :propertyCollector, :rootFolder, :searchIndex, :viewManager, :virtualDiskManager
-)
-
-MockVirtualDevice = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualDisk = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-  # From VirtualDisk
-  :capacityInKB, :shares,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualHardware = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.VirtualHardware.html
-  # From VirtualHardware
-  :device
-)
-
-MockVirtualMachineConfigInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.ConfigInfo.html
-  # From VirtualMachineConfigInfo
-  :hardware
-)
-
-MockVirtualMachineFileLayoutExFileInfo = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-  # From VirtualMachineFileLayoutExFileInfo
-  :key, :name, :size, :type, :uniqueSize
-)
-
-MockVirtualMachineGuestSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.GuestSummary.html
-  # From VirtualMachineGuestSummary
-  :hostName
-)
-
-MockVirtualMachineRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.RuntimeInfo.html
-  # From VirtualMachineRuntimeInfo
-  :bootTime, :cleanPowerOff, :connectionState, :faultToleranceState, :host, :maxCpuUsage, :maxMemoryUsage, :memoryOverhead,
-  :needSecondaryReason, :numMksConnections, :powerState, :question, :recordReplayState, :suspendInterval, :suspendTime, :toolsInstallerMounted
-)
-
-MockVirtualMachineSnapshotInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotInfo.html
-  # From MockVirtualMachineSnapshotInfo
-  :currentSnapshot, :rootSnapshotList
-)
-
-MockVirtualMachineSnapshotTree = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotTree.html
-  # From VirtualMachineSnapshotTree
-  :backupManifest, :childSnapshotList, :createTime, :description, :id, :name, :quiesced, :replaySupported,
-  :snapshot, :state, :vm
-)
-
-MockVirtualMachineSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.html
-  # From VirtualMachineSummary
-  :config, :customValue, :guest, :quickStats, :runtime, :storage, :vm
-)
-
-MockVirtualSCSIController = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-  # From VirtualSCSIController
-  :hotAddRemove, :scsiCtlrUnitNumber, :sharedBus,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-# --------------------
-# RBVMOMI only Objects
-# --------------------
-MockRbVmomiVIMConnection = Struct.new(
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  :serviceInstance, :serviceContent, :rootFolder, :root
-) do
-  # From https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  # Alias to serviceContent.searchIndex
-  def searchIndex
-    serviceContent.searchIndex
-  end
-  # Alias to serviceContent.propertyCollector
-  def propertyCollector
-    serviceContent.propertyCollector
-  end
-end
-
-# -------------------------------------------------------------------------------------------------------------
-# Mocking Methods
-# -------------------------------------------------------------------------------------------------------------
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-def mock_RbVmomi_VIM_ClusterComputeResource(options = {})
-  options[:name]  = 'Cluster' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockClusterComputeResource.new()
-
-  mock.name = options[:name]
-  # All cluster compute resources have a root Resource Pool
-  mock.resourcePool = mock_RbVmomi_VIM_ResourcePool({:name => options[:name]})
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ClusterComputeResource
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-def mock_RbVmomi_VIM_ContainerView(options = {})
-  mock = MockContainerView.new()
-  mock.container = options[:container]
-  mock.recursive = options[:recursive]
-  mock.type = options[:type]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-def mock_RbVmomi_VIM_ComputeResource(options = {})
-  options[:name]  = 'Compute' + rand(65536).to_s if options[:name].nil?
-  options[:hosts] = [{}] if options[:hosts].nil?
-
-  mock = MockComputeResource.new()
-
-  mock.name = options[:name]
-  mock.host = []
-
-  # A compute resource must have at least one host.
-  options[:hosts].each do |host_options|
-    mock_host = mock_RbVmomi_VIM_HostSystem(host_options)
-    mock_host.parent = mock
-    mock.host << mock_host
-  end
-
-  mock
-end
-
-# https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-def mock_RbVmomi_VIM_Connection(options = {})
-  options[:serviceInstance] = {} if options[:serviceInstance].nil?
-  options[:serviceContent]  = {} if options[:serviceContent].nil?
-
-  mock = MockRbVmomiVIMConnection.new()
-  mock.serviceContent = mock_RbVmomi_VIM_ServiceContent(options[:serviceContent])
-  options[:serviceInstance][:servicecontent] = mock.serviceContent if options[:serviceInstance][:servicecontent].nil?
-  mock.serviceInstance = mock_RbVmomi_VIM_ServiceInstance(options[:serviceInstance])
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datacenter(options = {})
-  options[:hostfolder_tree] = {} if options[:hostfolder_tree].nil?
-  options[:vmfolder_tree]   = {} if options[:vmfolder_tree].nil?
-  # Currently don't support mocking datastore tree
-  options[:datastores]      = [] if options[:datastores].nil?
-  options[:name]            = 'Datacenter' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockDatacenter.new()
-
-  mock.name = options[:name]
-  mock.hostFolder = mock_RbVmomi_VIM_Folder({ :name => 'hostFolderRoot'})
-  mock.vmFolder = mock_RbVmomi_VIM_Folder({ :name => 'vmFolderRoot'})
-  mock.datastore = []
-
-  # Create vmFolder hierarchy
-  recurse_folder_tree(options[:vmfolder_tree],mock.vmFolder.childEntity)
-
-  # Create hostFolder hierarchy
-  recurse_folder_tree(options[:hostfolder_tree],mock.hostFolder.childEntity)
-
-  # Create mock Datastores
-  options[:datastores].each do |datastorename|
-    mock_ds = mock_RbVmomi_VIM_Datastore({ :name => datastorename })
-    mock.datastore << mock_ds
-  end
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datacenter
-  end
-
-  mock
-end
-
-def recurse_folder_tree(tree, root_object)
-  tree.keys.each do |foldername|
-    folder_options = tree[foldername].nil? ? {} : tree[foldername]
-    folder_options[:name] = foldername if folder_options[:name].nil?
-
-    case folder_options[:object_type]
-    when 'vm'
-      child_object = mock_RbVmomi_VIM_VirtualMachine({ :name => folder_options[:name]})
-    when 'compute_resource'
-      child_object = mock_RbVmomi_VIM_ComputeResource({ :name => folder_options[:name]})
-    when 'cluster_compute_resource'
-      child_object = mock_RbVmomi_VIM_ClusterComputeResource({ :name => folder_options[:name]})
-    when 'resource_pool'
-      child_object =  mock_RbVmomi_VIM_ResourcePool({ :name => folder_options[:name]})
-    else
-      child_object = mock_RbVmomi_VIM_Folder({:name => foldername})
-    end
-
-    # Recursively create children - Default is the child_object is a Folder
-    case folder_options[:object_type]
-    when 'cluster_compute_resource'
-      # Append children into the root Resource Pool for a cluster, instead of directly into the cluster itself.
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool.resourcePool) unless folder_options[:children].nil?
-    when 'resource_pool'
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool) unless folder_options[:children].nil?
-    else
-      recurse_folder_tree(folder_options[:children],child_object.childEntity) unless folder_options[:children].nil?
-    end
-
-    root_object << child_object
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datastore(options = {})
-  options[:name] = 'Datastore' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockDatastore.new()
-
-  mock.name = options[:name]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datastore
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Folder(options = {})
-  options[:name] = 'Folder' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockFolder.new()
-
-  mock.name = options[:name]
-  mock.childEntity = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Folder
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-def mock_RbVmomi_VIM_HostSystem(options = {})
-  options[:memory_size]          = 4294967296 if options[:memory_size].nil? # 4GB RAM
-  options[:num_cpu]              = 1 if options[:num_cpu].nil?
-  options[:num_cores_per_cpu]    = 1 if options[:num_cores_per_cpu].nil?
-  options[:cpu_speed]            = 2048 if options[:cpu_speed].nil? # 2.0 GHz 
-  options[:cpu_model]            = 'Intel(R) Xeon(R) CPU E5-2697 v4 @ 2.0GHz' if options[:cpu_model].nil?
-  options[:maintenance_mode]     = false if options[:maintenance_mode].nil?
-  options[:overall_status]       = 'green' if options[:overall_status].nil?
-  options[:overall_cpu_usage]    = 1 if options[:overall_cpu_usage].nil?
-  options[:overall_memory_usage] = 1 if options[:overall_memory_usage].nil?
-  options[:name]                 = 'HOST' + rand(65536).to_s if options[:name].nil?
-  options[:config_issue]         = [] if options[:config_issue].nil?
-
-  mock = MockHostSystem.new()
-  mock.name = options[:name]
-  mock.summary = MockHostListSummary.new()
-  mock.summary.quickStats = MockHostListSummaryQuickStats.new()
-  mock.summary.hardware = MockHostHardwareSummary.new()
-  mock.hardware = MockHostSystemHostHardwareInfo.new()
-  mock.runtime = MockHostRuntimeInfo.new()
-
-  mock.hardware.cpuPkg = []
-  (1..options[:num_cpu]).each do |cpuid|
-    mockcpu = MockHostCpuPackage.new()
-    mockcpu.hz = options[:cpu_speed] * 1024 * 1024
-    mockcpu.description = options[:cpu_model]
-    mockcpu.index = 0
-    mock.hardware.cpuPkg << mockcpu
-  end
-
-  mock.runtime.inMaintenanceMode = options[:maintenance_mode]
-  mock.overallStatus = options[:overall_status]
-  mock.configIssue = options[:config_issue]
-
-  mock.summary.hardware.memorySize = options[:memory_size]
-  mock.hardware.memorySize = options[:memory_size]
-
-  mock.summary.hardware.cpuMhz = options[:cpu_speed]
-  mock.summary.hardware.numCpuCores = options[:num_cpu] * options[:num_cores_per_cpu]
-  mock.summary.hardware.numCpuPkgs = options[:num_cpu]
-  mock.summary.quickStats.overallCpuUsage = options[:overall_cpu_usage]
-  mock.summary.quickStats.overallMemoryUsage = options[:overall_memory_usage]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-def mock_RbVmomi_VIM_RetrieveResult(options = {})
-  options[:response] = [] if options[:response].nil?
-  mock = MockRetrieveResult.new()
-
-  mock.objects = []
-
-  options[:response].each do |response|
-    mock_objectdata = MockObjectContent.new()
-
-    mock_objectdata.propSet = []
-
-    mock_objectdata.obj = response[:object] 
-
-    # Mock the object properties
-    response.each do |key,value|
-      unless key == :object
-        mock_property = MockDynamicProperty.new()
-        mock_property.name = key
-        mock_property.val = value
-        mock_objectdata.propSet << mock_property
-      end
-    end
-    mock.objects << mock_objectdata
-  end
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-def mock_RbVmomi_VIM_PropertyCollector(options = {})
-  mock = MockPropertyCollector.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-def mock_RbVmomi_VIM_ServiceContent(options = {})
-  options[:propertyCollector] = {} if options[:propertyCollector].nil?
-  options[:datacenters]       = [] if options[:datacenters].nil?
-
-  mock = MockServiceContent.new()
-
-  mock.searchIndex = MockSearchIndex.new()
-  mock.viewManager = MockViewManager.new()
-  mock.virtualDiskManager = MockVirtualDiskManager.new()
-  mock.rootFolder = mock_RbVmomi_VIM_Folder({ :name => 'RootFolder' })
-
-  mock.propertyCollector = mock_RbVmomi_VIM_PropertyCollector(options[:propertyCollector])
-
-  # Create the DCs in this ServiceContent
-  options[:datacenters].each do |dc_options|
-    mock_dc = mock_RbVmomi_VIM_Datacenter(dc_options)
-    mock.rootFolder.childEntity << mock_dc
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-def mock_RbVmomi_VIM_ServiceInstance(options = {})
-  mock = MockServiceInstance.new()
-
-  mock.content = options[:servicecontent]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-def mock_RbVmomi_VIM_Task(options = {})
-  mock = MockTask.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-def mock_RbVmomi_VIM_ResourcePool(options = {})
-  options[:name] = 'ResourcePool' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockResourcePool.new()
-
-  mock.name = options[:name]
-  mock.resourcePool = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ResourcePool
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-def mock_RbVmomi_VIM_VirtualDisk(options = {})
-  options[:controllerKey] = rand(65536) if options[:controllerKey].nil?
-  options[:key]           = rand(65536) if options[:key].nil?
-  options[:label]         = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:unitNumber]    = rand(65536) if options[:unitNumber].nil?
-
-  mock = MockVirtualDisk.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.unitNumber = options[:unitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualDisk
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-def mock_RbVmomi_VIM_VirtualMachine(options = {})
-  options[:snapshot_tree] = nil if options[:snapshot_tree].nil?
-  options[:name] = 'VM' + rand(65536).to_s if options[:name].nil?
-  options[:path] = [] if options[:path].nil?
-
-  mock = MockVirtualMachine.new()
-  mock.config = MockVirtualMachineConfigInfo.new()
-  mock.config.hardware = MockVirtualHardware.new([])
-  mock.summary = MockVirtualMachineSummary.new()
-  mock.summary.runtime = MockVirtualMachineRuntimeInfo.new()
-  mock.summary.guest = MockVirtualMachineGuestSummary.new()
-  mock.runtime = mock.summary.runtime
-
-  mock.name = options[:name]
-  mock.summary.guest.hostName = options[:hostname]
-  mock.runtime.bootTime = options[:boottime]
-  mock.runtime.powerState = options[:powerstate]
-
-  unless options[:snapshot_tree].nil?
-    mock.snapshot = MockVirtualMachineSnapshotInfo.new()
-    mock.snapshot.rootSnapshotList = []
-    index = 0
-
-    # Create a recursive snapshot tree
-    recurse_snapshot_tree(options[:snapshot_tree],mock.snapshot.rootSnapshotList,index)
-  end
-
-  # Create an array of items that describe the path of the VM from the root folder
-  # all the way to the VM itself
-  mock.path = []
-  options[:path].each do |path_item|
-    mock_item = nil
-    case path_item[:type]
-    when 'folder'
-      mock_item = mock_RbVmomi_VIM_Folder({ :name => path_item[:name] })
-    when 'datacenter'
-      mock_item = mock_RbVmomi_VIM_Datacenter({ :name => path_item[:name] })
-    else
-      raise("Unknown mock type #{path_item[:type]} for mock_RbVmomi_VIM_VirtualMachine")
-    end
-    mock.path << [mock_item,path_item[:name]]
-  end
-  mock.path << [mock,options[:name]]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualMachine
-  end
-
-  mock
-end
-
-def recurse_snapshot_tree(tree, root_object, index)
-  tree.keys.each do |snapshotname|
-    snap_options = tree[snapshotname].nil? ? {} : tree[snapshotname]
-    snap = MockVirtualMachineSnapshotTree.new()
-    snap.id = index
-    snap.name = snapshotname
-    snap.childSnapshotList = []
-    snap.description = "Snapshot #{snapshotname}"
-    snap.snapshot = snap_options[:ref] unless snap_options[:ref].nil?
-
-    # Recursively create chilren
-    recurse_snapshot_tree(snap_options[:children],snap.childSnapshotList,index) unless snap_options[:children].nil?
-
-    root_object << snap
-    index += 1
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-def mock_RbVmomi_VIM_VirtualMachineDevice(options = {})
-  mock = MockVirtualDevice.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-def mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo(options = {})
-  options[:key] = rand(65536).to_s if options[:key].nil?
-
-  mock = MockVirtualMachineFileLayoutExFileInfo.new()
-
-  mock.key = options[:key]
-  mock.name = options[:name]
-  mock.size = options[:size]
-  mock.type = options[:type]
-  mock.uniqueSize = options[:uniqueSize]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-def mock_RbVmomi_VIM_VirtualMachineSnapshot(options = {})
-  mock = MockVirtualMachineSnapshot.new()
-  
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-def mock_RbVmomi_VIM_VirtualSCSIController(options = {})
-  options[:controllerKey]      = rand(65536) if options[:controllerKey].nil?
-  options[:key]                = rand(65536) if options[:key].nil?
-  options[:label]              = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:scsiCtlrUnitNumber] = 7 if options[:scsiCtlrUnitNumber].nil?
-
-  mock = MockVirtualSCSIController.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.scsiCtlrUnitNumber = options[:scsiCtlrUnitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualSCSIController
-  end
-
-  mock
-end

spec/spec_helper.rb

@@ -3,12 +3,10 @@ SimpleCov.start do
   add_filter '/spec/'
 end
 require 'helpers'
-require 'rbvmomi_helper'
-require 'rbvmomi'
 require 'rspec'
 require 'vmpooler'
 require 'redis'
-require 'vmpooler/statsd'
+require 'vmpooler/metrics'
 
 def project_root_dir
   File.dirname(File.dirname(__FILE__))

spec/unit/api/helpers_spec.rb

@@ -16,12 +16,12 @@ describe Vmpooler::API::Helpers do
 
   describe '#hostname_shorten' do
     [
-        ['example.com', 'not-example.com', 'example.com'],
-        ['example.com', 'example.com', 'example.com'],
-        ['sub.example.com', 'example.com', 'sub'],
-        ['example.com', nil, 'example.com']
-    ].each do |hostname, domain, expected|
-      it { expect(subject.hostname_shorten(hostname, domain)).to eq expected }
+        ['example.com', 'example'],
+        ['sub.example.com', 'sub'],
+        ['adjective-noun.example.com', 'adjective-noun'],
+        ['abc123.example.com', 'abc123']
+    ].each do |hostname, expected|
+      it { expect(subject.hostname_shorten(hostname)).to eq expected }
     end
   end
 
@@ -68,6 +68,7 @@ describe Vmpooler::API::Helpers do
 
   describe '#get_capacity_metrics' do
     let(:redis) { double('redis') }
+    let(:backend) { double('backend') }
 
     it 'adds up pools correctly' do
       pools = [
@@ -75,8 +76,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 5}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 1
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 2, total: 10, percent: 20.0})
     end
@@ -87,8 +87,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 5}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 0
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,0]
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 1, total: 10, percent: 10.0})
     end
@@ -99,13 +98,13 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 0}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 0
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,0]
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 1, total: 5, percent: 20.0})
     end
 
     it 'handles empty pool array' do
+      allow(redis).to receive(:pipelined).with(no_args).and_return []
       expect(subject.get_capacity_metrics([], redis)).to eq({current: 0, total: 0, percent: 0})
     end
   end
@@ -114,10 +113,10 @@ describe Vmpooler::API::Helpers do
     let(:redis) { double('redis') }
 
     it 'handles empty pool array' do
-      allow(redis).to receive(:scard).and_return 0
+      allow(redis).to receive(:pipelined).with(no_args).and_return [0]
       allow(redis).to receive(:get).and_return 0
 
-      expect(subject.get_queue_metrics([], redis)).to eq({pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
+      expect(subject.get_queue_metrics([], redis)).to eq({requested: 0, pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
     end
 
     it 'adds pool queues correctly' do
@@ -126,14 +125,14 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      pools.each do |p|
-        %w(pending ready running completed).each do |action|
-          allow(redis).to receive(:scard).with('vmpooler__' + action + '__' + p['name']).and_return 1
-        end
-      end
-      allow(redis).to receive(:get).and_return 1
+      # Mock returns 7*2 + 2 = 16 results (7 queue types for 2 pools + 2 global counters)
+      # For each pool: [request, processing, odcreate, pending, ready, running, completed]
+      # Plus 2 global counters: clone (1), ondemandclone (0)
+      # Results array: [1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1, 0]
+      #                [req,  proc,  odc,   pend, rdy,  run,  comp, clone, odc]
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 14})
     end
 
     it 'sets booting to 0 when negative calculation' do
@@ -142,14 +141,10 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      pools.each do |p|
-        %w(pending ready running completed).each do |action|
-          allow(redis).to receive(:scard).with('vmpooler__' + action + '__' + p['name']).and_return 1
-        end
-      end
-      allow(redis).to receive(:get).and_return 5
+      # Mock returns 7*2 + 2 = 16 results with clone=5 to cause negative booting
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 14})
     end
   end
 
@@ -271,23 +266,87 @@ describe Vmpooler::API::Helpers do
         }
       }
       let(:default_port) { 389 }
+      let(:default_encryption) do
+        {
+          :method => :start_tls,
+          :tls_options => { :ssl_version => 'TLSv1' }
+        }
+      end
+
+      context 'without a service account' do
         it 'should attempt ldap authentication' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when authentication is successful' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when authentication fails' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
+      end
+
+      context 'with a service account' do
+        let(:service_account_hash) do
+          {
+            :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+            :password => 's3cr3t'
+          }
+        end
+        let(:auth) {
+          {
+            'provider' => 'ldap',
+            ldap: {
+              'host' => host,
+              'base' => base,
+              'user_object' => user_object,
+              'service_account_hash' => service_account_hash
+            }
+          }
+        }
+        it 'should attempt ldap authentication' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when authentication is successful' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when authentication fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with an alternate ssl_version' do
+        let(:secure_encryption) do
+          {
+            :method => :start_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the alternate ssl_version when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+      end
 
       context 'with an alternate port' do
         let(:alternate_port) { 636 }
@@ -296,7 +355,27 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should specify the alternate port when authenticating' do
-          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, user_object, base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, default_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+      end
+
+      context 'with simple_tls and port 636' do
+        let(:secure_port) { 636 }
+        let(:secure_encryption) do
+          {
+            :method => :simple_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['port'] = secure_port
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the secure port and encryption options when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(secure_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
@@ -314,36 +393,165 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should attempt to bind with each base' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should not search the second base when the first binds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(true)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the second base when the first bind fails' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when any bind succeeds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when all bind attempts fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with multiple search user objects' do
+        let(:user_object) {
+          [
+            'uid',
+            'cn'
+          ]
+        }
+        before(:each) do
+          auth[:ldap]['user_object'] = user_object
+        end
+
+        it 'should attempt to bind with each user object' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should not search the second user object when the first binds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the second user object when the first bind fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when any bind succeeds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when all bind attempts fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with multiple search user objects and with multiple search bases' do
+        let(:user_object) {
+          [
+            'uid',
+            'cn'
+          ]
+        }
+        let(:base) {
+          [
+            'ou=user,dc=test,dc=com',
+            'ou=service,ou=user,dc=test,dc=com'
+          ]
+        }
+        before(:each) do
+          auth[:ldap]['base'] = base
+          auth[:ldap]['user_object'] = user_object
+        end
+
+        it 'should attempt to bind with each user object and base' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should not continue searching when the first combination binds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combinations when the first bind fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combinations when the first two binds fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combination when the first three binds fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when any bind succeeds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when all bind attempts fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
@@ -371,16 +579,25 @@ describe Vmpooler::API::Helpers do
     let(:base) { 'ou=users,dc=example,dc=com' }
     let(:username_str) { 'admin' }
     let(:password_str) { 's3cr3t' }
+    let(:encryption) do
+      {
+        :method => :start_tls,
+        :tls_options => { :ssl_version => 'TLSv1' }
+      }
+    end
+    let(:service_account_hash) do
+      {
+        :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+        :password => 's3cr3t'
+      }
+    end
     let(:ldap) { double('ldap') }
     it 'should create a new ldap connection' do
       allow(ldap).to receive(:bind)
       expect(Net::LDAP).to receive(:new).with(
         :host => host,
         :port => port,
-        :encryption => {
-          :method => :start_tls,
-          :tls_options => { :ssl_version => 'TLSv1' }
-        },
+        :encryption => encryption,
         :base => base,
         :auth => {
           :method => :simple,
@@ -389,21 +606,35 @@ describe Vmpooler::API::Helpers do
         }
       ).and_return(ldap)
 
-      subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)
+      subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)
     end
 
     it 'should return true when a bind is successful' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(true)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be true
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be true
     end
 
     it 'should return false when a bind fails' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(false)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be false
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be false
+    end
+
+    it 'should return true when a bind_as is successful' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(true)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be true
+    end
+
+    it 'should return false when a bind_as fails' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(false)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be false
     end
   end
 

spec/unit/api/input_validator_spec.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rack/test'
+require 'vmpooler/api/input_validator'
+
+describe Vmpooler::API::InputValidator do
+  let(:test_class) do
+    Class.new do
+      include Vmpooler::API::InputValidator
+    end
+  end
+  let(:validator) { test_class.new }
+
+  describe '#validate_hostname' do
+    it 'accepts valid hostnames' do
+      expect(validator.validate_hostname('test-host.example.com')).to be true
+      expect(validator.validate_hostname('host123')).to be true
+    end
+
+    it 'rejects invalid hostnames' do
+      result = validator.validate_hostname('invalid_host!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid hostname format')
+    end
+
+    it 'rejects hostnames that are too long' do
+      long_hostname = 'a' * 300
+      result = validator.validate_hostname(long_hostname)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+
+    it 'rejects empty hostnames' do
+      result = validator.validate_hostname('')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('required')
+    end
+  end
+
+  describe '#validate_pool_name' do
+    it 'accepts valid pool names' do
+      expect(validator.validate_pool_name('centos-7-x86_64')).to be true
+      expect(validator.validate_pool_name('ubuntu-2204')).to be true
+    end
+
+    it 'rejects invalid pool names' do
+      result = validator.validate_pool_name('invalid pool!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid pool name format')
+    end
+
+    it 'rejects pool names that are too long' do
+      result = validator.validate_pool_name('a' * 150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_tag' do
+    it 'accepts valid tags' do
+      expect(validator.validate_tag('project', 'test-123')).to be true
+      expect(validator.validate_tag('owner', 'user@example.com')).to be true
+    end
+
+    it 'rejects tags with invalid keys' do
+      result = validator.validate_tag('invalid key!', 'value')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid tag key format')
+    end
+
+    it 'rejects tags with invalid characters in value' do
+      result = validator.validate_tag('key', 'value<script>')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('invalid characters')
+    end
+
+    it 'rejects tags that are too long' do
+      result = validator.validate_tag('key', 'a' * 300)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_vm_count' do
+    it 'accepts valid VM counts' do
+      expect(validator.validate_vm_count(5)).to eq(5)
+      expect(validator.validate_vm_count('10')).to eq(10)
+    end
+
+    it 'rejects counts less than 1' do
+      result = validator.validate_vm_count(0)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at least 1')
+    end
+
+    it 'rejects counts greater than 100' do
+      result = validator.validate_vm_count(150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 100')
+    end
+
+    it 'rejects non-integer values' do
+      result = validator.validate_vm_count('abc')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('valid integer')
+    end
+  end
+
+  describe '#validate_disk_size' do
+    it 'accepts valid disk sizes' do
+      expect(validator.validate_disk_size(50)).to eq(50)
+      expect(validator.validate_disk_size('100')).to eq(100)
+    end
+
+    it 'rejects sizes less than 1' do
+      result = validator.validate_disk_size(0)
+      expect(result['ok']).to be false
+    end
+
+    it 'rejects sizes greater than 2048' do
+      result = validator.validate_disk_size(3000)
+      expect(result['ok']).to be false
+    end
+  end
+
+  describe '#validate_lifetime' do
+    it 'accepts valid lifetimes' do
+      expect(validator.validate_lifetime(24)).to eq(24)
+      expect(validator.validate_lifetime('48')).to eq(48)
+    end
+
+    it 'rejects lifetimes greater than 168 hours (1 week)' do
+      result = validator.validate_lifetime(200)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 168')
+    end
+  end
+
+  describe '#sanitize_json_body' do
+    it 'parses valid JSON' do
+      result = validator.sanitize_json_body('{"key": "value"}')
+      expect(result).to eq('key' => 'value')
+    end
+
+    it 'rejects invalid JSON' do
+      result = validator.sanitize_json_body('{invalid}')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid JSON')
+    end
+
+    it 'rejects non-object JSON' do
+      result = validator.sanitize_json_body('["array"]')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('must be a JSON object')
+    end
+
+    it 'rejects deeply nested JSON' do
+      deep_json = '{"a":{"b":{"c":{"d":{"e":{"f":"too deep"}}}}}}'
+      result = validator.sanitize_json_body(deep_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too complex')
+    end
+
+    it 'rejects bodies that are too large' do
+      large_json = '{"data":"' + ('a' * 20000) + '"}'
+      result = validator.sanitize_json_body(large_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too large')
+    end
+  end
+
+  describe '#validation_error?' do
+    it 'returns true for error responses' do
+      error = { 'ok' => false, 'error' => 'test error' }
+      expect(validator.validation_error?(error)).to be true
+    end
+
+    it 'returns false for successful responses' do
+      expect(validator.validation_error?(true)).to be false
+      expect(validator.validation_error?(5)).to be false
+    end
+  end
+end

spec/unit/collector_middleware_spec.rb

@@ -0,0 +1,191 @@
+
+require 'rack/test'
+require 'vmpooler/metrics/promstats/collector_middleware'
+
+
+describe Vmpooler::Metrics::Promstats::CollectorMiddleware do
+  include Rack::Test::Methods
+
+  # Reset the data store
+  before do
+    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+  end
+
+  let(:registry) do
+    Prometheus::Client::Registry.new
+  end
+
+  let(:original_app) do
+    ->(_) { [200, { 'Content-Type' => 'text/html' }, ['OK']] }
+  end
+
+  let!(:app) do
+    described_class.new(original_app, registry: registry)
+  end
+
+  let(:dummy_error) { RuntimeError.new("Dummy error from tests") }
+
+  it 'returns the app response' do
+    get '/foo'
+
+    expect(last_response).to be_ok
+    expect(last_response.body).to eql('OK')
+  end
+
+  it 'handles errors in the registry gracefully' do
+    counter = registry.get(:http_server_requests_total)
+    expect(counter).to receive(:increment).and_raise(dummy_error)
+
+    get '/foo'
+
+    expect(last_response).to be_ok
+  end
+
+  it 'traces request information' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.2)
+
+    get '/foo'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.25" => 1)
+  end
+
+  it 'normalizes paths containing /vm by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/foo/vm/bar-mumble-flame'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo/vm', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo/vm' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /ondemandvm by ' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/foo/ondemandvm/bar/fatman'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo/ondemandvm', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo/ondemandvm' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /token by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/token/secret-token-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/token', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/token' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /api/v1/token by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/api/v1/token/secret-token-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/api/v1/token', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/api/v1/token' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /img by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/img/image-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/img', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/img' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /lib by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/lib/xxxxx.js'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/lib', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/lib' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  context 'when the app raises an exception' do
+    let(:original_app) do
+      lambda do |env|
+        raise dummy_error if env['PATH_INFO'] == '/broken'
+
+        [200, { 'Content-Type' => 'text/html' }, ['OK']]
+      end
+    end
+
+    before do
+      get '/foo'
+    end
+
+    it 'traces exceptions' do
+      expect { get '/broken' }.to raise_error RuntimeError
+
+      metric = :http_server_exceptions_total
+      labels = { exception: 'RuntimeError' }
+      expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+    end
+  end
+
+  context 'when provided a custom metrics_prefix' do
+    let!(:app) do
+      described_class.new(
+        original_app,
+        registry: registry,
+        metrics_prefix: 'lolrus',
+      )
+    end
+
+    it 'provides alternate metric names' do
+      expect(
+        registry.get(:lolrus_requests_total),
+      ).to be_a(Prometheus::Client::Counter)
+      expect(
+        registry.get(:lolrus_request_duration_seconds),
+      ).to be_a(Prometheus::Client::Histogram)
+      expect(
+        registry.get(:lolrus_exceptions_total),
+      ).to be_a(Prometheus::Client::Counter)
+    end
+
+    it "doesn't register the default metrics" do
+      expect(registry.get(:http_server_requests_total)).to be(nil)
+      expect(registry.get(:http_server_request_duration_seconds)).to be(nil)
+      expect(registry.get(:http_server_exceptions_total)).to be(nil)
+    end
+  end
+end

spec/unit/dns/base_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+require 'vmpooler/dns/base'
+
+# This spec does not really exercise code paths but is merely used
+# to enforce that certain methods are defined in the base classes
+
+describe 'Vmpooler::PoolManager::Dns::Base' do
+  let(:logger) { MockLogger.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { {} }
+  let(:dns_plugin_name) { 'base' }
+  let(:dns_options) { { 'param' => 'value' } }
+
+  let(:fake_vm) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
+  }
+
+  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
+    metrics: metrics,
+    connpool_type: 'redis_connection_pool',
+    connpool_provider: 'testprovider',
+    size: 1,
+    timeout: 5
+  ) { MockRedis.new }
+  }
+
+  subject { Vmpooler::PoolManager::Dns::Base.new(config, logger, metrics, redis_connection_pool, dns_plugin_name, dns_options) }
+
+  # Helper attr_reader methods
+  describe '#logger' do
+    it 'should come from the provider initialization' do
+      expect(subject.logger).to be(logger)
+    end
+  end
+
+  describe '#metrics' do
+    it 'should come from the provider initialization' do
+      expect(subject.metrics).to be(metrics)
+    end
+  end
+
+  describe '#dns_options' do
+    it 'should come from the provider initialization' do
+      expect(subject.dns_options).to be(dns_options)
+    end
+  end
+
+  describe '#pool_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: '#{poolname}'
+    alias: [ 'mockpool' ]
+    template: 'Templates/pool1'
+    folder: 'Pooler/pool1'
+    datastore: 'datastore0'
+    size: 5
+    timeout: 10
+    ready_ttl: 1440
+    clone_target: 'cluster1'
+EOT
+      )
+    }
+    context 'Given a pool that does not exist' do
+      it 'should return nil' do
+        expect(subject.pool_config('missing_pool')).to be_nil
+      end
+    end
+
+    context 'Given a pool that does exist' do
+      it 'should return the pool\'s configuration' do
+        result = subject.pool_config(poolname)
+        expect(result['name']).to eq(poolname)
+      end
+    end
+  end
+
+  describe '#dns_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :#{dns_plugin_name}:
+    option1: 'value1'
+EOT
+      )
+    }
+
+    context 'Given a dns plugin with no configuration' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :bad_dns:
+    option1: 'value1'
+    option2: 'value1'
+EOT
+      )
+    }
+      it 'should return nil' do
+        expect(subject.dns_config).to be_nil
+      end
+    end
+
+    context 'Given a correct dns config name' do
+      it 'should return the dns\'s configuration' do
+        result = subject.dns_config
+        expect(result['option1']).to eq('value1')
+      end
+    end
+  end
+
+  describe '#global_config' do
+    it 'should come from the dns initialization' do
+      expect(subject.global_config).to be(config)
+    end
+  end
+
+  describe '#name' do
+    it "should come from the dns initialization" do
+      expect(subject.name).to eq(dns_plugin_name)
+    end
+  end
+
+  describe '#get_ip' do
+    it 'calls redis hget with vm name and ip' do
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with("vmpooler__vm__vm1", 'ip')
+      end
+      subject.get_ip(fake_vm['name'])
+    end
+  end
+
+  describe '#provided_pools' do
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: 'pool1'
+    dns_config: 'base'
+  - name: 'pool2'
+    dns_config: 'base'
+  - name: 'otherpool'
+    dns_config: 'other provider'
+  - name: 'no name'
+EOT
+      )
+    }
+
+    it "should return pools serviced by this provider" do
+      expect(subject.provided_pools).to eq(['pool1','pool2'])
+    end
+  end
+
+  describe '#create_or_replace_record' do
+    it 'should raise error' do
+      expect{subject.create_or_replace_record('pool')}.to raise_error(/does not implement create_or_replace_record/)
+    end
+  end
+
+  describe '#delete_record' do
+    it 'should raise error' do
+      expect{subject.delete_record('pool')}.to raise_error(/does not implement delete_record/)
+    end
+  end
+end

spec/unit/dns_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe 'Vmpooler::Dns' do
+  let(:dns_class) { 'mock-dnsservice' }
+  let(:dns_config_name) { 'mock' }
+  let(:pool) { 'pool1' }
+  let(:config) { YAML.load(<<~EOT
+  ---
+  :dns_configs:
+    :mock:
+      dns_class: 'mock'
+      domain: 'example.com'
+  :pools:
+    - name: 'pool1'
+      dns_plugin: 'mock'
+EOT
+  )}
+  subject { Vmpooler::Dns.new }
+
+  describe '.get_dns_plugin_class_by_name' do
+    it 'returns the plugin class for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_class_by_name(config, dns_config_name)
+      expect(result).to eq('mock')
+    end
+  end
+
+  describe '.get_domain_for_pool' do
+    it 'returns the domain for the specified pool' do
+      result = Vmpooler::Dns.get_domain_for_pool(config, pool)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_domain_by_name' do
+    it 'returns the domain for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_domain_by_name(config, dns_config_name)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_config_classes' do
+    it 'returns the list of dns plugin classes' do
+      result = Vmpooler::Dns.get_dns_plugin_config_classes(config)
+      expect(result).to eq(['mock'])
+    end
+  end
+
+  describe '#load_from_gems' do
+    let(:gem_name) { 'mock-dnsservice' }
+    let(:translated_gem_name) { 'mock/dnsservice' }
+
+    before(:each) do
+      allow(subject).to receive(:require).with(gem_name).and_return(true)
+    end
+
+    it 'loads the specified gem' do
+      expect(subject).to receive(:require).with("vmpooler/dns/#{translated_gem_name}")
+      result = subject.load_from_gems(gem_name)
+      expect(result).to eq("vmpooler/dns/#{translated_gem_name}")
+    end
+  end
+end

spec/unit/env_config.rb

@@ -0,0 +1,224 @@
+require 'vmpooler'
+require 'climate_control'
+require 'mock_redis'
+
+describe 'Vmpooler' do
+  describe 'config' do
+
+    describe 'environment variables' do
+
+      test_integer = '5'
+      test_string = 'test_string'
+      test_bool = 'true'
+      test_cases = [
+        ['migration_limit', test_integer, nil],
+        ['task_limit', test_integer, 10],
+        ['vm_checktime', test_integer, 1],
+        ['vm_lifetime', test_integer, 24],
+        ['timeout', test_integer, nil],
+        ['vm_lifetime_auth', test_integer, nil],
+        ['max_tries', test_integer, nil],
+        ['retry_factor', test_integer, nil],
+        ['prefix', test_string, ""],
+        ['logfile', test_string, nil],
+        ['site_name', test_string, nil],
+        ['clone_target', test_string, nil],
+        ['create_folders', test_bool, nil],
+        ['create_template_delta_disks', test_bool, nil],
+        ['create_linked_clones', test_bool, nil],
+        ['experimental_features', test_bool, nil],
+        ['purge_unconfigured_resources', test_bool, nil],
+        ['usage_stats', test_bool, nil],
+        ['request_logger', test_bool, nil],
+        ['extra_config', test_string, nil],
+      ]
+
+      test_cases.each do |key, value, default|
+        it "should set a value for #{key}" do
+          with_modified_env "#{key.upcase}": value do
+            test_value = value
+            test_value = Integer(value) if value =~ /\d/
+            config = Vmpooler.config
+
+            expect(config[:config][key]).to eq(test_value)
+          end
+        end
+
+        it "should set a default value for each #{key}" do
+          config = Vmpooler.config
+
+          expect(config[:config][key]).to eq(default)
+        end
+
+        if value =~ /\d/
+          it "should not set bad_data as a value for #{key}" do
+            with_modified_env "#{key.upcase}": 'bad_data' do
+              config = Vmpooler.config
+
+              expect(config[:config][key]).to eq(default)
+            end
+          end
+        end
+      end
+    end
+
+    describe 'redis environment variables' do
+      let(:redis) { MockRedis.new }
+
+      test_cases = [
+        ['server', 'redis', 'localhost'],
+        ['port', '4567', nil],
+        ['password', 'testpass', nil],
+        ['data_ttl', '500', 168],
+      ]
+
+      test_cases.each do |key, value, default|
+        it "should set a value for #{key}" do
+          allow(Vmpooler).to receive(:new_redis).and_return(redis)
+          with_modified_env "REDIS_#{key.upcase}": value do
+            test_value = value
+            test_value = Integer(value) if value =~ /\d/
+            config = Vmpooler.config
+
+            expect(config[:redis][key]).to eq(test_value)
+          end
+        end
+
+        it "should set a default value for each #{key}" do
+          config = Vmpooler.config
+
+          expect(config[:redis][key]).to eq(default)
+        end
+
+        if value =~ /\d/
+          it "should not set bad_data as a value for #{key}" do
+            with_modified_env "#{key.upcase}": 'bad_data' do
+              config = Vmpooler.config
+
+              expect(config[:redis][key]).to eq(default)
+            end
+          end
+        end
+      end
+    end
+
+    describe 'statsd environment variables' do
+
+      test_cases = [
+        ['prefix', 'vmpooler', nil],
+        ['port', '4567', nil],
+      ]
+
+      test_cases.each do |key, value, default|
+        it "should set a value for #{key}" do
+          with_modified_env STATSD_SERVER: 'test', "STATSD_#{key.upcase}": value do
+            test_value = value
+            test_value = Integer(value) if value =~ /\d/
+            config = Vmpooler.config
+
+            expect(config[:statsd][key]).to eq(test_value) end
+        end
+
+        it "should set a default value for each #{key}" do
+          with_modified_env STATSD_SERVER: 'test' do
+            config = Vmpooler.config
+
+            expect(config[:statsd][key]).to eq(default)
+          end
+        end
+
+        if value =~ /\d/
+          it "should not set bad_data as a value for #{key}" do
+            with_modified_env STATSD_SERVER: 'test', "STATSD_#{key.upcase}": 'bad_data' do
+              config = Vmpooler.config
+
+              expect(config[:statsd][key]).to eq(default)
+            end
+          end
+        end
+      end
+    end
+
+    describe 'graphite environment variables' do
+
+      test_cases = [
+        ['prefix', 'vmpooler', nil],
+        ['port', '4567', nil],
+      ]
+
+      test_cases.each do |key, value, default|
+        it "should set a value for #{key}" do
+          with_modified_env GRAPHITE_SERVER: 'test', "GRAPHITE_#{key.upcase}": value do
+            test_value = value
+            test_value = Integer(value) if value =~ /\d/
+            config = Vmpooler.config
+
+            expect(config[:graphite][key]).to eq(test_value)
+          end
+        end
+
+        it "should set a default value for each #{key}" do
+          with_modified_env GRAPHITE_SERVER: 'test' do
+            config = Vmpooler.config
+
+            expect(config[:graphite][key]).to eq(default)
+          end
+        end
+
+        if value =~ /\d/
+          it "should not set bad_data as a value for #{key}" do
+            with_modified_env GRAPHITE_SERVER: 'test', "GRAPHITE_#{key.upcase}": 'bad_data' do
+              config = Vmpooler.config
+
+              expect(config[:graphite][key]).to eq(default)
+            end
+          end
+        end
+      end
+    end
+
+    describe 'ldap environment variables' do
+
+      test_cases = [
+        ['host', 'test', nil],
+        ['port', '4567', nil],
+        ['base', 'dc=example,dc=com', nil],
+        ['user_object', 'uid', nil],
+      ]
+
+      test_cases.each do |key, value, default|
+        it "should set a value for #{key}" do
+          with_modified_env AUTH_PROVIDER: 'ldap', "LDAP_#{key.upcase}": value do
+            test_value = value
+            test_value = Integer(value) if value =~ /\d/
+            config = Vmpooler.config
+
+            expect(config[:auth][:ldap][key]).to eq(test_value)
+          end
+        end
+
+        it "should set a default value for each #{key}" do
+          with_modified_env AUTH_PROVIDER: 'ldap' do
+            config = Vmpooler.config
+
+            expect(config[:auth][:ldap][key]).to eq(default)
+          end
+        end
+
+        if value =~ /\d/
+          it "should not set bad_data as a value for #{key}" do
+            with_modified_env AUTH_PROVIDER: 'ldap', "LDAP_#{key.upcase}": 'bad_data' do
+              config = Vmpooler.config
+
+              expect(config[:auth][:ldap][key]).to eq(default)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def with_modified_env(options, &block)
+    ClimateControl.modify(options, &block)
+  end
+end

spec/unit/generic_connection_pool_spec.rb

@@ -1,16 +1,18 @@
 require 'spec_helper'
 
 describe 'GenericConnectionPool' do
-  let(:metrics) { Vmpooler::DummyStatsd.new }
-  let(:metric_prefix) { 'prefix' }
-  let(:default_metric_prefix) { 'connectionpool' }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:connpool_type) { 'test_connection_pool' }
+  let(:connpool_provider) { 'testprovider' }
+  let(:default_connpool_type) { 'connectionpool' }
   let(:connection_object) { double('connection') }
   let(:pool_size) { 1 }
   let(:pool_timeout) { 1 }
 
   subject { Vmpooler::PoolManager::GenericConnectionPool.new(
               metrics: metrics,
-              metric_prefix: metric_prefix,
+              connpool_type: connpool_type,
+              connpool_provider: connpool_provider,
               size: pool_size,
               timeout: pool_timeout
             ) { connection_object }
@@ -23,6 +25,13 @@ describe 'GenericConnectionPool' do
       connection: 'connection'
     }}
 
+    it 'should error out when connection pool could not establish no nothing' do
+      newsub = Vmpooler.redis_connection_pool("foo,bar", "1234", "fuba", 1, 1, metrics, 0)
+      expect { newsub.with_metrics do |conn_pool_object|
+        conn_pool_object.srem('foo', "bar")
+      end }.to raise_error Redis::CannotConnectError
+    end
+
     it 'should return a connection object when grabbing one from the pool' do
       subject.with_metrics do |conn_pool_object|
         expect(conn_pool_object).to be(connection_object)
@@ -65,7 +74,7 @@ describe 'GenericConnectionPool' do
 
     context 'When metrics are configured' do
       it 'should emit a gauge metric when the connection is grabbed and released' do
-        expect(metrics).to receive(:gauge).with(/\.available/,Integer).exactly(2).times
+        expect(metrics).to receive(:gauge).with(/connection_available/,Integer).exactly(2).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -73,7 +82,7 @@ describe 'GenericConnectionPool' do
       end
 
       it 'should emit a timing metric when the connection is grabbed' do
-        expect(metrics).to receive(:timing).with(/\.waited/,Integer).exactly(1).times
+        expect(metrics).to receive(:timing).with(/connection_waited/,Integer).exactly(1).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -81,8 +90,8 @@ describe 'GenericConnectionPool' do
       end
 
       it 'should emit metrics with the specified prefix' do
-        expect(metrics).to receive(:gauge).with(/#{metric_prefix}\./,Integer).at_least(1).times
-        expect(metrics).to receive(:timing).with(/#{metric_prefix}\./,Integer).at_least(1).times
+        expect(metrics).to receive(:gauge).with(/#{connpool_type}\./,Integer).at_least(1).times
+        expect(metrics).to receive(:timing).with(/#{connpool_type}\./,Integer).at_least(1).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -90,11 +99,11 @@ describe 'GenericConnectionPool' do
       end
 
       context 'Metrix prefix is missing' do
-        let(:metric_prefix) { nil }
+        let(:connpool_type) { nil }
 
         it 'should emit metrics with default prefix' do
-          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
-          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:gauge).with(/#{default_connpool_type}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_connpool_type}\./,Integer).at_least(1).times
 
           subject.with_metrics do |conn1|
             # do nothing
@@ -103,11 +112,11 @@ describe 'GenericConnectionPool' do
       end
 
       context 'Metrix prefix is empty' do
-        let(:metric_prefix) { '' }
+        let(:connpool_type) { '' }
 
         it 'should emit metrics with default prefix' do
-          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
-          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:gauge).with(/#{default_connpool_type}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_connpool_type}\./,Integer).at_least(1).times
 
           subject.with_metrics do |conn1|
             # do nothing

spec/unit/promstats_spec.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'prometheus' do
+  logger = MockLogger.new
+  params = { 'prefix' => 'test', 'prometheus_prefix'  => 'mtest', 'prometheus_endpoint'  => 'eptest' }
+  subject = Vmpooler::Metrics::Promstats.new(logger, params)
+  let(:logger) { MockLogger.new }
+
+  describe '#initialise' do
+    it 'returns a Metrics object' do
+      expect(Vmpooler::Metrics::Promstats.new(logger)).to be_a(Vmpooler::Metrics)
+    end
+  end
+
+  describe '#find_metric' do
+    context "Single Value Parameters" do
+      let!(:foo_metrics) do
+        { metric_suffixes: { bar: 'baz' },
+          param_labels: %i[first second last] }
+      end
+      let!(:labels_hash) { { labels: { :first => nil, :second => nil, :last => nil } } }
+      before { 
+        subject.instance_variable_set(:@p_metrics, { foo: foo_metrics, torun: %i[api] }) 
+        subject.instance_variable_set(:@torun, [ :api ]) 
+      }
+  
+      it 'returns the metric for a given label including parsed labels' do
+        expect(subject.find_metric('foo.bar')).to include(metric_name: 'mtest_foo_bar')
+        expect(subject.find_metric('foo.bar')).to include(foo_metrics)
+        expect(subject.find_metric('foo.bar')).to include(labels_hash)
+      end
+
+      it 'raises an error when the given label is not present in metrics' do
+        expect { subject.find_metric('bogus') }.to raise_error(RuntimeError, 'Invalid Metric bogus for bogus')
+      end
+  
+      it 'raises an error when the given label specifies metric_suffixes but the following suffix not present in metrics' do
+        expect { subject.find_metric('foo.metric_suffixes.bogus') }.to raise_error(RuntimeError, 'Invalid Metric foo_metric_suffixes for foo.metric_suffixes.bogus')
+      end
+    end
+
+    context "Node Name Handling" do
+      let!(:node_metrics) do
+        { metric_name: 'connection_to',
+          param_labels: %i[node],
+          torun: %i[api] 
+        }
+      end
+      let!(:nodename_hash) { { labels: { :node => 'test.bar.net'}}}
+      before { 
+        subject.instance_variable_set(:@p_metrics, { connection_to: node_metrics })
+        subject.instance_variable_set(:@torun, [ :api ]) 
+      }
+
+      it 'Return final remaining fields (e.g. fqdn) in last label' do
+        expect(subject.find_metric('connection_to.test.bar.net')).to include(nodename_hash)
+      end
+    end
+  end
+
+  context 'setup_prometheus_metrics' do
+    before(:all) do
+      Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+      subject.setup_prometheus_metrics(%i[api manager])
+    end
+
+    describe '#setup_prometheus_metrics' do
+      it 'calls add_prometheus_metric for each item in list' do
+        Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+        expect(subject).to receive(:add_prometheus_metric).at_least(subject.vmpooler_metrics_table.size).times
+        subject.setup_prometheus_metrics(%i[api manager])
+      end
+    end
+
+    describe '#increment' do
+      it 'Increments checkout.nonresponsive.#{template_backend}' do
+        template_backend = 'test'
+        expect { subject.increment("checkout.nonresponsive.#{template_backend}") }.to change {
+          metric, po = subject.get("checkout.nonresponsive.#{template_backend}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.empty. + requested' do
+        requested = 'test'
+        expect { subject.increment('checkout.empty.' + requested) }.to change {
+          metric, po = subject.get('checkout.empty.' + requested)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.success. + vmtemplate' do
+        vmtemplate = 'test-template'
+        expect { subject.increment('checkout.success.' + vmtemplate) }.to change {
+          metric, po = subject.get('checkout.success.' + vmtemplate)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.invalid. + bad_template' do
+        bad_template = 'test-template'
+        expect { subject.increment('checkout.invalid.' + bad_template) }.to change {
+          metric, po = subject.get('checkout.invalid.' + bad_template)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.invalid.unknown' do
+        expect { subject.increment('checkout.invalid.unknown') }.to change {
+          metric, po = subject.get('checkout.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments delete.failed' do
+        bad_template = 'test-template'
+        expect { subject.increment('delete.failed') }.to change {
+          metric, po = subject.get('delete.failed')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments delete.success' do
+        bad_template = 'test-template'
+        expect { subject.increment('delete.success') }.to change {
+          metric, po = subject.get('delete.success')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_generate.duplicaterequests' do
+        bad_template = 'test-template'
+        expect { subject.increment('ondemandrequest_generate.duplicaterequests') }.to change {
+          metric, po = subject.get('ondemandrequest_generate.duplicaterequests')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_generate.success' do
+        bad_template = 'test-template'
+        expect { subject.increment('ondemandrequest_generate.success') }.to change {
+          metric, po = subject.get('ondemandrequest_generate.success')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_fail.toomanyrequests.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("ondemandrequest_fail.toomanyrequests.#{bad_template}") }.to change {
+          metric, po = subject.get("ondemandrequest_fail.toomanyrequests.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_fail.invalid.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("ondemandrequest_fail.invalid.#{bad_template}") }.to change {
+          metric, po = subject.get("ondemandrequest_fail.invalid.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments config.invalid.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("config.invalid.#{bad_template}") }.to change {
+          metric, po = subject.get("config.invalid.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments config.invalid.unknown' do
+        expect { subject.increment('config.invalid.unknown') }.to change {
+          metric, po = subject.get('config.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments poolreset.invalid.#{bad_pool}' do
+        bad_pool = 'test-pool'
+        expect { subject.increment("poolreset.invalid.#{bad_pool}") }.to change {
+          metric, po = subject.get("poolreset.invalid.#{bad_pool}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments poolreset.invalid.unknown' do
+        expect { subject.increment('poolreset.invalid.unknown') }.to change {
+          metric, po = subject.get('poolreset.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.markedasfailed.#{pool}' do
+        pool = 'test-pool'
+        expect { subject.increment("errors.markedasfailed.#{pool}") }.to change {
+          metric, po = subject.get("errors.markedasfailed.#{pool}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.duplicatehostname.#{pool_name}' do
+        pool_name = 'test-pool'
+        expect { subject.increment("errors.duplicatehostname.#{pool_name}") }.to change {
+          metric, po = subject.get("errors.duplicatehostname.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.staledns.#{pool_name}' do
+        pool_name = 'test-pool'
+        expect { subject.increment("errors.staledns.#{pool_name}") }.to change {
+          metric, po = subject.get("errors.staledns.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments user.#{user}.#{operation}.#{poolname}' do
+        user = 'myuser'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("user.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("user.#{user}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments usage_litmus.#{user}.#{operation}.#{poolname}' do
+        user = 'myuser'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_litmus.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_litmus.#{user}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}' do
+        jenkins_instance = 'jenkins_test_instance'
+        value_stream = 'notional_value'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}' do
+        branch = 'treetop'
+        project = 'test-project'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}' do
+        job_name = 'a-job'
+        component_to_test = 'component-name'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments connect.open' do
+        expect { subject.increment('connect.open') }.to change {
+          metric, po = subject.get('connect.open')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments connect.fail' do
+        expect { subject.increment('connect.fail') }.to change {
+          metric, po = subject.get('connect.fail')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments migrate_from.#{vm_hash[\'host_name\']}' do
+        vm_hash = { 'host_name': 'testhost.testdomain' }
+        expect { subject.increment("migrate_from.#{vm_hash['host_name']}") }.to change {
+          metric, po = subject.get("migrate_from.#{vm_hash['host_name']}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments "migrate_to.#{dest_host_name}"' do
+        dest_host_name = 'testhost.testdomain'
+        expect { subject.increment("migrate_to.#{dest_host_name}") }.to change {
+          metric, po = subject.get("migrate_to.#{dest_host_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label http_requests_vm_total.#{method}.#{subpath}.#{operation}' do
+        method = 'get'
+        subpath = 'template'
+        operation = 'something'
+        expect { subject.increment("http_requests_vm_total.#{method}.#{subpath}.#{operation}") }.to change {
+          metric, po = subject.get("http_requests_vm_total.#{method}.#{subpath}.#{operation}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+
+    end
+
+    describe '#gauge' do
+      # metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
+      it 'sets value of ready.#{pool_name} to $redis.scard("vmpooler__ready__#{pool_name}"))' do
+        # is there a specific redis value that should be tested?
+        pool_name = 'test-pool'
+        test_value = 42
+        expect { subject.gauge("ready.#{pool_name}", test_value) }.to change {
+          metric, po = subject.get("ready.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.from(0).to(42)
+      end
+      # metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
+      it 'sets value of running.#{pool_name} to $redis.scard("vmpooler__running__#{pool_name}"))' do
+        # is there a specific redis value that should be tested?
+        pool_name = 'test-pool'
+        test_value = 42
+        expect { subject.gauge("running.#{pool_name}", test_value) }.to change {
+          metric, po = subject.get("running.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.from(0).to(42)
+      end
+    end
+
+    describe '#timing' do
+      it 'sets histogram value of time_to_ready_state.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("time_to_ready_state.#{pool}", finish) }.to change {
+          metric, po = subject.get("time_to_ready_state.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of clone.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("clone.#{pool}", finish) }.to change {
+          metric, po = subject.get("clone.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of migrate.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("migrate.#{pool}", finish) }.to change {
+          metric, po = subject.get("migrate.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of destroy.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("destroy.#{pool}", finish) }.to change {
+          metric, po = subject.get("destroy.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+    end
+  end
+end

spec/unit/providers/base_spec.rb

@@ -6,7 +6,7 @@ require 'vmpooler/providers/base'
 
 describe 'Vmpooler::PoolManager::Provider::Base' do
   let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
   let(:config) { {} }
   let(:provider_name) { 'base' }
   let(:provider_options) { { 'param' => 'value' } }
@@ -22,7 +22,9 @@ describe 'Vmpooler::PoolManager::Provider::Base' do
     fake_vm
   }
 
-  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, provider_name, provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, provider_name, provider_options) }
 
   # Helper attr_reader methods
   describe '#logger' do

spec/unit/providers/dummy_spec.rb

@@ -3,7 +3,7 @@ require 'vmpooler/providers/dummy'
 
 describe 'Vmpooler::PoolManager::Provider::Dummy' do
   let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
   let(:pool_name) { 'pool1' }
   let(:other_pool_name) { 'pool2' }
   let(:vm_name) { 'vm1' }
@@ -91,7 +91,9 @@ EOT
     )
   }
 
-  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, 'dummy', provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, redis_connection_pool, 'dummy', provider_options) }
 
   describe '#name' do
     it 'should be dummy' do

spec/unit/providers_spec.rb

@@ -12,12 +12,8 @@ describe 'providers' do
   end
 
   it '#load_all_providers' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(Vmpooler::Providers.load_all_providers).to eq(p)
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
   it '#installed_providers' do
@@ -30,21 +26,18 @@ describe 'providers' do
   end
 
   it '#load_by_name' do
-    expect(Vmpooler::Providers.load_by_name('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to_not match(%r{,})
   end
 
-  it '#load only vpshere' do
-    expect(providers.load_from_gems('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+  it '#load only dummy' do
+    expect(providers.load_from_gems('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(providers.load_from_gems('dummy').join(', ')).to_not match(%r{,})
   end
 
   it '#load all providers from gems' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(providers.load_from_gems).to eq(p)
-
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
 

spec/unit/queue_reliability_spec.rb

@@ -0,0 +1,497 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'vmpooler/pool_manager'
+
+describe 'Vmpooler::PoolManager - Queue Reliability Features' do
+  let(:logger) { MockLogger.new }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { redis } }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { YAML.load(<<~EOT
+    ---
+    :config:
+      task_limit: 10
+      vm_checktime: 1
+      vm_lifetime: 12
+      prefix: 'pooler-'
+      dlq_enabled: true
+      dlq_ttl: 168
+      dlq_max_entries: 100
+      purge_enabled: true
+      purge_dry_run: false
+      max_pending_age: 7200
+      max_ready_age: 86400
+      max_completed_age: 3600
+      health_check_enabled: true
+      health_check_interval: 300
+      health_thresholds:
+        pending_queue_max: 100
+        ready_queue_max: 500
+        dlq_max_warning: 100
+        dlq_max_critical: 1000
+        stuck_vm_age_threshold: 7200
+    :providers:
+      :dummy: {}
+    :pools:
+      - name: 'test-pool'
+        size: 5
+        provider: 'dummy'
+    EOT
+    )
+  }
+
+  subject { Vmpooler::PoolManager.new(config, logger, redis_connection_pool, metrics) }
+
+  describe 'Dead-Letter Queue (DLQ)' do
+    let(:vm) { 'vm-abc123' }
+    let(:pool) { 'test-pool' }
+    let(:error_class) { 'StandardError' }
+    let(:error_message) { 'template does not exist' }
+    let(:request_id) { 'req-123' }
+    let(:pool_alias) { 'test-alias' }
+
+    before(:each) do
+      redis_connection_pool.with do |redis_connection|
+        allow(redis_connection).to receive(:zadd)
+        allow(redis_connection).to receive(:zcard).and_return(0)
+        allow(redis_connection).to receive(:expire)
+      end
+    end
+
+    describe '#dlq_enabled?' do
+      it 'returns true when dlq_enabled is true in config' do
+        expect(subject.dlq_enabled?).to be true
+      end
+
+      it 'returns false when dlq_enabled is false in config' do
+        config[:config]['dlq_enabled'] = false
+        expect(subject.dlq_enabled?).to be false
+      end
+    end
+
+    describe '#dlq_ttl' do
+      it 'returns configured TTL' do
+        expect(subject.dlq_ttl).to eq(168)
+      end
+
+      it 'returns default TTL when not configured' do
+        config[:config].delete('dlq_ttl')
+        expect(subject.dlq_ttl).to eq(168)
+      end
+    end
+
+    describe '#dlq_max_entries' do
+      it 'returns configured max entries' do
+        expect(subject.dlq_max_entries).to eq(100)
+      end
+
+      it 'returns default max entries when not configured' do
+        config[:config].delete('dlq_max_entries')
+        expect(subject.dlq_max_entries).to eq(10000)
+      end
+    end
+
+    describe '#move_to_dlq' do
+      context 'when DLQ is enabled' do
+        it 'adds entry to DLQ sorted set' do
+          redis_connection_pool.with do |redis_connection|
+            dlq_key = 'vmpooler__dlq__pending'
+            
+            expect(redis_connection).to receive(:zadd).with(dlq_key, anything, anything)
+            expect(redis_connection).to receive(:expire).with(dlq_key, anything)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, 
+                               redis_connection, request_id: request_id, pool_alias: pool_alias)
+          end
+        end
+
+        it 'includes error details in DLQ entry' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).to receive(:zadd) do |_key, _score, entry|
+              expect(entry).to include(vm)
+              expect(entry).to include(error_message)
+              expect(entry).to include(error_class)
+            end
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'increments DLQ metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with('vmpooler_dlq.pending.count')
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'enforces max entries limit' do
+          redis_connection_pool.with do |redis_connection|
+            allow(redis_connection).to receive(:zcard).and_return(150)
+            expect(redis_connection).to receive(:zremrangebyrank).with(anything, 0, 49)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+
+      context 'when DLQ is disabled' do
+        before { config[:config]['dlq_enabled'] = false }
+
+        it 'does not add entry to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).not_to receive(:zadd)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+    end
+  end
+
+  describe 'Auto-Purge' do
+    describe '#purge_enabled?' do
+      it 'returns true when purge_enabled is true in config' do
+        expect(subject.purge_enabled?).to be true
+      end
+
+      it 'returns false when purge_enabled is false in config' do
+        config[:config]['purge_enabled'] = false
+        expect(subject.purge_enabled?).to be false
+      end
+    end
+
+    describe '#purge_dry_run?' do
+      it 'returns false when purge_dry_run is false in config' do
+        expect(subject.purge_dry_run?).to be false
+      end
+
+      it 'returns true when purge_dry_run is true in config' do
+        config[:config]['purge_dry_run'] = true
+        expect(subject.purge_dry_run?).to be true
+      end
+    end
+
+    describe '#max_pending_age' do
+      it 'returns configured max age' do
+        expect(subject.max_pending_age).to eq(7200)
+      end
+
+      it 'returns default max age when not configured' do
+        config[:config].delete('max_pending_age')
+        expect(subject.max_pending_age).to eq(7200)
+      end
+    end
+
+    describe '#purge_pending_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old' }
+      let(:new_vm) { 'vm-new' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (3 hours old, exceeds 2 hour threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'clone', (Time.now - 10800).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      context 'when not in dry-run mode' do
+        it 'purges stale pending VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be false
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", new_vm)).to be true
+          end
+        end
+
+        it 'moves purged VMs to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).to receive(:move_to_dlq).with(
+              old_vm, pool, 'pending', 'Purge', anything, redis_connection, anything
+            )
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+
+        it 'increments purge metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with("vmpooler_purge.pending.#{pool}.count")
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+
+      context 'when in dry-run mode' do
+        before { config[:config]['purge_dry_run'] = true }
+
+        it 'detects but does not purge stale VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be true
+          end
+        end
+
+        it 'does not move to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).not_to receive(:move_to_dlq)
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+    end
+
+    describe '#purge_ready_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-ready' }
+      let(:new_vm) { 'vm-new-ready' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (25 hours old, exceeds 24 hour threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'ready', (Time.now - 90000).to_s)
+          
+          # New VM (2 hours old, within threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'ready', (Time.now - 7200).to_s)
+        end
+      end
+
+      it 'moves stale ready VMs to completed queue' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_ready_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be true
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+
+    describe '#purge_completed_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-completed' }
+      let(:new_vm) { 'vm-new-completed' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (2 hours old, exceeds 1 hour threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'destroy', (Time.now - 7200).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'destroy', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'removes stale completed VMs' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_completed_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+  end
+
+  describe 'Health Checks' do
+    describe '#health_check_enabled?' do
+      it 'returns true when health_check_enabled is true in config' do
+        expect(subject.health_check_enabled?).to be true
+      end
+
+      it 'returns false when health_check_enabled is false in config' do
+        config[:config]['health_check_enabled'] = false
+        expect(subject.health_check_enabled?).to be false
+      end
+    end
+
+    describe '#health_thresholds' do
+      it 'returns configured thresholds' do
+        thresholds = subject.health_thresholds
+        expect(thresholds['pending_queue_max']).to eq(100)
+        expect(thresholds['stuck_vm_age_threshold']).to eq(7200)
+      end
+
+      it 'merges with defaults when partially configured' do
+        config[:config]['health_thresholds'] = { 'pending_queue_max' => 200 }
+        thresholds = subject.health_thresholds
+        
+        expect(thresholds['pending_queue_max']).to eq(200)
+        expect(thresholds['ready_queue_max']).to eq(500) # default
+      end
+    end
+
+    describe '#calculate_queue_ages' do
+      let(:pool) { 'test-pool' }
+      let(:vm1) { 'vm-1' }
+      let(:vm2) { 'vm-2' }
+      let(:vm3) { 'vm-3' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          redis_connection.hset("vmpooler__vm__#{vm1}", 'clone', (Time.now - 3600).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm2}", 'clone', (Time.now - 7200).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm3}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'calculates ages for all VMs' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, vm2, vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(3)
+          expect(ages[0]).to be_within(5).of(3600)
+          expect(ages[1]).to be_within(5).of(7200)
+          expect(ages[2]).to be_within(5).of(1800)
+        end
+      end
+
+      it 'skips VMs with missing timestamps' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, 'vm-nonexistent', vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(2)
+        end
+      end
+    end
+
+    describe '#determine_health_status' do
+      let(:base_metrics) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50 }
+            }
+          },
+          'errors' => {
+            'dlq_total_size' => 50,
+            'stuck_vm_count' => 2
+          }
+        }
+      end
+
+      it 'returns healthy when all metrics are within thresholds' do
+        status = subject.determine_health_status(base_metrics)
+        expect(status).to eq('healthy')
+      end
+
+      it 'returns degraded when DLQ size exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 150
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when DLQ size exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 1500
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns degraded when pending queue exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 120
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when pending queue exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 250
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns unhealthy when stuck VM count exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['stuck_vm_count'] = 60
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+    end
+
+    describe '#push_health_metrics' do
+      let(:metrics_data) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'oldest_age' => 3600, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50, 'oldest_age' => 7200 },
+              'completed' => { 'size' => 5 }
+            }
+          },
+          'tasks' => {
+            'clone' => { 'active' => 3 },
+            'ondemand' => { 'active' => 2, 'pending' => 5 }
+          },
+          'errors' => {
+            'dlq_total_size' => 25,
+            'stuck_vm_count' => 2,
+            'orphaned_metadata_count' => 3
+          }
+        }
+      end
+
+      it 'pushes status metric' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.status', 0)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes error metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.dlq.total_size', 25)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.stuck_vms.count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.orphaned_metadata.count', 3)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes per-pool queue metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.size', 10)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.oldest_age', 3600)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.stuck_count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.ready.size', 50)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes task metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.clone.active', 3)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.active', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.pending', 5)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+    end
+  end
+end