diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..4ae585a
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,13 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
diff --git a/.github/workflows/auto_release_prep.yml b/.github/workflows/auto_release_prep.yml
new file mode 100644
index 0000000..57a12de
--- /dev/null
+++ b/.github/workflows/auto_release_prep.yml
@@ -0,0 +1,12 @@
+name: Automated release prep
+
+on:
+  workflow_dispatch:
+
+jobs:
+  auto_release_prep:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/auto_release_prep.yml@v1
+    secrets: inherit
+    with:
+      project-type: ruby
+      version-file-path: lib/vmpooler/version.rb
diff --git a/.github/workflows/dependabot_merge.yml b/.github/workflows/dependabot_merge.yml
new file mode 100644
index 0000000..75b9cea
--- /dev/null
+++ b/.github/workflows/dependabot_merge.yml
@@ -0,0 +1,8 @@
+name: Dependabot auto-merge
+
+on: pull_request
+
+jobs:
+  dependabot_merge:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/dependabot_merge.yml@v1
+    secrets: inherit
diff --git a/.github/workflows/ensure_label.yml b/.github/workflows/ensure_label.yml
new file mode 100644
index 0000000..50a5fa8
--- /dev/null
+++ b/.github/workflows/ensure_label.yml
@@ -0,0 +1,8 @@
+name: Ensure label
+
+on: pull_request
+
+jobs:
+  ensure_label:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/ensure_label.yml@v1
+    secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..d020d40
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,58 @@
+name: Release Gem
+
+on: workflow_dispatch
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.repository == 'puppetlabs/vmpooler'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Current Version
+        uses: actions/github-script@v7
+        id: cv
+        with:
+          script: |
+            const { data: response } = await github.rest.repos.getLatestRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            console.log(`The latest release is ${response.tag_name}`)
+            return response.tag_name
+          result-encoding: string
+
+      - name: Get Next Version
+        id: nv
+        run: |
+          version=$(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev)
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "Found version $version from lib/vmpooler/version.rb"
+
+      - name: Tag Release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.nv.outputs.version }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          bodyfile: release-notes.md
+          draft: false
+          prerelease: false
+
+      # This step should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+      - name: Install Ruby jruby-9.4.12.1
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 'jruby-9.4.12.1'
+
+      - name: Build gem
+        run: gem build *.gemspec
+
+      - name: Publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: '${{ secrets.RUBYGEMS_AUTH_TOKEN }}'
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
new file mode 100644
index 0000000..ba273f5
--- /dev/null
+++ b/.github/workflows/security.yml
@@ -0,0 +1,39 @@
+name: Security
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  scan:
+    name: Mend Scanning
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout repo content
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - name: setup ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    # setup a package lock if one doesn't exist, otherwise do nothing
+    - name: check lock
+      run: '[ -f "Gemfile.lock" ] && echo "package lock file exists, skipping" || bundle lock'
+    # install java
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+    # download mend
+    - name: download_mend
+      run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
+    - name: run mend
+      run: java -jar wss-unified-agent.jar
+      env:
+        WS_APIKEY: ${{ secrets.MEND_API_KEY }}
+        WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
+        WS_USERKEY: ${{ secrets.MEND_TOKEN }}
+        WS_PRODUCTNAME: RE
+        WS_PROJECTNAME: ${{ github.event.repository.name }}
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
new file mode 100644
index 0000000..d93859a
--- /dev/null
+++ b/.github/workflows/testing.yml
@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run Rubocop
+      run: bundle exec rake rubocop
+
+  spec_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run spec tests
+      run: bundle exec rake test
diff --git a/.github_changelog_generator b/.github_changelog_generator
new file mode 100644
index 0000000..ebeb260
--- /dev/null
+++ b/.github_changelog_generator
@@ -0,0 +1,5 @@
+project=vmpooler
+user=puppetlabs
+exclude_labels=maintenance
+github-api=https://api.github.com
+release-branch=main
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0e49c05..700a888 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
+.bundle/
 .vagrant/
-results.xml
+coverage/
+vendor/
+.dccache
 .ruby-version
-Gemfile.lock
 Gemfile.local
-vendor
+results.xml
 /vmpooler.yaml
-.bundle
-coverage
diff --git a/.rubocop.yml b/.rubocop.yml
index 6c94c01..0fe1eff 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -17,7 +17,7 @@ Style/Documentation:
   Enabled: false
 
 # Line length is not useful
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: false
 
 # Empty method definitions over more than one line is ok
@@ -51,6 +51,10 @@ Metrics/ModuleLength:
 Style/WordArray:
   Enabled: false
 
+# RedundantBegin is causing lib/pool_manager & vsphere.rb to fail in Ruby 2.5+
+Style/RedundantBegin:
+  Enabled: false
+
 # Either sytnax for regex is ok
 Style/RegexpLiteral:
   Enabled: false
@@ -58,10 +62,11 @@ Style/RegexpLiteral:
 # In some cases readability is better without these cops enabled
 Style/ConditionalAssignment:
   Enabled: false
-Next:
+Style/Next:
   Enabled: false
 Metrics/ParameterLists:
   Max: 10
+  MaxOptionalParameters: 10
 Style/GuardClause:
   Enabled: false
 
@@ -69,3 +74,28 @@ Style/GuardClause:
 Layout/EndOfLine:
   EnforcedStyle: lf
 
+# Added in 0.80, don't really care about the change
+Style/HashEachMethods:
+  Enabled: false
+
+# Added in 0.80, don't really care about the change
+Style/HashTransformKeys:
+  Enabled: false
+
+# Added in 0.80, don't really care about the change
+Style/HashTransformValues:
+  Enabled: false
+
+# These short variable names make sense as exceptions to the rule, but generally I think short variable names do hurt readability
+Naming/MethodParameterName:
+  AllowedNames:
+    - vm
+    - dc
+    - s
+    - x
+    - f
+
+# Standard comparisons seem more readable
+Style/NumericPredicate:
+  Enabled: true
+  EnforcedStyle: comparison
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
index 3d824d7..7046eb6 100644
--- a/.rubocop_todo.yml
+++ b/.rubocop_todo.yml
@@ -10,9 +10,9 @@
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
 # SupportedStyles: with_first_parameter, with_fixed_indentation
-Layout/AlignParameters:
+Layout/ParameterAlignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 9
 # Cop supports --auto-correct.
@@ -21,13 +21,13 @@ Layout/AlignParameters:
 Layout/CaseIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 Layout/ClosingParenthesisIndentation:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -55,17 +55,17 @@ Layout/EmptyLinesAroundModuleBody:
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
 # SupportedStyles: special_inside_parentheses, consistent, align_braces
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: Width, IgnoredPatterns.
 Layout/IndentationWidth:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -73,7 +73,7 @@ Layout/IndentationWidth:
 # SupportedStyles: symmetrical, new_line, same_line
 Layout/MultilineMethodCallBraceLayout:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -87,20 +87,14 @@ Layout/SpaceAroundEqualsInParameterDefault:
 # Cop supports --auto-correct.
 Layout/SpaceAroundKeyword:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowForAlignment.
 Layout/SpaceAroundOperators:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-Layout/SpaceInsideBrackets:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 8
 # Cop supports --auto-correct.
@@ -115,17 +109,17 @@ Layout/SpaceInsideHashLiteralBraces:
 # Cop supports --auto-correct.
 Layout/SpaceInsideParens:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Configuration parameters: AllowSafeAssignment.
 Lint/AssignmentInCondition:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
-Lint/HandleExceptions:
+Lint/SuppressedException:
   Exclude:
     - 'lib/vmpooler/api/dashboard.rb'
 
@@ -147,12 +141,6 @@ Lint/UselessAssignment:
     - 'lib/vmpooler/api/dashboard.rb'
     - 'lib/vmpooler/api/helpers.rb'
 
-# Offense count: 2
-# Cop supports --auto-correct.
-Performance/RedundantMatch:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
-
 # Offense count: 6
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -160,15 +148,7 @@ Performance/RedundantMatch:
 Style/AndOr:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: braces, no_braces, context_dependent
-Style/BracesAroundHashParameters:
-  Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 Style/CaseEquality:
@@ -189,7 +169,7 @@ Style/For:
 Style/HashSyntax:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
@@ -197,7 +177,7 @@ Style/HashSyntax:
 Style/IfUnlessModifier:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
@@ -205,13 +185,13 @@ Style/IfUnlessModifier:
 # SupportedStyles: both, prefix, postfix
 Style/NegatedIf:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
 Style/Not:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -220,33 +200,33 @@ Style/Not:
 Style/NumericPredicate:
   Exclude:
     - 'spec/**/*'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/ParallelAssignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowSafeAssignment.
 Style/ParenthesesAroundCondition:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/PerlBackrefs:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist.
 # NamePrefix: is_, has_, have_
 # NamePrefixBlacklist: is_, has_, have_
 # NameWhitelist: is_a?
-Style/PredicateName:
+Naming/PredicateName:
   Exclude:
     - 'spec/**/*'
     - 'lib/vmpooler/api/helpers.rb'
@@ -255,7 +235,7 @@ Style/PredicateName:
 # Cop supports --auto-correct.
 Style/RedundantParentheses:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -276,7 +256,7 @@ Style/RedundantSelf:
 # SupportedStyles: single_quotes, double_quotes
 Style/StringLiterals:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -289,9 +269,9 @@ Style/TernaryParentheses:
 # Offense count: 2
 # Configuration parameters: EnforcedStyle, SupportedStyles.
 # SupportedStyles: snake_case, camelCase
-Style/VariableName:
+Naming/VariableName:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 978a549..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-cache: bundler
-sudo: false
-language: ruby
-
-matrix:
-  include:
-    - rvm: 2.4.5
-      env: "CHECK=rubocop"
-
-    - rvm: 2.4.5
-      env: "CHECK=test"
-
-    - rvm: 2.5.3
-      env: "CHECK=test"
-
-    - rvm: jruby-9.2.5.0
-      env: "CHECK=test"
-
-  # Remove the allow_failures section once
-  # Rubocop is required for Travis to pass a build
-  allow_failures:
-    - rvm: 2.4.5
-      env: "CHECK=rubocop"
-
-install:
-  - gem update --system
-  - gem install bundler
-  - bundle --version
-  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
-script:
-  - "bundle exec rake $CHECK"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ef80643..af092e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,128 +1,927 @@
-# Change Log
+# Changelog
 
-All notable changes to this project will be documented in this file.
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
 
-The format is based on
-[Keep a Changelog](http://keepachangelog.com)
-& makes a strong effort to adhere to
-[Semantic Versioning](http://semver.org).
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
 
-Tracking in this Changelog began for this project with the tagging of version 0.1.0.
-If you're looking for changes from before this, refer to the project's
-git logs & PR history.
-# [Unreleased](https://github.com/puppetlabs/vmpooler/compare/0.7.0...master)
+**Implemented enhancements:**
 
-### Fixed
-- Correctly detect create\_linked\_clone on a pool level (POOLER-147)
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-# [0.7.0](https://github.com/puppetlabs/vmpooler/compare/0.6.3...0.7.0)
+**Fixed bugs:**
 
-### Added
-- Add capability to disable linked clones for vsphere provider (POOLER-147)
-- Add running host to VM data returned from /vm/hostname (POOLER-142)
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-# [0.6.3](https://github.com/puppetlabs/vmpooler/compare/0.6.2...0.6.3)
+## [3.7.0](https://github.com/puppetlabs/vmpooler/tree/3.7.0) (2025-06-04)
 
-### Added
-- Add capability to configure pool cluster via config api (POOLER-143)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.6.0...3.7.0)
 
-# [0.6.2](https://github.com/puppetlabs/vmpooler/compare/0.6.1...0.6.2)
+**Implemented enhancements:**
 
-### Added
-- Validate a machine responds to vm\_ready? at checkout (POOLER-140)
+- \(P4DEVOPS-6096\) Include VMs that have been requested but not moved to pending when getting queue metrics [\#681](https://github.com/puppetlabs/vmpooler/pull/681) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump redis from 5.1.0 to 5.2.0 [\#675](https://github.com/puppetlabs/vmpooler/pull/675) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.1.0 to 13.2.1 [\#673](https://github.com/puppetlabs/vmpooler/pull/673) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.8 to 5.1.0 [\#665](https://github.com/puppetlabs/vmpooler/pull/665) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rspec from 3.12.0 to 3.13.0 [\#664](https://github.com/puppetlabs/vmpooler/pull/664) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.1 to 1.4.0 [\#663](https://github.com/puppetlabs/vmpooler/pull/663) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.43.0 to 0.44.0 [\#662](https://github.com/puppetlabs/vmpooler/pull/662) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.41.0 to 0.43.0 [\#658](https://github.com/puppetlabs/vmpooler/pull/658) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.18.0 to 0.19.0 [\#653](https://github.com/puppetlabs/vmpooler/pull/653) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump sinatra from 3.1.0 to 3.2.0 [\#652](https://github.com/puppetlabs/vmpooler/pull/652) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.40.0 to 0.41.0 [\#650](https://github.com/puppetlabs/vmpooler/pull/650) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.37.0 to 0.40.0 [\#643](https://github.com/puppetlabs/vmpooler/pull/643) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.0.6 to 13.1.0 [\#638](https://github.com/puppetlabs/vmpooler/pull/638) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.2 to 1.3.0 [\#635](https://github.com/puppetlabs/vmpooler/pull/635) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-# [0.6.1](https://github.com/puppetlabs/vmpooler/compare/0.6.0...0.6.1)
+**Fixed bugs:**
 
-### Added
-- Vmpooler /status legacy api optimization
+- Bump opentelemetry-sdk from 1.4.0 to 1.4.1 [\#672](https://github.com/puppetlabs/vmpooler/pull/672) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8.1 to 2.2.9 [\#671](https://github.com/puppetlabs/vmpooler/pull/671) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.3.0 to 1.3.1 [\#668](https://github.com/puppetlabs/vmpooler/pull/668) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8 to 2.2.8.1 [\#666](https://github.com/puppetlabs/vmpooler/pull/666) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump concurrent-ruby from 1.2.2 to 1.2.3 [\#660](https://github.com/puppetlabs/vmpooler/pull/660) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.1 to 6.4.2 [\#655](https://github.com/puppetlabs/vmpooler/pull/655) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.0 to 6.4.1 [\#654](https://github.com/puppetlabs/vmpooler/pull/654) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.22.2 to = 0.22.3 [\#646](https://github.com/puppetlabs/vmpooler/pull/646) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-concurrent\_ruby requirement from = 0.21.1 to = 0.21.2 [\#645](https://github.com/puppetlabs/vmpooler/pull/645) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.0 to 1.3.1 [\#642](https://github.com/puppetlabs/vmpooler/pull/642) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump prometheus-client from 4.2.1 to 4.2.2 [\#641](https://github.com/puppetlabs/vmpooler/pull/641) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.7 to 5.0.8 [\#637](https://github.com/puppetlabs/vmpooler/pull/637) ([dependabot[bot]](https://github.com/apps/dependabot))
+- \(RE-15817\) Reword fail warning and get error from redis before generating message [\#633](https://github.com/puppetlabs/vmpooler/pull/633) ([isaac-hammes](https://github.com/isaac-hammes))
 
-# [0.6.0](https://github.com/puppetlabs/vmpooler/compare/0.5.1...0.6.0)
+**Merged pull requests:**
 
-### Fixed
-- Ensure migrations and pending evaluations are processed FIFO (POOLER-141)
+- \(P4DEVOPS-6096\) Fix gems to prevent warnings in logs [\#685](https://github.com/puppetlabs/vmpooler/pull/685) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(maint\) Revert gems to last release [\#683](https://github.com/puppetlabs/vmpooler/pull/683) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump actions/setup-java from 3 to 4 [\#648](https://github.com/puppetlabs/vmpooler/pull/648) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/github-script from 6 to 7 [\#644](https://github.com/puppetlabs/vmpooler/pull/644) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Added
-- Vmpooler pool statistic endpoint optimization
+## [3.6.0](https://github.com/puppetlabs/vmpooler/tree/3.6.0) (2023-10-05)
 
-### Fixed
- - Ensure a checked out VM stays in a queue during checkout (POOLER-140)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.1...3.6.0)
 
-# [0.5.1](https://github.com/puppetlabs/vmpooler/compare/0.5.0...0.5.1)
+**Fixed bugs:**
 
-# [0.5.0](https://github.com/puppetlabs/vmpooler/compare/0.4.0...0.5.0)
+- \(maint\) Fix message for timeout notification. [\#624](https://github.com/puppetlabs/vmpooler/pull/624) ([isaac-hammes](https://github.com/isaac-hammes))
 
-### Fixed
- - Eliminate window for checked out VM to be discovered (POOLER-139)
+**Merged pull requests:**
 
-# [0.4.0](https://github.com/puppetlabs/vmpooler/compare/0.3.0...0.4.0)
+- Bump rubocop from 1.56.3 to 1.56.4 [\#631](https://github.com/puppetlabs/vmpooler/pull/631) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.3.1 to 6.4.0 [\#630](https://github.com/puppetlabs/vmpooler/pull/630) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.2 to 1.56.3 [\#628](https://github.com/puppetlabs/vmpooler/pull/628) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/checkout from 3 to 4 [\#627](https://github.com/puppetlabs/vmpooler/pull/627) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.24.1 to = 0.24.2 [\#626](https://github.com/puppetlabs/vmpooler/pull/626) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.1 to 1.56.2 [\#625](https://github.com/puppetlabs/vmpooler/pull/625) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.0 to 1.56.1 [\#623](https://github.com/puppetlabs/vmpooler/pull/623) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Fixed
- - Improve support for configuration via environment variables (POOLER-137)
- - Support multiple pool backends per alias (POOLER-138)
- - Remove redis server testing requirement
+## [3.5.1](https://github.com/puppetlabs/vmpooler/tree/3.5.1) (2023-08-24)
 
-# [0.3.0](https://github.com/puppetlabs/vmpooler/compare/0.2.2...0.3.0)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.0...3.5.1)
 
-### Fixed
-- Sync pool size before dashboard is displayed (POOLER-132)
-- Remove a failed VM from the ready queue (POOLER-133)
-- Begin checking ready VMs to ensure alive after 1 minute by default
-- Ensure that metric nodes for vm usage stats are consistent
+**Fixed bugs:**
 
-### Added
-- Add capability to ship VM usage metrics (POOLER-134)
+- \(maint\) Fix bugs from redis and timeout notification updates. [\#621](https://github.com/puppetlabs/vmpooler/pull/621) ([isaac-hammes](https://github.com/isaac-hammes))
 
-# [0.2.2](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
+## [3.5.0](https://github.com/puppetlabs/vmpooler/tree/3.5.0) (2023-08-23)
 
-### Fixed
-- Return label used to request VMs when fulfilling VM requests (POOLER-131)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.4.0...3.5.0)
 
-# [0.2.1](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+**Implemented enhancements:**
 
-### Fixed
-- Better handle delta disk creation errors (POOLER-130)
+- Improve LDAP auth [\#616](https://github.com/puppetlabs/vmpooler/issues/616)
+- \(maint\) Raise error when ip address is not given to vm after clone. [\#619](https://github.com/puppetlabs/vmpooler/pull/619) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(POD-8\) Add timeout\_notification config to log warning before vm is destroyed. [\#618](https://github.com/puppetlabs/vmpooler/pull/618) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(RE-15565\) Add ability to use bind\_as with a service account [\#617](https://github.com/puppetlabs/vmpooler/pull/617) ([yachub](https://github.com/yachub))
 
-### Added
-- Re-write check\_pool in pool\_manager to improve readability
-- Add a docker-compose file for testing vmpooler
-- Add capability to weight backends when an alias spans multiple backends (POOLER-129)
+**Merged pull requests:**
 
-# [0.2.0](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+- Bump puma from 6.3.0 to 6.3.1 [\#615](https://github.com/puppetlabs/vmpooler/pull/615) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Fixed
-- (POOLER-128) VM specific mutex objects are not dereferenced when a VM is destroyed
-- A VM that is being destroyed is reported as discovered
+## [3.4.0](https://github.com/puppetlabs/vmpooler/tree/3.4.0) (2023-08-18)
 
-### Added
-- Adds a new mechanism to load providers from any gem or file path
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.3.0...3.4.0)
 
-# [0.1.0](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...master)
+**Implemented enhancements:**
 
-### Fixed
-- Remove unused method `find_pool` and related pending tests
-- Setting `max_tries` results in an infinite loop (POOLER-124)
-- Do not evaluate folders as VMs in `get_pool_vms` (POOLER-40)
-- Expire redis VM key when clone fails (POOLER-31)
-- Remove all usage of propertyCollector
-- Replace `find_vm` search mechanism (POOLER-68)
-- Fix configuration file loading (POOLER-103)
-- Update vulnerable dependencies (POOLER-101)
+- \(POD-10\) Log reason for failed VM checks. [\#611](https://github.com/puppetlabs/vmpooler/pull/611) ([isaac-hammes](https://github.com/isaac-hammes))
 
-### Added
+**Closed issues:**
 
-- Allow API and manager to run separately (POOLER-109)
-- Add configuration API endpoint (POOLER-107)
-- Add option to disable VM hostname mismatch checks
-- Add a gemspec file
-- Add time remaining information (POOLER-81)
-- Ship metrics for clone to ready time (POOLER-34)
-- Reduce duplicate checking of VMs
-- Reduce object lookups when retrieving VMs and folders
-- Optionally create delta disks for pool templates
-- Drop support for any ruby before 2.3
-- Add support for multiple LDAP search base DNs (POOLER-113)
-- Ensure a VM is only destroyed once (POOLER-112)
-- Add support for setting redis server port and password
-- Greatly reduce time it takes to add disks
-- Add Dockerfile that does not bundle redis
-- Add vmpooler.service to support systemd managing the service
+- Log reason connection on port 22 of a failed VM [\#609](https://github.com/puppetlabs/vmpooler/issues/609)
+
+## [3.3.0](https://github.com/puppetlabs/vmpooler/tree/3.3.0) (2023-08-16)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.2.0...3.3.0)
+
+**Closed issues:**
+
+- Redis 5.x Deprecations [\#603](https://github.com/puppetlabs/vmpooler/issues/603)
+
+**Merged pull requests:**
+
+- Update rubocop requirement from ~\> 1.55.1 to ~\> 1.56.0 [\#608](https://github.com/puppetlabs/vmpooler/pull/608) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.2.0](https://github.com/puppetlabs/vmpooler/tree/3.2.0) (2023-08-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.1.0...3.2.0)
+
+**Implemented enhancements:**
+
+- \(maint\) Update opentelemetry gems. [\#606](https://github.com/puppetlabs/vmpooler/pull/606) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump jruby to 9.4.3.0 and bundle update [\#604](https://github.com/puppetlabs/vmpooler/pull/604) ([yachub](https://github.com/yachub))
+
+**Fixed bugs:**
+
+- \(RE-15692\) Do not attempt loading DNS classes if none are defined [\#602](https://github.com/puppetlabs/vmpooler/pull/602) ([yachub](https://github.com/yachub))
+
+**Closed issues:**
+
+- Fix startup error when not using any dns plugins [\#601](https://github.com/puppetlabs/vmpooler/issues/601)
+
+**Merged pull requests:**
+
+- Bump prometheus-client from 4.1.0 to 4.2.1 [\#599](https://github.com/puppetlabs/vmpooler/pull/599) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.54.2 to ~\> 1.55.1 [\#597](https://github.com/puppetlabs/vmpooler/pull/597) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.7 to 2.2.8 [\#594](https://github.com/puppetlabs/vmpooler/pull/594) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.51.0 to ~\> 1.54.2 [\#593](https://github.com/puppetlabs/vmpooler/pull/593) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.2.2 to 6.3.0 [\#586](https://github.com/puppetlabs/vmpooler/pull/586) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump connection\_pool from 2.4.0 to 2.4.1 [\#583](https://github.com/puppetlabs/vmpooler/pull/583) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.50.1 to ~\> 1.51.0 [\#582](https://github.com/puppetlabs/vmpooler/pull/582) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.1 to 1.2.2 [\#581](https://github.com/puppetlabs/vmpooler/pull/581) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.6.4 to 2.2.7 [\#579](https://github.com/puppetlabs/vmpooler/pull/579) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.1.0](https://github.com/puppetlabs/vmpooler/tree/3.1.0) (2023-05-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.0.0...3.1.0)
+
+**Merged pull requests:**
+
+- Bump rubocop from 1.50.1 to 1.50.2 [\#578](https://github.com/puppetlabs/vmpooler/pull/578) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update puma requirement from ~\> 5.0, \>= 5.0.4 to \>= 5.0.4, \< 7 [\#577](https://github.com/puppetlabs/vmpooler/pull/577) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.19.1 to = 0.23.0 [\#576](https://github.com/puppetlabs/vmpooler/pull/576) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Migrate issue management to Jira [\#575](https://github.com/puppetlabs/vmpooler/pull/575) ([yachub](https://github.com/yachub))
+- Bump jruby to 9.4.2.0 [\#574](https://github.com/puppetlabs/vmpooler/pull/574) ([yachub](https://github.com/yachub))
+- Update rubocop requirement from ~\> 1.28.2 to ~\> 1.50.1 [\#573](https://github.com/puppetlabs/vmpooler/pull/573) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update sinatra requirement from ~\> 2.0 to \>= 2, \< 4 [\#572](https://github.com/puppetlabs/vmpooler/pull/572) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.17.1 to 0.18.0 [\#571](https://github.com/puppetlabs/vmpooler/pull/571) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update prometheus-client requirement from ~\> 2.0 to \>= 2, \< 5 [\#566](https://github.com/puppetlabs/vmpooler/pull/566) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack-test from 2.0.2 to 2.1.0 [\#564](https://github.com/puppetlabs/vmpooler/pull/564) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rack requirement from ~\> 2.2 to \>= 2.2, \< 4.0 [\#562](https://github.com/puppetlabs/vmpooler/pull/562) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-exporter-jaeger requirement from = 0.20.1 to = 0.22.0 [\#524](https://github.com/puppetlabs/vmpooler/pull/524) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [3.0.0](https://github.com/puppetlabs/vmpooler/tree/3.0.0) (2023-03-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.5.0...3.0.0)
+
+**Breaking changes:**
+
+- Direct Users to vmpooler-deployment [\#568](https://github.com/puppetlabs/vmpooler/pull/568) ([yachub](https://github.com/yachub))
+- \(RE-15124\) Implement DNS Plugins and Remove api v1 and v2 [\#551](https://github.com/puppetlabs/vmpooler/pull/551) ([yachub](https://github.com/yachub))
+
+## [2.5.0](https://github.com/puppetlabs/vmpooler/tree/2.5.0) (2023-03-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.4.0...2.5.0)
+
+**Implemented enhancements:**
+
+- \(RE-15161\) Use timeout builtin to TCPSocket when opening sockets. [\#555](https://github.com/puppetlabs/vmpooler/pull/555) ([isaac-hammes](https://github.com/isaac-hammes))
+
+**Merged pull requests:**
+
+- Add docs and update actions [\#550](https://github.com/puppetlabs/vmpooler/pull/550) ([yachub](https://github.com/yachub))
+- \(RE-15111\) Migrate Snyk to Mend Scanning [\#546](https://github.com/puppetlabs/vmpooler/pull/546) ([yachub](https://github.com/yachub))
+- \(RE-14811\) Remove DIO as codeowners [\#517](https://github.com/puppetlabs/vmpooler/pull/517) ([yachub](https://github.com/yachub))
+- Add Snyk action and Move to RE org [\#511](https://github.com/puppetlabs/vmpooler/pull/511) ([yachub](https://github.com/yachub))
+- Add release-engineering to codeowners [\#508](https://github.com/puppetlabs/vmpooler/pull/508) ([yachub](https://github.com/yachub))
+- Update docker/Gemfile.lock [\#503](https://github.com/puppetlabs/vmpooler/pull/503) ([yachub](https://github.com/yachub))
+
+## [2.4.0](https://github.com/puppetlabs/vmpooler/tree/2.4.0) (2022-07-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.3.0...2.4.0)
+
+**Merged pull requests:**
+
+- \(maint\) Bump version to 2.4.0 [\#502](https://github.com/puppetlabs/vmpooler/pull/502) ([sbeaulie](https://github.com/sbeaulie))
+- \(bug\) Prevent failing VMs to be retried infinitely \(ondemand\) [\#501](https://github.com/puppetlabs/vmpooler/pull/501) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-3138\) vmpooler v2 api missing vm/hostname [\#500](https://github.com/puppetlabs/vmpooler/pull/500) ([sbeaulie](https://github.com/sbeaulie))
+- Update rubocop requirement from ~\> 1.1.0 to ~\> 1.28.2 [\#499](https://github.com/puppetlabs/vmpooler/pull/499) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.30.0 to 0.31.0 [\#496](https://github.com/puppetlabs/vmpooler/pull/496) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-redis requirement from = 0.21.2 to = 0.21.3 [\#494](https://github.com/puppetlabs/vmpooler/pull/494) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 5.5.2 to 5.6.4 [\#490](https://github.com/puppetlabs/vmpooler/pull/490) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.19.3 to = 0.19.4 [\#478](https://github.com/puppetlabs/vmpooler/pull/478) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [2.3.0](https://github.com/puppetlabs/vmpooler/tree/2.3.0) (2022-04-07)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.2.0...2.3.0)
+
+**Merged pull requests:**
+
+- \(maint\) Fix deprecation warning for redis ruby library [\#489](https://github.com/puppetlabs/vmpooler/pull/489) ([sbeaulie](https://github.com/sbeaulie))
+- Add OTel HttpClient Instrumentation [\#477](https://github.com/puppetlabs/vmpooler/pull/477) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Update dev tooling and related docs [\#476](https://github.com/puppetlabs/vmpooler/pull/476) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Connect domain settings to pools, create v2 API [\#475](https://github.com/puppetlabs/vmpooler/pull/475) ([genebean](https://github.com/genebean))
+
+## [2.2.0](https://github.com/puppetlabs/vmpooler/tree/2.2.0) (2021-12-30)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.1.0...2.2.0)
+
+**Merged pull requests:**
+
+- Bump version to 2.2.0 [\#473](https://github.com/puppetlabs/vmpooler/pull/473) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Fix EXTRA\_CONFIG merge behavior [\#472](https://github.com/puppetlabs/vmpooler/pull/472) ([sbeaulie](https://github.com/sbeaulie))
+- Update to latest OTel gems [\#471](https://github.com/puppetlabs/vmpooler/pull/471) ([genebean](https://github.com/genebean))
+- Add additional data to spans in api/v1.rb [\#400](https://github.com/puppetlabs/vmpooler/pull/400) ([genebean](https://github.com/genebean))
+
+## [2.1.0](https://github.com/puppetlabs/vmpooler/tree/2.1.0) (2021-12-13)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.0.0...2.1.0)
+
+**Merged pull requests:**
+
+- Ensure all configured providers are loaded [\#470](https://github.com/puppetlabs/vmpooler/pull/470) ([genebean](https://github.com/genebean))
+- \(maint\) Adding a provider method tag\_vm\_user [\#469](https://github.com/puppetlabs/vmpooler/pull/469) ([sbeaulie](https://github.com/sbeaulie))
+- Update testing.yml [\#468](https://github.com/puppetlabs/vmpooler/pull/468) ([sbeaulie](https://github.com/sbeaulie))
+- Move vsphere specific methods out of vmpooler [\#467](https://github.com/puppetlabs/vmpooler/pull/467) ([sbeaulie](https://github.com/sbeaulie))
+- Release prep for v2.0.0 [\#465](https://github.com/puppetlabs/vmpooler/pull/465) ([genebean](https://github.com/genebean))
+
+## [2.0.0](https://github.com/puppetlabs/vmpooler/tree/2.0.0) (2021-12-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.3.0...2.0.0)
+
+**Merged pull requests:**
+
+- Use credentials file for Rubygems auth [\#466](https://github.com/puppetlabs/vmpooler/pull/466) ([genebean](https://github.com/genebean))
+- Add Gem release workflow [\#464](https://github.com/puppetlabs/vmpooler/pull/464) ([genebean](https://github.com/genebean))
+- Update icon in the readme to reference this repo [\#463](https://github.com/puppetlabs/vmpooler/pull/463) ([genebean](https://github.com/genebean))
+- \(DIO-2769\) Move vsphere provider to its own gem [\#462](https://github.com/puppetlabs/vmpooler/pull/462) ([genebean](https://github.com/genebean))
+
+## [1.3.0](https://github.com/puppetlabs/vmpooler/tree/1.3.0) (2021-11-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.2.0...1.3.0)
+
+**Merged pull requests:**
+
+- \(DIO-2675\) Undo pool size & template overrides [\#461](https://github.com/puppetlabs/vmpooler/pull/461) ([genebean](https://github.com/genebean))
+- \(DIO-2186\) Token migration [\#460](https://github.com/puppetlabs/vmpooler/pull/460) ([genebean](https://github.com/genebean))
+
+## [1.2.0](https://github.com/puppetlabs/vmpooler/tree/1.2.0) (2021-09-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.2...1.2.0)
+
+**Merged pull requests:**
+
+- \(DIO-2621\) Make LDAP encryption configurable [\#459](https://github.com/puppetlabs/vmpooler/pull/459) ([genebean](https://github.com/genebean))
+
+## [1.1.2](https://github.com/puppetlabs/vmpooler/tree/1.1.2) (2021-08-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.1...1.1.2)
+
+**Merged pull requests:**
+
+- \(DIO-541\) Fix jenkins and user usage metrics [\#458](https://github.com/puppetlabs/vmpooler/pull/458) ([yachub](https://github.com/yachub))
+
+## [1.1.1](https://github.com/puppetlabs/vmpooler/tree/1.1.1) (2021-08-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0...1.1.1)
+
+**Merged pull requests:**
+
+- \(POOLER-198\) Fix otel warning: Bump otel gems to 0.17.0 [\#457](https://github.com/puppetlabs/vmpooler/pull/457) ([yachub](https://github.com/yachub))
+
+## [1.1.0](https://github.com/puppetlabs/vmpooler/tree/1.1.0) (2021-08-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0-rc.1...1.1.0)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#455](https://github.com/puppetlabs/vmpooler/pull/455) ([yachub](https://github.com/yachub))
+
+## [1.1.0-rc.1](https://github.com/puppetlabs/vmpooler/tree/1.1.0-rc.1) (2021-08-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.0.0...1.1.0-rc.1)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#454](https://github.com/puppetlabs/vmpooler/pull/454) ([yachub](https://github.com/yachub))
+- Update OTel gems to 0.15.0 [\#450](https://github.com/puppetlabs/vmpooler/pull/450) ([genebean](https://github.com/genebean))
+- Migrate testing to GH Actions from Travis [\#446](https://github.com/puppetlabs/vmpooler/pull/446) ([genebean](https://github.com/genebean))
+
+## [1.0.0](https://github.com/puppetlabs/vmpooler/tree/1.0.0) (2021-02-02)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.2...1.0.0)
+
+**Merged pull requests:**
+
+- Update OTel gems to 0.13.z [\#447](https://github.com/puppetlabs/vmpooler/pull/447) ([genebean](https://github.com/genebean))
+- \(DIO-1503\) Fix regex for ondemand instances [\#445](https://github.com/puppetlabs/vmpooler/pull/445) ([genebean](https://github.com/genebean))
+- \(maint\) Update lightstep pre-deploy ghaction to v0.2.6 [\#440](https://github.com/puppetlabs/vmpooler/pull/440) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.2](https://github.com/puppetlabs/vmpooler/tree/0.18.2) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.1...0.18.2)
+
+**Merged pull requests:**
+
+- Remove usage of redis multi from api [\#438](https://github.com/puppetlabs/vmpooler/pull/438) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Fix checkout counter allocation [\#437](https://github.com/puppetlabs/vmpooler/pull/437) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.18.1](https://github.com/puppetlabs/vmpooler/tree/0.18.1) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.0...0.18.1)
+
+**Merged pull requests:**
+
+- Update Puma to 5.0.4 from ~4.3 [\#436](https://github.com/puppetlabs/vmpooler/pull/436) ([genebean](https://github.com/genebean))
+- \(MAINT\) Fix checkout counter allocation [\#435](https://github.com/puppetlabs/vmpooler/pull/435) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-193\) Mark checked out VM as active [\#434](https://github.com/puppetlabs/vmpooler/pull/434) ([mattkirby](https://github.com/mattkirby))
+- Update to OTel 0.8.0 [\#432](https://github.com/puppetlabs/vmpooler/pull/432) ([genebean](https://github.com/genebean))
+- \(POOLER-192\) Use Rubocop 1.0 [\#423](https://github.com/puppetlabs/vmpooler/pull/423) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.0](https://github.com/puppetlabs/vmpooler/tree/0.18.0) (2020-10-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.17.0...0.18.0)
+
+**Merged pull requests:**
+
+- \(maint\) Speedup the tagging method [\#422](https://github.com/puppetlabs/vmpooler/pull/422) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-1065\) Add lightstep gh action [\#421](https://github.com/puppetlabs/vmpooler/pull/421) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.17.0](https://github.com/puppetlabs/vmpooler/tree/0.17.0) (2020-10-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.3...0.17.0)
+
+**Merged pull requests:**
+
+- \(DIO-1059\) Optionally add snapshot tuning params at clone time [\#419](https://github.com/puppetlabs/vmpooler/pull/419) ([suckatrash](https://github.com/suckatrash))
+
+## [0.16.3](https://github.com/puppetlabs/vmpooler/tree/0.16.3) (2020-10-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.2...0.16.3)
+
+**Merged pull requests:**
+
+- \(POOLER-191\) Add checking for running instances that are not in active [\#418](https://github.com/puppetlabs/vmpooler/pull/418) ([mattkirby](https://github.com/mattkirby))
+
+## [0.16.2](https://github.com/puppetlabs/vmpooler/tree/0.16.2) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.1...0.16.2)
+
+**Merged pull requests:**
+
+- Bump OTel Sinatra to 0.7.1 [\#417](https://github.com/puppetlabs/vmpooler/pull/417) ([genebean](https://github.com/genebean))
+
+## [0.16.1](https://github.com/puppetlabs/vmpooler/tree/0.16.1) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.0...0.16.1)
+
+## [0.16.0](https://github.com/puppetlabs/vmpooler/tree/0.16.0) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.15.0...0.16.0)
+
+**Merged pull requests:**
+
+- Update to OTel 0.7.0 [\#416](https://github.com/puppetlabs/vmpooler/pull/416) ([genebean](https://github.com/genebean))
+
+## [0.15.0](https://github.com/puppetlabs/vmpooler/tree/0.15.0) (2020-09-30)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.9...0.15.0)
+
+**Merged pull requests:**
+
+- \(maint\) Centralize dependency management in the gemspec [\#407](https://github.com/puppetlabs/vmpooler/pull/407) ([sbeaulie](https://github.com/sbeaulie))
+- \(pooler-180\) Add healthcheck endpoint, spec testing [\#406](https://github.com/puppetlabs/vmpooler/pull/406) ([suckatrash](https://github.com/suckatrash))
+
+## [0.14.9](https://github.com/puppetlabs/vmpooler/tree/0.14.9) (2020-09-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.8...0.14.9)
+
+**Merged pull requests:**
+
+- Adding make to the other two Dockerfiles [\#405](https://github.com/puppetlabs/vmpooler/pull/405) ([genebean](https://github.com/genebean))
+
+## [0.14.8](https://github.com/puppetlabs/vmpooler/tree/0.14.8) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.7...0.14.8)
+
+**Merged pull requests:**
+
+- Fix mixup of gem placement. [\#404](https://github.com/puppetlabs/vmpooler/pull/404) ([genebean](https://github.com/genebean))
+
+## [0.14.7](https://github.com/puppetlabs/vmpooler/tree/0.14.7) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.6...0.14.7)
+
+**Merged pull requests:**
+
+- Add OTel resource detectors [\#401](https://github.com/puppetlabs/vmpooler/pull/401) ([genebean](https://github.com/genebean))
+- Add distributed tracing [\#399](https://github.com/puppetlabs/vmpooler/pull/399) ([genebean](https://github.com/genebean))
+
+## [0.14.6](https://github.com/puppetlabs/vmpooler/tree/0.14.6) (2020-09-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.5...0.14.6)
+
+**Merged pull requests:**
+
+- \(POOLER-184\) Pool manager retry and exit on failure [\#398](https://github.com/puppetlabs/vmpooler/pull/398) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Add promstats component check [\#397](https://github.com/puppetlabs/vmpooler/pull/397) ([rooneyshuman](https://github.com/rooneyshuman))
+- Test vmpooler on latest 2.5 [\#393](https://github.com/puppetlabs/vmpooler/pull/393) ([mattkirby](https://github.com/mattkirby))
+- Update rbvmomi requirement from ~\> 2.1 to \>= 2.1, \< 4.0 [\#391](https://github.com/puppetlabs/vmpooler/pull/391) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [0.14.5](https://github.com/puppetlabs/vmpooler/tree/0.14.5) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.4...0.14.5)
+
+**Merged pull requests:**
+
+- \(MAINT\) Fix Staledns error counter [\#396](https://github.com/puppetlabs/vmpooler/pull/396) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.4](https://github.com/puppetlabs/vmpooler/tree/0.14.4) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.3...0.14.4)
+
+**Merged pull requests:**
+
+- \(MAINT\) Normalise all tokens for stats [\#395](https://github.com/puppetlabs/vmpooler/pull/395) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.3](https://github.com/puppetlabs/vmpooler/tree/0.14.3) (2020-08-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.2...0.14.3)
+
+**Merged pull requests:**
+
+- \(POOLER-186\) Fix template alias evaluation with backend weight of 0 [\#394](https://github.com/puppetlabs/vmpooler/pull/394) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Clarity refactor of Prom Stats code [\#390](https://github.com/puppetlabs/vmpooler/pull/390) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.2](https://github.com/puppetlabs/vmpooler/tree/0.14.2) (2020-08-03)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.1...0.14.2)
+
+**Merged pull requests:**
+
+- Ensure lifetime is set when creating ondemand instances [\#392](https://github.com/puppetlabs/vmpooler/pull/392) ([mattkirby](https://github.com/mattkirby))
+- Fix vmpooler folder purging [\#389](https://github.com/puppetlabs/vmpooler/pull/389) ([mattkirby](https://github.com/mattkirby))
+
+## [0.14.1](https://github.com/puppetlabs/vmpooler/tree/0.14.1) (2020-07-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.0...0.14.1)
+
+**Merged pull requests:**
+
+- Correctly handle multiple pools of same alias in ondemand checkout [\#388](https://github.com/puppetlabs/vmpooler/pull/388) ([mattkirby](https://github.com/mattkirby))
+- Update travis config to remove deprecated style [\#387](https://github.com/puppetlabs/vmpooler/pull/387) ([rooneyshuman](https://github.com/rooneyshuman))
+- Update Dependabot config file [\#386](https://github.com/puppetlabs/vmpooler/pull/386) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.14.0](https://github.com/puppetlabs/vmpooler/tree/0.14.0) (2020-07-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.3...0.14.0)
+
+**Merged pull requests:**
+
+- Add a note on jruby 9.2.11 and redis connection pooling changes [\#384](https://github.com/puppetlabs/vmpooler/pull/384) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-167\) Allow for network configuration at vm clone time [\#382](https://github.com/puppetlabs/vmpooler/pull/382) ([rooneyshuman](https://github.com/rooneyshuman))
+- \(POOLER-160\) Add Prometheus Metrics to vmpooler [\#372](https://github.com/puppetlabs/vmpooler/pull/372) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.13.3](https://github.com/puppetlabs/vmpooler/tree/0.13.3) (2020-06-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.2...0.13.3)
+
+**Merged pull requests:**
+
+- \(POOLER-174\) Reduce duplicate of on demand code introduced in POOLER-158 [\#383](https://github.com/puppetlabs/vmpooler/pull/383) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.2](https://github.com/puppetlabs/vmpooler/tree/0.13.2) (2020-06-05)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.1...0.13.2)
+
+**Merged pull requests:**
+
+- Rescue and warn when graphite connection cannot be opened [\#379](https://github.com/puppetlabs/vmpooler/pull/379) ([mattkirby](https://github.com/mattkirby))
+
+## [0.13.1](https://github.com/puppetlabs/vmpooler/tree/0.13.1) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.0...0.13.1)
+
+**Merged pull requests:**
+
+- \(maint\) Fix merge issue [\#378](https://github.com/puppetlabs/vmpooler/pull/378) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.0](https://github.com/puppetlabs/vmpooler/tree/0.13.0) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.12.0...0.13.0)
+
+**Merged pull requests:**
+
+- \(POOLER-166\) Check for stale dns records [\#377](https://github.com/puppetlabs/vmpooler/pull/377) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-158\) Add support for ondemand provisioning [\#375](https://github.com/puppetlabs/vmpooler/pull/375) ([mattkirby](https://github.com/mattkirby))
+
+## [0.12.0](https://github.com/puppetlabs/vmpooler/tree/0.12.0) (2020-05-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.3...0.12.0)
+
+**Merged pull requests:**
+
+- \(POOLER-171\) Enable support for multiple user objects [\#376](https://github.com/puppetlabs/vmpooler/pull/376) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.11.3](https://github.com/puppetlabs/vmpooler/tree/0.11.3) (2020-04-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.2...0.11.3)
+
+**Merged pull requests:**
+
+- \(DIO-608\) vmpooler SUT handed out multiple times [\#374](https://github.com/puppetlabs/vmpooler/pull/374) ([sbeaulie](https://github.com/sbeaulie))
+- \(MAINT\) Update CODEOWNERS [\#373](https://github.com/puppetlabs/vmpooler/pull/373) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.11.2](https://github.com/puppetlabs/vmpooler/tree/0.11.2) (2020-04-16)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.1...0.11.2)
+
+**Merged pull requests:**
+
+- \(POOLER-161\) Fix extending vm lifetime when max lifetime is set [\#371](https://github.com/puppetlabs/vmpooler/pull/371) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-165\) Fix purge\_unconfigured\_folders [\#370](https://github.com/puppetlabs/vmpooler/pull/370) ([mattkirby](https://github.com/mattkirby))
+- Update rake requirement from ~\> 12.3 to \>= 12.3, \< 14.0 [\#369](https://github.com/puppetlabs/vmpooler/pull/369) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.11.1](https://github.com/puppetlabs/vmpooler/tree/0.11.1) (2020-03-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.0...0.11.1)
+
+**Merged pull requests:**
+
+- Remove providers addition to docker-compose.yml [\#368](https://github.com/puppetlabs/vmpooler/pull/368) ([mattkirby](https://github.com/mattkirby))
+- Add Dependabot to keep gems updated [\#367](https://github.com/puppetlabs/vmpooler/pull/367) ([genebean](https://github.com/genebean))
+- Update gem dependencies to latest versions [\#366](https://github.com/puppetlabs/vmpooler/pull/366) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-156\) Detect redis connection failures [\#365](https://github.com/puppetlabs/vmpooler/pull/365) ([mattkirby](https://github.com/mattkirby))
+- Add a .dockerignore file [\#363](https://github.com/puppetlabs/vmpooler/pull/363) ([mattkirby](https://github.com/mattkirby))
+
+## [0.11.0](https://github.com/puppetlabs/vmpooler/tree/0.11.0) (2020-03-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.3...0.11.0)
+
+**Merged pull requests:**
+
+- Pin to JRuby 9.2.9 in Dockerfiles [\#362](https://github.com/puppetlabs/vmpooler/pull/362) ([highb](https://github.com/highb))
+- Manual Rubocop Fixes [\#361](https://github.com/puppetlabs/vmpooler/pull/361) ([highb](https://github.com/highb))
+- "Unsafe" rubocop fixes [\#360](https://github.com/puppetlabs/vmpooler/pull/360) ([highb](https://github.com/highb))
+- Fix Rubocop "safe" auto-corrections [\#359](https://github.com/puppetlabs/vmpooler/pull/359) ([highb](https://github.com/highb))
+- Remove duplicate of 0.10.2 from CHANGELOG [\#358](https://github.com/puppetlabs/vmpooler/pull/358) ([highb](https://github.com/highb))
+- \(POOLER-157\) Add extra\_config option to vmpooler [\#357](https://github.com/puppetlabs/vmpooler/pull/357) ([mattkirby](https://github.com/mattkirby))
+
+## [0.10.3](https://github.com/puppetlabs/vmpooler/tree/0.10.3) (2020-03-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.2...0.10.3)
+
+**Merged pull requests:**
+
+- Release 0.10.3 [\#356](https://github.com/puppetlabs/vmpooler/pull/356) ([highb](https://github.com/highb))
+- \(POOLER-154\) Delay vm host update until after migration completes [\#355](https://github.com/puppetlabs/vmpooler/pull/355) ([highb](https://github.com/highb))
+
+## [0.10.2](https://github.com/puppetlabs/vmpooler/tree/0.10.2) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.1...0.10.2)
+
+## [0.10.1](https://github.com/puppetlabs/vmpooler/tree/0.10.1) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.0...0.10.1)
+
+## [0.10.0](https://github.com/puppetlabs/vmpooler/tree/0.10.0) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.1...0.10.0)
+
+**Merged pull requests:**
+
+- Update changelog for 0.10.0 release [\#354](https://github.com/puppetlabs/vmpooler/pull/354) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-153\) Add endpoint for resetting a pool [\#353](https://github.com/puppetlabs/vmpooler/pull/353) ([mattkirby](https://github.com/mattkirby))
+
+## [0.9.1](https://github.com/puppetlabs/vmpooler/tree/0.9.1) (2020-01-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.0...0.9.1)
+
+**Merged pull requests:**
+
+- Generate a wider set of legal names [\#351](https://github.com/puppetlabs/vmpooler/pull/351) ([nicklewis](https://github.com/nicklewis))
+
+## [0.9.0](https://github.com/puppetlabs/vmpooler/tree/0.9.0) (2019-12-12)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.2...0.9.0)
+
+**Closed issues:**
+
+- find\_cluster in vsphere\_helper doesn't support host folders [\#205](https://github.com/puppetlabs/vmpooler/issues/205)
+
+**Merged pull requests:**
+
+- \(QENG-7531\) Add Marked as Failed Stat [\#350](https://github.com/puppetlabs/vmpooler/pull/350) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-123\) Implement a max TTL [\#349](https://github.com/puppetlabs/vmpooler/pull/349) ([sbeaulie](https://github.com/sbeaulie))
+- Support nested host folders in find\_cluster\(\) [\#348](https://github.com/puppetlabs/vmpooler/pull/348) ([seanmil](https://github.com/seanmil))
+- Update CHANGELOG for 0.8.2 [\#347](https://github.com/puppetlabs/vmpooler/pull/347) ([highb](https://github.com/highb))
+
+## [0.8.2](https://github.com/puppetlabs/vmpooler/tree/0.8.2) (2019-11-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.1...0.8.2)
+
+**Merged pull requests:**
+
+- Update rubocop configs [\#346](https://github.com/puppetlabs/vmpooler/pull/346) ([highb](https://github.com/highb))
+- \(QENG-7530\) Add check for unique hostnames [\#345](https://github.com/puppetlabs/vmpooler/pull/345) ([highb](https://github.com/highb))
+- \(QENG-7530\) Fix hostname\_shorten regex [\#344](https://github.com/puppetlabs/vmpooler/pull/344) ([highb](https://github.com/highb))
+- Update changelog for 0.8.1 release [\#343](https://github.com/puppetlabs/vmpooler/pull/343) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.1](https://github.com/puppetlabs/vmpooler/tree/0.8.1) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.0...0.8.1)
+
+**Merged pull requests:**
+
+- Add spicy-proton to vmpooler.gemspec [\#342](https://github.com/puppetlabs/vmpooler/pull/342) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.0](https://github.com/puppetlabs/vmpooler/tree/0.8.0) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.2...0.8.0)
+
+**Merged pull requests:**
+
+- \(QENG-7530\) Make VM names more human readable [\#341](https://github.com/puppetlabs/vmpooler/pull/341) ([highb](https://github.com/highb))
+
+## [0.7.2](https://github.com/puppetlabs/vmpooler/tree/0.7.2) (2019-10-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.1...0.7.2)
+
+**Merged pull requests:**
+
+- Simplify declaration of checkoutlock mutex [\#340](https://github.com/puppetlabs/vmpooler/pull/340) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-150\) Synchronize checkout operations for API [\#339](https://github.com/puppetlabs/vmpooler/pull/339) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-148\) Fix undefined variable bug in \_check\_ready\_vm. [\#338](https://github.com/puppetlabs/vmpooler/pull/338) ([quorten](https://github.com/quorten))
+- Add CODEOWNERS file to vmpooler [\#337](https://github.com/puppetlabs/vmpooler/pull/337) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.1](https://github.com/puppetlabs/vmpooler/tree/0.7.1) (2019-08-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.0...0.7.1)
+
+**Merged pull requests:**
+
+- \(POOLER-147\) Fix create\_linked\_clone pool option [\#336](https://github.com/puppetlabs/vmpooler/pull/336) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update changelog for 0.7.0 release [\#335](https://github.com/puppetlabs/vmpooler/pull/335) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.0](https://github.com/puppetlabs/vmpooler/tree/0.7.0) (2019-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.3...0.7.0)
+
+**Merged pull requests:**
+
+- \(POOLER-142\) Add running host to vm API data [\#334](https://github.com/puppetlabs/vmpooler/pull/334) ([mattkirby](https://github.com/mattkirby))
+- Make it possible to disable linked clones [\#333](https://github.com/puppetlabs/vmpooler/pull/333) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.3](https://github.com/puppetlabs/vmpooler/tree/0.6.3) (2019-07-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.2...0.6.3)
+
+**Closed issues:**
+
+- Named snapshots? [\#140](https://github.com/puppetlabs/vmpooler/issues/140)
+
+**Merged pull requests:**
+
+- \(POOLER-143\) Add clone\_target config change to API [\#332](https://github.com/puppetlabs/vmpooler/pull/332) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Update changelog for 0.6.2 [\#331](https://github.com/puppetlabs/vmpooler/pull/331) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.2](https://github.com/puppetlabs/vmpooler/tree/0.6.2) (2019-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.1...0.6.2)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Fix typo in domain [\#330](https://github.com/puppetlabs/vmpooler/pull/330) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-140\) Ensure a VM is alive at checkout [\#329](https://github.com/puppetlabs/vmpooler/pull/329) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.1](https://github.com/puppetlabs/vmpooler/tree/0.6.1) (2019-05-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.0...0.6.1)
+
+**Merged pull requests:**
+
+- Update Changelog ahead of building 0.6.1 [\#328](https://github.com/puppetlabs/vmpooler/pull/328) ([sbeaulie](https://github.com/sbeaulie))
+- Update API.md \[skip ci\] [\#327](https://github.com/puppetlabs/vmpooler/pull/327) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Optimize the status api using redis pipeline [\#326](https://github.com/puppetlabs/vmpooler/pull/326) ([sbeaulie](https://github.com/sbeaulie))
+- Update changelog ahead of 0.6.0 release. [\#325](https://github.com/puppetlabs/vmpooler/pull/325) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.0](https://github.com/puppetlabs/vmpooler/tree/0.6.0) (2019-04-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.1...0.6.0)
+
+**Merged pull requests:**
+
+- \(QENG-7201\) Vmpooler pool statistic endpoint optimization [\#324](https://github.com/puppetlabs/vmpooler/pull/324) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-141\) Fix order of processing migrating and pending queues [\#323](https://github.com/puppetlabs/vmpooler/pull/323) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Add bundler to dockerfile\_local [\#322](https://github.com/puppetlabs/vmpooler/pull/322) ([mattkirby](https://github.com/mattkirby))
+- Update changelog to 0.5.1 [\#321](https://github.com/puppetlabs/vmpooler/pull/321) ([mattkirby](https://github.com/mattkirby))
+
+## [0.5.1](https://github.com/puppetlabs/vmpooler/tree/0.5.1) (2019-04-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.0...0.5.1)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Ensure a running VM stays in a queue [\#320](https://github.com/puppetlabs/vmpooler/pull/320) ([mattkirby](https://github.com/mattkirby))
+- Fix Dockerfile link in readme and add note about http requests for dev [\#316](https://github.com/puppetlabs/vmpooler/pull/316) ([briancain](https://github.com/briancain))
+
+## [0.5.0](https://github.com/puppetlabs/vmpooler/tree/0.5.0) (2019-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.4.0...0.5.0)
+
+**Merged pull requests:**
+
+- \(POOLER-139\) Fix discovering checked out VM [\#318](https://github.com/puppetlabs/vmpooler/pull/318) ([mattkirby](https://github.com/mattkirby))
+
+## [0.4.0](https://github.com/puppetlabs/vmpooler/tree/0.4.0) (2019-02-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.3.0...0.4.0)
+
+**Merged pull requests:**
+
+- \(MAINT\) Update changelog for 0.4.0 release [\#315](https://github.com/puppetlabs/vmpooler/pull/315) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-138\) Support multiple pools per alias [\#314](https://github.com/puppetlabs/vmpooler/pull/314) ([mattkirby](https://github.com/mattkirby))
+- Update dockerfile jruby to 9.2 [\#313](https://github.com/puppetlabs/vmpooler/pull/313) ([mattkirby](https://github.com/mattkirby))
+- Stop testing ruby 2.3.x [\#312](https://github.com/puppetlabs/vmpooler/pull/312) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-137\) Support integer environment variables [\#311](https://github.com/puppetlabs/vmpooler/pull/311) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update travis to test latest ruby [\#309](https://github.com/puppetlabs/vmpooler/pull/309) ([mattkirby](https://github.com/mattkirby))
+
+## [0.3.0](https://github.com/puppetlabs/vmpooler/tree/0.3.0) (2018-12-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.2...0.3.0)
+
+**Merged pull requests:**
+
+- Change version 0.2.2 to 0.3.0 [\#310](https://github.com/puppetlabs/vmpooler/pull/310) ([mattkirby](https://github.com/mattkirby))
+- Ensure nodes are consistent for usage stats [\#308](https://github.com/puppetlabs/vmpooler/pull/308) ([mattkirby](https://github.com/mattkirby))
+- Update changelog for 0.2.3 [\#307](https://github.com/puppetlabs/vmpooler/pull/307) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-134\) Ship VM usage stats [\#306](https://github.com/puppetlabs/vmpooler/pull/306) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-133\) Identify when a ready VM has failed [\#305](https://github.com/puppetlabs/vmpooler/pull/305) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-37\) Document HTTP responses [\#304](https://github.com/puppetlabs/vmpooler/pull/304) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-132\) Sync pool size on dashboard start [\#303](https://github.com/puppetlabs/vmpooler/pull/303) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.2](https://github.com/puppetlabs/vmpooler/tree/0.2.2) (2018-10-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
+
+**Merged pull requests:**
+
+- Update changelog version in preparation for release [\#302](https://github.com/puppetlabs/vmpooler/pull/302) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-131\) Return requested name when getting VMs [\#301](https://github.com/puppetlabs/vmpooler/pull/301) ([mattkirby](https://github.com/mattkirby))
+- Add docker-compose and dockerfile to support it [\#300](https://github.com/puppetlabs/vmpooler/pull/300) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.1](https://github.com/puppetlabs/vmpooler/tree/0.2.1) (2018-09-19)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+
+**Merged pull requests:**
+
+- Bump version for vmpooler in changelog [\#299](https://github.com/puppetlabs/vmpooler/pull/299) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-129\) Allow setting weights for backends [\#298](https://github.com/puppetlabs/vmpooler/pull/298) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-130\) Improve delta disk creation handling [\#297](https://github.com/puppetlabs/vmpooler/pull/297) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-114\) Refactor check\_pool in pool\_manager [\#296](https://github.com/puppetlabs/vmpooler/pull/296) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.0](https://github.com/puppetlabs/vmpooler/tree/0.2.0) (2018-07-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+
+**Closed issues:**
+
+- create release [\#262](https://github.com/puppetlabs/vmpooler/issues/262)
+- Add API to delete a snapshot [\#163](https://github.com/puppetlabs/vmpooler/issues/163)
+
+**Merged pull requests:**
+
+- \(MAINT\) release 0.2.0 [\#294](https://github.com/puppetlabs/vmpooler/pull/294) ([mattkirby](https://github.com/mattkirby))
+- Remove VM from completed only after destroy [\#293](https://github.com/puppetlabs/vmpooler/pull/293) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-128\) Remove references to VM mutex when destroying [\#292](https://github.com/puppetlabs/vmpooler/pull/292) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Document config via environment [\#291](https://github.com/puppetlabs/vmpooler/pull/291) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) change domain to example.com [\#290](https://github.com/puppetlabs/vmpooler/pull/290) ([steveax](https://github.com/steveax))
+- Update entrypoint in dockerfile for vmpooler gem [\#289](https://github.com/puppetlabs/vmpooler/pull/289) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) release 0.1.0 [\#288](https://github.com/puppetlabs/vmpooler/pull/288) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-66\) Purge vms and folders no longer configured [\#274](https://github.com/puppetlabs/vmpooler/pull/274) ([mattkirby](https://github.com/mattkirby))
+- Adds a new mechanism to load providers from any gem or file path automatically [\#263](https://github.com/puppetlabs/vmpooler/pull/263) ([logicminds](https://github.com/logicminds))
+
+## [0.1.0](https://github.com/puppetlabs/vmpooler/tree/0.1.0) (2018-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...0.1.0)
+
+**Closed issues:**
+
+- jruby 1.7.8 does not support safe\_load [\#243](https://github.com/puppetlabs/vmpooler/issues/243)
+- YAML.safe\_load does not work with symbols in config file [\#240](https://github.com/puppetlabs/vmpooler/issues/240)
+- vmpooler fails to fetch vm with dummy provider [\#238](https://github.com/puppetlabs/vmpooler/issues/238)
+- Any interest in VRA7 support? [\#235](https://github.com/puppetlabs/vmpooler/issues/235)
+- Do not have a hardcoded list of VM providers [\#230](https://github.com/puppetlabs/vmpooler/issues/230)
+- Use a dynamic check\_pool period [\#226](https://github.com/puppetlabs/vmpooler/issues/226)
+- vmpooler doesn't seem to recognize ready VMs [\#218](https://github.com/puppetlabs/vmpooler/issues/218)
+- `find\_vmdks` in `vsphere\_helper` should not use `vmdk\_datastore.\_connection` [\#213](https://github.com/puppetlabs/vmpooler/issues/213)
+- `get\_base\_vm\_container\_from` in `vsphere\_helper` ensures the wrong connection [\#212](https://github.com/puppetlabs/vmpooler/issues/212)
+- `close` in vsphere\_helper throws an error if a connection was never made [\#211](https://github.com/puppetlabs/vmpooler/issues/211)
+- `find\_pool` in vsphere\_helper.rb has subtle errors [\#210](https://github.com/puppetlabs/vmpooler/issues/210)
+- `find\_pool` in vsphere\_helper tends to throw instead of returning nil for missing pools [\#209](https://github.com/puppetlabs/vmpooler/issues/209)
+- Vsphere connections are always insecure \(Ignore cert errors\) [\#207](https://github.com/puppetlabs/vmpooler/issues/207)
+- `find\_folder` in vsphere\_helper.rb has subtle errors [\#204](https://github.com/puppetlabs/vmpooler/issues/204)
+- Should not use `abort` in vsphere\_helper [\#203](https://github.com/puppetlabs/vmpooler/issues/203)
+- No reason why get\_snapshot\_list is defined in vsphere\_helper [\#202](https://github.com/puppetlabs/vmpooler/issues/202)
+- Setting max\_tries in configuration results in vSphereHelper going into infinite loop [\#199](https://github.com/puppetlabs/vmpooler/issues/199)
+- "connect.open" metric is doubled up if a connection is broken [\#195](https://github.com/puppetlabs/vmpooler/issues/195)
+- Remove the use of global variables in the vSphere helper [\#194](https://github.com/puppetlabs/vmpooler/issues/194)
+- Should exit Threads cleanly [\#193](https://github.com/puppetlabs/vmpooler/issues/193)
+- check\_ready\_vm unnecessarily calls open\_socket [\#185](https://github.com/puppetlabs/vmpooler/issues/185)
+- Feature Request: Add provider support [\#181](https://github.com/puppetlabs/vmpooler/issues/181)
+- Document all possible HTTP response codes for endpoints [\#166](https://github.com/puppetlabs/vmpooler/issues/166)
+- Add API to clone new VM from existing VM snapshot [\#165](https://github.com/puppetlabs/vmpooler/issues/165)
+- vsphere\_helper.rb: find\_least\_used\_host should warn if no suitable hosts are found [\#164](https://github.com/puppetlabs/vmpooler/issues/164)
+- find\_vm uses just hostname delta, vSphere searchIndex matches on FQDN [\#141](https://github.com/puppetlabs/vmpooler/issues/141)
+- Tagging does not support boolean values [\#135](https://github.com/puppetlabs/vmpooler/issues/135)
+- POST to /api/v1/token returns WEBrick::HTTPStatus::LengthRequired error [\#132](https://github.com/puppetlabs/vmpooler/issues/132)
+- vmpooler throwing exceptions [\#129](https://github.com/puppetlabs/vmpooler/issues/129)
+- NilClass error when running API without Graphite configured [\#81](https://github.com/puppetlabs/vmpooler/issues/81)
+- Manually removing VM's result in state mis-match [\#80](https://github.com/puppetlabs/vmpooler/issues/80)
+- Add support for customization specs [\#79](https://github.com/puppetlabs/vmpooler/issues/79)
+
+**Merged pull requests:**
+
+- \(maint\) Fix vmpooler require in bin/vmpooler [\#287](https://github.com/puppetlabs/vmpooler/pull/287) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Remove ruby 2.2.10 from travis config [\#286](https://github.com/puppetlabs/vmpooler/pull/286) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Add changelog and contributing guidlines [\#285](https://github.com/puppetlabs/vmpooler/pull/285) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Remove find\_pool and update pending tests [\#283](https://github.com/puppetlabs/vmpooler/pull/283) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-109\) Allow API to run independently [\#281](https://github.com/puppetlabs/vmpooler/pull/281) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-81\) Add time remaining information [\#280](https://github.com/puppetlabs/vmpooler/pull/280) ([smcelmurry](https://github.com/smcelmurry))
+- Revert "\(POOLER-81\) Add time\_remaining information" [\#279](https://github.com/puppetlabs/vmpooler/pull/279) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Fix test reference to find\_vm [\#278](https://github.com/puppetlabs/vmpooler/pull/278) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-34\) Ship clone request to ready time to metrics [\#277](https://github.com/puppetlabs/vmpooler/pull/277) ([smcelmurry](https://github.com/smcelmurry))
+- \(POOLER-81\) Add time\_remaining information [\#276](https://github.com/puppetlabs/vmpooler/pull/276) ([smcelmurry](https://github.com/smcelmurry))
+- Add jruby 9.2 to travis testing [\#275](https://github.com/puppetlabs/vmpooler/pull/275) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-124\) Fix evaluation of max\_tries [\#273](https://github.com/puppetlabs/vmpooler/pull/273) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-40\) Do not return folders with get\_pool\_vms [\#272](https://github.com/puppetlabs/vmpooler/pull/272) ([mattkirby](https://github.com/mattkirby))
+- Ensure template deltas are created once [\#271](https://github.com/puppetlabs/vmpooler/pull/271) ([mattkirby](https://github.com/mattkirby))
+- Do not run duplicate instances of inventory check for a pool [\#270](https://github.com/puppetlabs/vmpooler/pull/270) ([mattkirby](https://github.com/mattkirby))
+- Eliminate duplicate VM object lookups where possible [\#269](https://github.com/puppetlabs/vmpooler/pull/269) ([mattkirby](https://github.com/mattkirby))
+- Reduce object lookups for finding folders [\#268](https://github.com/puppetlabs/vmpooler/pull/268) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-113\) Add support for multiple LDAP search bases [\#267](https://github.com/puppetlabs/vmpooler/pull/267) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-31\) Expire redis vm key when clone fails [\#266](https://github.com/puppetlabs/vmpooler/pull/266) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-112\) Ensure a VM is only destroyed once [\#265](https://github.com/puppetlabs/vmpooler/pull/265) ([mattkirby](https://github.com/mattkirby))
+- Adds a gemspec file [\#264](https://github.com/puppetlabs/vmpooler/pull/264) ([logicminds](https://github.com/logicminds))
+- Change default vsphere connection behavior [\#261](https://github.com/puppetlabs/vmpooler/pull/261) ([mattkirby](https://github.com/mattkirby))
+- Remove propertyCollector from add\_disk [\#260](https://github.com/puppetlabs/vmpooler/pull/260) ([mattkirby](https://github.com/mattkirby))
+- Update ruby versions for travis [\#259](https://github.com/puppetlabs/vmpooler/pull/259) ([mattkirby](https://github.com/mattkirby))
+- Update to generic launcher [\#258](https://github.com/puppetlabs/vmpooler/pull/258) ([frozenfoxx](https://github.com/frozenfoxx))
+- Add support for setting redis port and password [\#257](https://github.com/puppetlabs/vmpooler/pull/257) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-107\) Add configuration API endpoint [\#256](https://github.com/puppetlabs/vmpooler/pull/256) ([mattkirby](https://github.com/mattkirby))
+- Create vmpooler.service [\#255](https://github.com/puppetlabs/vmpooler/pull/255) ([frozenfoxx](https://github.com/frozenfoxx))
+- \(POOLER-101\) Update nokogiri and net-ldap [\#254](https://github.com/puppetlabs/vmpooler/pull/254) ([mattkirby](https://github.com/mattkirby))
+- Add dockerfile without redis [\#253](https://github.com/puppetlabs/vmpooler/pull/253) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-103\) Fix configuration file loading [\#252](https://github.com/puppetlabs/vmpooler/pull/252) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-68\) Replace find\_vm search mechanism [\#251](https://github.com/puppetlabs/vmpooler/pull/251) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Add the last boot time for each pool [\#250](https://github.com/puppetlabs/vmpooler/pull/250) ([sbeaulie](https://github.com/sbeaulie))
+- Fix typo in error message [\#249](https://github.com/puppetlabs/vmpooler/pull/249) ([teancom](https://github.com/teancom))
+- Identify when ESXi host quickstats do not return [\#248](https://github.com/puppetlabs/vmpooler/pull/248) ([mattkirby](https://github.com/mattkirby))
+- Update jruby version for travis to 9.1.13.0 [\#247](https://github.com/puppetlabs/vmpooler/pull/247) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-96\) Setting the Rubygems version [\#246](https://github.com/puppetlabs/vmpooler/pull/246) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-93\) Extend API endpoint to provide just what is needed [\#245](https://github.com/puppetlabs/vmpooler/pull/245) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-92\) Add the alias information in the API status page for each… [\#244](https://github.com/puppetlabs/vmpooler/pull/244) ([sbeaulie](https://github.com/sbeaulie))
+- \(QENG-5305\) Improve vmpooler host selection [\#242](https://github.com/puppetlabs/vmpooler/pull/242) ([mattkirby](https://github.com/mattkirby))
+- Allow user to specify a configuration file in VMPOOLER\_CONFIG\_FILE variable [\#241](https://github.com/puppetlabs/vmpooler/pull/241) ([amcdson](https://github.com/amcdson))
+- Fix no implicit conversion to rational from nil [\#239](https://github.com/puppetlabs/vmpooler/pull/239) ([sbeaulie](https://github.com/sbeaulie))
+- Updated Vagrant box and associated docs [\#237](https://github.com/puppetlabs/vmpooler/pull/237) ([genebean](https://github.com/genebean))
+- \(GH-226\) Respond quickly to VMs being consumed [\#236](https://github.com/puppetlabs/vmpooler/pull/236) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-89\) Identify when config issue is present [\#234](https://github.com/puppetlabs/vmpooler/pull/234) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Update template delta script for moved vsphere credentials [\#233](https://github.com/puppetlabs/vmpooler/pull/233) ([ScottGarman](https://github.com/ScottGarman))
+- Fix rubocop [\#232](https://github.com/puppetlabs/vmpooler/pull/232) ([glennsarti](https://github.com/glennsarti))
+- \(GH-230\) Dynamically load VM Providers [\#231](https://github.com/puppetlabs/vmpooler/pull/231) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Remove phantom VMs that are in Redis but don't exist in provider [\#229](https://github.com/puppetlabs/vmpooler/pull/229) ([glennsarti](https://github.com/glennsarti))
+- Update find\_least\_used\_compatible\_host to specify pool [\#228](https://github.com/puppetlabs/vmpooler/pull/228) ([mattkirby](https://github.com/mattkirby))
+- \(GH-226\) Use a dynamic pool\_check loop period [\#227](https://github.com/puppetlabs/vmpooler/pull/227) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Update development documentation [\#225](https://github.com/puppetlabs/vmpooler/pull/225) ([glennsarti](https://github.com/glennsarti))
+- \(GH-213\) Remove use of private \_connection method [\#224](https://github.com/puppetlabs/vmpooler/pull/224) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-83\) Add ability to specify a datacenter for vsphere [\#223](https://github.com/puppetlabs/vmpooler/pull/223) ([glennsarti](https://github.com/glennsarti))
+- Added Vagrant setup and fixed the Dockerfile so it actually works [\#222](https://github.com/puppetlabs/vmpooler/pull/222) ([genebean](https://github.com/genebean))
+- Adding support for multiple vsphere providers [\#221](https://github.com/puppetlabs/vmpooler/pull/221) ([sbeaulie](https://github.com/sbeaulie))
+- Refactor get\_cluster\_host\_utilization method [\#220](https://github.com/puppetlabs/vmpooler/pull/220) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Pin rack to 1.x [\#219](https://github.com/puppetlabs/vmpooler/pull/219) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-72\)\(POOLER-70\)\(POOLER-52\) Move Pool Manager to use the VM Provider [\#216](https://github.com/puppetlabs/vmpooler/pull/216) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Emit console messages when debugging is enabled [\#215](https://github.com/puppetlabs/vmpooler/pull/215) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\)\(POOLER-52\) Create a functional vSphere Provider  [\#214](https://github.com/puppetlabs/vmpooler/pull/214) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop violations  [\#208](https://github.com/puppetlabs/vmpooler/pull/208) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix credentials in vsphere\_helper [\#200](https://github.com/puppetlabs/vmpooler/pull/200) ([glennsarti](https://github.com/glennsarti))
+- Update usage of global variablesin vsphere\_helper [\#198](https://github.com/puppetlabs/vmpooler/pull/198) ([mattkirby](https://github.com/mattkirby))
+- Remove duplicate of metrics.connect.open [\#197](https://github.com/puppetlabs/vmpooler/pull/197) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-73\) Add spec tests for vsphere\_helper [\#196](https://github.com/puppetlabs/vmpooler/pull/196) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop offenses [\#191](https://github.com/puppetlabs/vmpooler/pull/191) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Prepare to refactor VSphere code into a VM Provider [\#190](https://github.com/puppetlabs/vmpooler/pull/190) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Refactor clone\_vm to take pool configuration object [\#189](https://github.com/puppetlabs/vmpooler/pull/189) ([glennsarti](https://github.com/glennsarti))
+- \(GH-185\) Remove unnecessary checks in check\_ready\_vm  [\#188](https://github.com/puppetlabs/vmpooler/pull/188) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Only load rubocop rake tasks if gem is available [\#187](https://github.com/puppetlabs/vmpooler/pull/187) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Add rubocop and allow failures in Travis CI [\#183](https://github.com/puppetlabs/vmpooler/pull/183) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-73\) Update unit tests prior to refactoring [\#182](https://github.com/puppetlabs/vmpooler/pull/182) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-71\) Add dummy authentication provider [\#180](https://github.com/puppetlabs/vmpooler/pull/180) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Enhance VM Pooler developer experience [\#177](https://github.com/puppetlabs/vmpooler/pull/177) ([glennsarti](https://github.com/glennsarti))
+
+
+
+\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 0000000..b47017c
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1,10 @@
+
+# This will cause RE to be assigned review of any opened PRs against
+# the branches containing this file.
+# See https://help.github.com/en/articles/about-code-owners for info on how to
+# take ownership of parts of the code base that should be reviewed by another
+# team.
+
+# RE will be the default owners for everything in the repo.
+*                                     @puppetlabs/release-engineering
+
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5d350eb..edf18f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,9 +1,6 @@
 # How to contribute
 
-Third-party patches are essential for keeping vmpooler great. We want to keep it as easy as possible to contribute changes that
-get things working in your environment. There are a few guidelines that we
-need contributors to follow so that we can have a chance of keeping on
-top of things.
+Third-party patches are essential for keeping VMPooler great. We want to keep it as easy as possible to contribute changes that get things working in your environment. There are a few guidelines that we need contributors to follow so that we can have a chance of keeping on top of things.
 
 ## Getting Started
 
@@ -18,28 +15,25 @@ top of things.
 
 * Create a topic branch from where you want to base your work.
   * This is usually the master branch.
-  * Only target release branches if you are certain your fix must be on that
-    branch.
-  * To quickly create a topic branch based on master; `git checkout -b
-    fix/master/my_contribution master`. Please avoid working directly on the
-    `master` branch.
+  * Only target release branches if you are certain your fix must be on that branch.
+  * To quickly create a topic branch based on master: `git checkout -b fix/master/my_contribution master`. Please avoid working directly on the `master` branch.
 * Make commits of logical units.
 * Check for unnecessary whitespace with `git diff --check` before committing.
 * Make sure your commit messages are in the proper format.
 
-````
-    (POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
+```plain
+(POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
 
-    Without this patch applied the example commit message in the CONTRIBUTING
-    document is not a concrete example.  This is a problem because the
-    contributor is left to imagine what the commit message should look like
-    based on a description rather than an example.  This patch fixes the
-    problem by making the example concrete and imperative.
+Without this patch applied the example commit message in the CONTRIBUTING
+document is not a concrete example.  This is a problem because the
+contributor is left to imagine what the commit message should look like
+based on a description rather than an example.  This patch fixes the
+problem by making the example concrete and imperative.
 
-    The first line is a real life imperative statement with a ticket number
-    from our issue tracker.  The body describes the behavior without the patch,
-    why this is a problem, and how the patch fixes the problem when applied.
-````
+The first line is a real life imperative statement with a ticket number
+from our issue tracker.  The body describes the behavior without the patch,
+why this is a problem, and how the patch fixes the problem when applied.
+```
 
 * Make sure you have added the necessary tests for your changes.
 * Run _all_ the tests to assure nothing else was accidentally broken.
@@ -48,36 +42,32 @@ top of things.
 
 ### Documentation
 
-For changes of a trivial nature to comments and documentation, it is not
-always necessary to create a new ticket in Jira. In this case, it is
-appropriate to start the first line of a commit with '(doc)' instead of
-a ticket number.
+For changes of a trivial nature to comments and documentation, it is not always necessary to create a new ticket in Jira. In this case, it is appropriate to start the first line of a commit with '(doc)' instead of a ticket number.
 
-````
-    (doc) Add documentation commit example to CONTRIBUTING
+```plain
+(doc) Add documentation commit example to CONTRIBUTING
 
-    There is no example for contributing a documentation commit
-    to the Puppet repository. This is a problem because the contributor
-    is left to assume how a commit of this nature may appear.
+There is no example for contributing a documentation commit
+to the Puppet repository. This is a problem because the contributor
+is left to assume how a commit of this nature may appear.
 
-    The first line is a real life imperative statement with '(doc)' in
-    place of what would have been the ticket number in a
-    non-documentation related commit. The body describes the nature of
-    the new documentation or comments added.
-````
+The first line is a real life imperative statement with '(doc)' in
+place of what would have been the ticket number in a
+non-documentation related commit. The body describes the nature of
+the new documentation or comments added.
+```
 
 ## Submitting Changes
 
-* Sign the [Contributor License Agreement](http://links.puppetlabs.com/cla).
+* Sign the Contributor License Agreement.
 * Push your changes to a topic branch in your fork of the repository.
 * Submit a pull request to the repository in the puppetlabs organization.
 * Update your Jira ticket to mark that you have submitted code and are ready for it to be reviewed (Status: Ready for Merge).
   * Include a link to the pull request in the ticket.
-* The Puppet SRE team looks at Pull Requests on a regular basis.
-* After feedback has been given we expect responses within two weeks. After two
-  weeks we may close the pull request if it isn't showing any activity.
+* The Puppet Release Engineering team looks at Pull Requests on a regular basis.
+* After feedback has been given we expect responses within two weeks. After two weeks we may close the pull request if it isn't showing any activity.
 
-# Additional Resources
+## Additional Resources
 
 * [Puppet Labs community guildelines](http://docs.puppetlabs.com/community/community_guidelines.html)
 * [Bug tracker (Jira)](http://tickets.puppetlabs.com)
diff --git a/Gemfile b/Gemfile
index dfa75dd..0313b80 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,43 +1,13 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
-gem 'json', '>= 1.8'
-gem 'pickup', '~> 0.0.11'
-gem 'puma', '~> 3.11'
-gem 'rack', '~> 2.0'
-gem 'rake', '~> 12.3'
-gem 'redis', '~> 4.0'
-gem 'rbvmomi', '~> 1.13'
-gem 'sinatra', '~> 2.0'
-gem 'net-ldap', '~> 0.16'
-gem 'statsd-ruby', '~> 1.4.0', :require => 'statsd'
-gem 'connection_pool', '~> 2.2'
-gem 'nokogiri', '~> 1.8'
-
-group :development do
-  gem 'pry'
-end
-
-# Test deps
-group :test do
-  # required in order for the providers auto detect mechanism to work
-  gem 'vmpooler', path: './'
-  gem 'mock_redis', '>= 0.17.0'
-  gem 'rack-test', '>= 0.6'
-  gem 'rspec', '>= 3.2'
-  gem 'simplecov', '>= 0.11.2'
-  gem 'yarjuf', '>= 2.0'
-  gem 'climate_control', '>= 0.2.0'
-  # Rubocop would be ok jruby but for now we only use it on
-  # MRI or Windows platforms
-  gem "rubocop", :platforms => [:ruby, :x64_mingw]
-end
+gemspec
 
 # Evaluate Gemfile.local if it exists
-if File.exists? "#{__FILE__}.local"
+if File.exist? "#{__FILE__}.local"
   instance_eval(File.read("#{__FILE__}.local"))
 end
 
 # Evaluate ~/.gemfile if it exists
-if File.exists?(File.join(Dir.home, '.gemfile'))
+if File.exist?(File.join(Dir.home, '.gemfile'))
   instance_eval(File.read(File.join(Dir.home, '.gemfile')))
 end
diff --git a/Gemfile.lock b/Gemfile.lock
new file mode 100644
index 0000000..a63b584
--- /dev/null
+++ b/Gemfile.lock
@@ -0,0 +1,219 @@
+PATH
+  remote: .
+  specs:
+    vmpooler (3.8.1)
+      concurrent-ruby (~> 1.1)
+      connection_pool (~> 2.4)
+      deep_merge (~> 1.2)
+      net-ldap (~> 0.16)
+      opentelemetry-exporter-jaeger (= 0.23.0)
+      opentelemetry-instrumentation-concurrent_ruby (= 0.21.1)
+      opentelemetry-instrumentation-http_client (= 0.22.2)
+      opentelemetry-instrumentation-rack (= 0.23.4)
+      opentelemetry-instrumentation-redis (= 0.25.3)
+      opentelemetry-instrumentation-sinatra (= 0.23.2)
+      opentelemetry-resource_detectors (= 0.24.2)
+      opentelemetry-sdk (~> 1.8)
+      pickup (~> 0.0.11)
+      prometheus-client (>= 2, < 5)
+      puma (>= 5.0.4, < 7)
+      rack (>= 2.2, < 4.0)
+      rake (~> 13.0)
+      redis (~> 5.0)
+      sinatra (>= 2, < 4)
+      spicy-proton (~> 2.1)
+      statsd-ruby (~> 1.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    base64 (0.1.2)
+    bindata (2.5.1)
+    builder (3.3.0)
+    climate_control (1.2.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    deep_merge (1.2.2)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    faraday (2.13.1)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
+    ffi (1.17.2-java)
+    google-cloud-env (2.2.1)
+      faraday (>= 1.0, < 3.a)
+    json (2.12.2)
+    json (2.12.2-java)
+    language_server-protocol (3.17.0.5)
+    logger (1.7.0)
+    method_source (1.1.0)
+    mock_redis (0.37.0)
+    mustermann (3.0.3)
+      ruby2_keywords (~> 0.0.1)
+    net-http (0.6.0)
+      uri
+    net-ldap (0.19.0)
+    nio4r (2.7.4)
+    nio4r (2.7.4-java)
+    opentelemetry-api (1.5.0)
+    opentelemetry-common (0.20.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-jaeger (0.23.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+      thrift
+    opentelemetry-instrumentation-base (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-concurrent_ruby (0.21.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http_client (0.22.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rack (0.23.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-redis (0.25.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-sinatra (0.23.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-resource_detectors (0.24.2)
+      google-cloud-env
+      opentelemetry-sdk (~> 1.0)
+    opentelemetry-sdk (1.8.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.11.0)
+      opentelemetry-api (~> 1.0)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
+    pickup (0.0.11)
+    prism (1.4.0)
+    prometheus-client (4.2.4)
+      base64
+    pry (0.15.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry (0.15.2-java)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+      spoon (~> 0.0)
+    puma (6.6.0)
+      nio4r (~> 2.0)
+    puma (6.6.0-java)
+      nio4r (~> 2.0)
+    racc (1.8.1)
+    racc (1.8.1-java)
+    rack (2.2.17)
+    rack-protection (3.2.0)
+      base64 (>= 0.1.0)
+      rack (~> 2.2, >= 2.2.4)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    redis (5.4.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.24.0)
+      connection_pool
+    regexp_parser (2.10.0)
+    rexml (3.4.1)
+    rspec (3.13.1)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.4)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.4)
+    rubocop (1.56.4)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.1)
+    simplecov_json_formatter (0.1.4)
+    sinatra (3.2.0)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.2.0)
+      tilt (~> 2.0)
+    spicy-proton (2.1.15)
+      bindata (~> 2.3)
+    spoon (0.0.6)
+      ffi
+    statsd-ruby (1.5.0)
+    thor (1.3.2)
+    thrift (0.22.0)
+    tilt (2.6.0)
+    unicode-display_width (2.6.0)
+    uri (1.0.3)
+    yarjuf (2.0.0)
+      builder
+      rspec (~> 3)
+
+PLATFORMS
+  arm64-darwin-22
+  arm64-darwin-23
+  arm64-darwin-25
+  universal-java-11
+  universal-java-17
+  x86_64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  climate_control (>= 0.2.0)
+  mock_redis (= 0.37.0)
+  pry
+  rack-test (>= 0.6)
+  rspec (>= 3.2)
+  rubocop (~> 1.56.0)
+  simplecov (>= 0.11.2)
+  thor (~> 1.0, >= 1.0.1)
+  vmpooler!
+  yarjuf (>= 2.0)
+
+BUNDLED WITH
+   2.4.18
diff --git a/PROVIDER_API.md b/PROVIDER_API.md
deleted file mode 100644
index 0210c3b..0000000
--- a/PROVIDER_API.md
+++ /dev/null
@@ -1,83 +0,0 @@
-# Provider API
-
-## Create a new provider gem from scratch
-
-### Requirements
-1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
-2. the main provider code file should be named the same at the name of the provider. ie. (vpshere == lib/vmpooler/providers/vsphere.rb)
-3. The gem must be installed on the same machine as vmpooler
-4. The provider name must be referenced in the vmpooler config file in order for it to be loaded.
-5. Your gem name or repository name should contain vmpooler-<name>-provider so the community can easily search provider plugins
-   for vmpooler.
-### 1. Use bundler to create the provider scaffolding
-
-```
-bundler gem --test=rspec --no-exe --no-ext vmpooler-spoof-provider
-cd vmpooler-providers-spoof/
-mkdir -p ./lib/vmpooler/providers
-cd ./lib/vmpooler/providers
-touch spoof.rb
-
-```
-
-There may be some boilerplate files there were generated, just delete those.
-
-### 2. Create the main provider file
-Ensure the main provider file uses the following code.
-
-
-```ruby
-# lib/vmpooler/providers/spoof.rb
-require 'yaml'
-require 'vmpooler/providers/base'
-
-module Vmpooler
-  class PoolManager
-    class Provider
-      class Spoof < Vmpooler::PoolManager::Provider::Base
-      
-      # at this time it is not documented which methods should be implemented
-      # have a look at the vmpooler/providers/vpshere provider for examples
-
-      end
-
-    end
-  end
-end
-  
-
-```
-
-### 3. Fill out your gemspec
-Ensure you fill out your gemspec file to your specifications.  If you need a dependency please make sure you require them.
-
-`spec.add_dependency "vmware", "~> 1.15"`.
-
-At a minimum you may want to add the vmpooler gem as a dev dependency so you can use it during testing.
-
-`spec.add_dev_dependency "vmpooler", "~> 1.15"`
-
-or in your Gemfile
-
-```ruby
-
-gem 'vmpooler', github: 'puppetlabs/vmpooler'
-```
-
-Also make sure this dependency can be loaded by jruby.  If the dependency cannot be used by jruby don't use it.
-
-### 4. Create some tests
-Your provider code should be tested before releasing.  Copy and refactor some tests from the vmpooler gem under 
-`spec/unit/providers/dummy_spec.rb`
-
-### 5. Publish
-Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
-
-
-## Available Third Party Providers
-Be the first to update this list.  Create a provider today!
-
-
-## Example provider
-You can use the following [repo as an example](https://github.com/logicminds/vmpooler-vsphere-provider) of how to setup your provider gem.
-
diff --git a/README.md b/README.md
index 1a77553..a05bcec 100644
--- a/README.md
+++ b/README.md
@@ -1,36 +1,101 @@
-![vmpooler](https://raw.github.com/sschneid/vmpooler/master/lib/vmpooler/public/img/logo.gif)
+![VMPooler](lib/vmpooler/public/img/logo.png)
 
-# vmpooler
+# VMPooler
 
-vmpooler provides configurable 'pools' of instantly-available (running) virtual machines.
+- [VMPooler](#vmpooler)
+  - [Usage](#usage)
+    - [Migrating to v3](#migrating-to-v3)
+    - [v2.0.0 note](#v200-note)
+  - [Installation](#installation)
+    - [Dependencies](#dependencies)
+      - [Redis](#redis)
+      - [Other gems](#other-gems)
+  - [Configuration](#configuration)
+  - [Components](#components)
+    - [API](#api)
+    - [Dashboard](#dashboard)
+  - [Related tools and resources](#related-tools-and-resources)
+    - [Command-line Utility](#command-line-utility)
+    - [Vagrant plugin](#vagrant-plugin)
+  - [Development](#development)
+    - [docker-compose](#docker-compose)
+    - [Running docker-compose inside Vagrant](#running-docker-compose-inside-vagrant)
+    - [URLs when using docker-compose](#urls-when-using-docker-compose)
+  - [Update the Gemfile Lock](#update-the-gemfile-lock)
+  - [Releasing](#releasing)
+  - [License](#license)
 
+VMPooler provides configurable 'pools' of instantly-available (pre-provisioned) and/or on-demand (provisioned on request) virtual machines.
 
 ## Usage
 
-At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day.  Dynamic cloning of VM templates initially worked fine for this, but added several seconds to each test run and was unable to account for failed clone tasks.  By pushing these operations to a backend service, we were able to both speed up tests and eliminate test failures due to underlying infrastructure failures.
+At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. VMPooler manages the life cycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand.
 
+The recommended method for deploying VMPooler is via [https://github.com/puppetlabs/vmpooler-deployment](vmpooler-deployment).
+
+### Migrating to v3
+
+Starting with the v3.x release, management of DNS records is implemented as DNS plugins, similar to compute providers. This means each pool configuration should be pointing to a configuration object in `:dns_config` to determine it's method of record management.
+
+For those using the global `DOMAIN` environment variable or global `:config.domain` key, this means records were not previously being managed by VMPooler (presumably managed via dynamic dns), so it's value should be moved to `:dns_configs:<INSERT_YOUR_OWN_SYMBOL>:domain` with the value for `dns_class` for the config set to `dynamic-dns`.
+
+For example, the following < v3.x configuration:
+
+```yaml
+:config:
+  domain: 'example.com'
+```
+
+becomes:
+
+```yaml
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+```
+
+Then any pools that should have records created via the dns config above should now reference the named dns config in the `dns_plugin` key:
+
+```yaml
+:pools:
+  - name: 'debian-8-x86_64'
+    dns_plugin: 'example'
+```
+
+For those using the GCE provider, [vmpooler-provider-gce](https://github.com/puppetlabs/vmpooler-provider-gce), as of version 1.x the DNS management has been decoupled. See <https://github.com/puppetlabs/vmpooler-provider-gce#migrating-to-v1>
+
+### v2.0.0 note
+
+As of version 2.0.0, all providers other than the dummy one are now separate gems. Historically the vSphere provider was included within VMPooler itself. That code has been moved to the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository and the `vmpooler-provider-vsphere` gem. To migrate from VMPooler 1.x to 2.0 you will need to ensure that `vmpooler-provider-vsphere` is installed along side the `vmpooler` gem. See the  [Provider API](docs/PROVIDER_API.md) docs for more information.
 
 ## Installation
 
-### Prerequisites
-
-vmpooler is available as a gem
-
-To use the gem `gem install vmpooler`
+The recommended method of installation is via the Helm chart located in [puppetlabs/vmpooler-deployment](https://github.com/puppetlabs/vmpooler-deployment). That repository also provides Docker images of VMPooler.
 
 ### Dependencies
 
-Vmpooler requires a [Redis](http://redis.io/) server. This is the datastore used for vmpooler's inventory and queueing services.
+#### Redis
 
-### Configuration
+VMPooler requires a [Redis](http://redis.io/) server. This is the data store used for VMPooler's inventory and queuing services.
 
-Configuration for vmpooler may be provided via environment variables, or a configuration file.
+#### Other gems
 
-Please see this [configuration](docs/configuration.md) document for more details about configuring vmpooler via environment variables.
+VMPooler itself and the dev environment talked about below require additional Ruby gems to function. You can update the currently required ones for VMPooler by running `./update-gemfile-lock.sh`. The gems for the dev environment can be updated by running `./docker/update-gemfile-lock.sh`. These scripts will utilize the container on the FROM line of the Dockerfile to update the Gemfile.lock in the root of this repo and in the docker folder, respectively.
+
+## Configuration
+
+Configuration for VMPooler may be provided via environment variables, or a configuration file.
+
+The provided configuration defaults are reasonable for  small VMPooler instances with a few pools. If you plan to run a large VMPooler instance it is important to consider configuration values appropriate for the instance of your size in order to avoid starving the provider, or Redis, of connections.
+
+VMPooler uses a connection pool for Redis to improve efficiency and ensure thread safe usage. At Puppet, we run an instance with about 100 pools at any given time. We have to provide it with 200 Redis connections to the Redis connection pool, and a timeout for connections of 40 seconds, to avoid timeouts. Because metrics are generated for connection available and waited, your metrics provider will need to be able to cope with this volume. Prometheus or StatsD is recommended to ensure metrics get delivered reliably.
+
+Please see this [configuration](docs/configuration.md) document for more details about configuring VMPooler via environment variables.
 
 The following YAML configuration sets up two pools, `debian-7-i386` and `debian-7-x86_64`, which contain 5 running VMs each:
 
-```
+```yaml
 ---
 :providers:
   :vsphere:
@@ -63,45 +128,13 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian-
 
 See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for additional configuration options and parameters or for supporting multiple providers.
 
-### Running via Docker
+## Components
 
-A [Dockerfile](/docker/Dockerfile) is included in this repository to allow running vmpooler inside a Docker container. A configuration file can be used via volume mapping, and specifying the destination as the configuration file via environment variables, or the application can be configured with environment variables alone. The Dockerfile provides an entrypoint so you may choose whether to run API, or manager services. The default behavior will run both. To build and run:
-
-```
-docker build -t vmpooler . && docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-```
-
-To run only the API and dashboard
-
-```
-docker run -p 80:4567 -it vmpooler api
-```
-
-To run only the manager component
-
-```
-docker run -it vmpooler manager
-```
-
-### docker-compose
-
-A docker-compose file is provided to support running vmpooler easily via docker-compose.
-
-```
-docker-compose -f docker/docker-compose.yml up
-```
-
-### Running Docker inside Vagrant
-
-A vagrantfile is included in this repository. Please see [vagrant instructions](docs/vagrant.md) for details.
-
-## API and Dashboard
-
-vmpooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
+VMPooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
 
 ### API
 
-vmpooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
+VMPooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
 
 ### Dashboard
 
@@ -111,24 +144,71 @@ A dashboard is provided to offer real-time statistics and historical graphs.  It
 
 [Graphite](http://graphite.wikidot.com/) is required for historical data retrieval.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for details.
 
-## Command-line Utility
+## Related tools and resources
 
-- The [vmpooler_client.py](https://github.com/puppetlabs/vmpooler-client) CLI utility provides easy access to the vmpooler service. The tool is cross-platform and written in Python.
-- [vmfloaty](https://github.com/briancain/vmfloaty) is a ruby based CLI tool and scripting library written in ruby.
+### Command-line Utility
 
-## Vagrant plugin
+- [vmfloaty](https://github.com/puppetlabs/vmfloaty) is a ruby based CLI tool and scripting library. We consider it the primary way for users to interact with VMPooler.
 
-- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler) Use Vagrant to create and manage your vmpooler instances.
+### Vagrant plugin
 
-## Development and further documentation
+- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler): Use Vagrant to create and manage your VMPooler instances.
 
-For more information about setting up a development instance of vmpooler or other subjects, see the [docs/](docs) directory.
+## Development
 
-## Build status
+### docker-compose
 
-[![Build Status](https://travis-ci.org/puppetlabs/vmpooler.png?branch=master)](https://travis-ci.org/puppetlabs/vmpooler)
+A docker-compose file is provided to support running VMPooler and associated tools locally. This is useful for development because your local code is used to build the gem used in the docker-compose environment. The compose environment also pulls in the latest providers via git. Details of this setup are stored in the `docker/` folder.
 
+```bash
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+### Running docker-compose inside Vagrant
+
+A Vagrantfile is included in this repository so as to provide a reproducible development environment.
+
+```bash
+vagrant up
+vagrant ssh
+cd /vagrant
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+The Vagrant environment also contains multiple rubies you can utilize for spec test and the like. You can see a list of the pre-installed ones when you log in as part of the message of the day.
+
+For more information about setting up a development instance of VMPooler or other subjects, see the [docs/](docs) directory.
+
+### URLs when using docker-compose
+
+| Endpoint          | URL                                                                   |
+|-------------------|-----------------------------------------------------------------------|
+| Redis Commander   | [http://localhost:8079](http://localhost:8079)                        |
+| API               | [http://localhost:8080/api/v1]([http://localhost:8080/api/v1)         |
+| Dashboard         | [http://localhost:8080/dashboard/](http://localhost:8080/dashboard/)  |
+| Metrics (API)     | [http://localhost:8080/prometheus]([http://localhost:8080/prometheus) |
+| Metrics (Manager) | [http://localhost:8081/prometheus]([http://localhost:8081/prometheus) |
+| Jaeger            | [http://localhost:8082](http://localhost:8082)                        |
+
+Additionally, the Redis instance can be accessed at `localhost:6379`.
+
+## Update the Gemfile Lock
+
+To update the `Gemfile.lock` run `./update-gemfile-lock`.
+
+Verify, and update if needed, that the docker tag in the script and GitHub action workflows matches what is used in the [vmpooler-deployment Dockerfile](https://github.com/puppetlabs/vmpooler-deployment/blob/main/docker/Dockerfile).
+
+## Releasing
+
+Follow these steps to publish a new GitHub release, and build and push the gem to <https://rubygems.org>.
+
+1. Bump the "VERSION" in `lib/vmpooler/version.rb` appropriately based on changes in `CHANGELOG.md` since the last release.
+2. Run `./release-prep` to update `Gemfile.lock` and `CHANGELOG.md`.
+3. Commit and push changes to a new branch, then open a pull request against `main` and be sure to add the "maintenance" label.
+4. After the pull request is approved and merged, then navigate to Actions --> Release Gem --> run workflow --> Branch: main --> Run workflow.
 
 ## License
 
-vmpooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.
+VMPooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.
diff --git a/Vagrantfile b/Vagrantfile
index e0a9e66..38598e4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -1,8 +1,12 @@
 # vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
 Vagrant.configure("2") do |config|
   config.vm.box = "genebean/centos-7-rvm-multi"
-  config.vm.network "forwarded_port", guest: 4567, host: 4567
-  config.vm.network "forwarded_port", guest: 8080, host: 8080
+  config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose
+  config.vm.network "forwarded_port", guest: 6379, host: 6379 # Redis
+  config.vm.network "forwarded_port", guest: 8079, host: 8079 # Redis Commander
+  config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose
+  config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose
+  config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose
   config.vm.provision "shell", inline: <<-SCRIPT
     mkdir /var/log/vmpooler
     chown vagrant:vagrant /var/log/vmpooler
@@ -11,9 +15,18 @@ Vagrant.configure("2") do |config|
     usermod -aG docker vagrant
     systemctl enable docker
     systemctl start docker
-    docker build -t vmpooler /vagrant
+    curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+    chmod +x /usr/local/bin/docker-compose
+    ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+    docker-compose --version
+    cd /vagrant
+    docker-compose -f docker/docker-compose.yml build
     docker images
-    echo 'To use the container with the dummy provider do this after "vagrant ssh":'
-    echo "docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler"
   SCRIPT
+
+  # config.vm.provider "virtualbox" do |v|
+  #   v.memory = 2048
+  #   v.cpus = 2
+  # end
+  
 end
diff --git a/bin/vmpooler b/bin/vmpooler
index b84a139..3483349 100755
--- a/bin/vmpooler
+++ b/bin/vmpooler
@@ -1,41 +1,59 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'vmpooler'
+require 'vmpooler/version'
 
 config = Vmpooler.config
+logger_file = config[:config]['logfile']
+prefix = config[:config]['prefix']
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
-logger_file = config[:config]['logfile']
+redis_connection_pool_size = config[:redis]['connection_pool_size']
+redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
+redis_reconnect_attempts = config[:redis]['reconnect_attempts']
+tracing_enabled = config[:tracing]['enabled']
+tracing_jaeger_host = config[:tracing]['jaeger_host']
 
-metrics = Vmpooler.new_metrics(config)
+logger = Vmpooler::Logger.new logger_file
+metrics = Vmpooler::Metrics.init(logger, config)
+
+version = Vmpooler::VERSION
+
+startup_args = ARGV
+Vmpooler.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
 
 torun_threads = []
 if ARGV.count == 0
-  torun = ['api', 'manager']
+  torun = %i[api manager]
 else
   torun = []
-  torun << 'api' if ARGV.include? 'api'
-  torun << 'manager' if ARGV.include? 'manager'
+  torun << :api if ARGV.include?('api')
+  torun << :manager if ARGV.include?('manager')
   exit(2) if torun.empty?
 end
 
-if torun.include? 'api'
+if torun.include?(:api)
   api = Thread.new do
-    thr = Vmpooler::API.new
     redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
-    thr.helpers.configure(config, redis, metrics)
-    thr.helpers.execute!
+    Vmpooler::API.execute(torun, config, redis, metrics, logger)
   end
   torun_threads << api
+elsif metrics.respond_to?(:setup_prometheus_metrics)
+  # Run the cut down API - Prometheus Metrics only.
+  prometheus_only_api = Thread.new do
+    Vmpooler::API.execute(torun, config, nil, metrics, logger)
+  end
+  torun_threads << prometheus_only_api
 end
 
-if torun.include? 'manager'
+if torun.include?(:manager)
   manager = Thread.new do
     Vmpooler::PoolManager.new(
       config,
-      Vmpooler.new_logger(logger_file),
-      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      logger,
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics, redis_reconnect_attempts),
       metrics
     ).execute!
   end
@@ -49,6 +67,4 @@ if ENV['VMPOOLER_DEBUG']
   end
 end
 
-torun_threads.each do |th|
-  th.join
-end
+torun_threads.each(&:join)
diff --git a/docker/Dockerfile b/docker/Dockerfile
deleted file mode 100644
index 97c196f..0000000
--- a/docker/Dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2-jdk
-
-ARG vmpooler_version=0.5.0
-
-COPY docker/docker-entrypoint.sh /usr/local/bin/
-
-ENV LOGFILE=/dev/stdout \
-      RACK_ENV=production
-
-RUN gem install vmpooler -v ${vmpooler_version} && \
-      chmod +x /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/docker/Dockerfile-aio b/docker/Dockerfile-aio
deleted file mode 100644
index 7988cb3..0000000
--- a/docker/Dockerfile-aio
+++ /dev/null
@@ -1,32 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2-jdk
-
-RUN mkdir -p /var/lib/vmpooler
-
-WORKDIR /var/lib/vmpooler
-
-ADD Gemfile* /var/lib/vmpooler/
-RUN bundle install --system
-
-RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby
-
-RUN echo "deb http://httpredir.debian.org/debian jessie main" >/etc/apt/sources.list.d/jessie-main.list
-RUN apt-get update && apt-get install -y redis-server && rm -rf /var/lib/apt/lists/*
-
-COPY . /var/lib/vmpooler
-
-ENV VMPOOLER_LOG /var/log/vmpooler.log
-CMD \
-    /etc/init.d/redis-server start \
-    && /var/lib/vmpooler/scripts/vmpooler_init.sh start \
-    && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \
-    tail -f ${VMPOOLER_LOG}
diff --git a/docker/Dockerfile_local b/docker/Dockerfile_local
deleted file mode 100644
index 7a47516..0000000
--- a/docker/Dockerfile_local
+++ /dev/null
@@ -1,21 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2-jdk
-
-COPY docker/docker-entrypoint.sh /usr/local/bin/
-COPY ./ ./
-
-ENV RACK_ENV=production
-
-RUN gem install bundler && bundle install && gem build vmpooler.gemspec && gem install vmpooler*.gem && \
-      chmod +x /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
deleted file mode 100644
index b007866..0000000
--- a/docker/docker-compose.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# For local development run with a dummy provider
-version: '3.2'
-services:
-  vmpooler:
-    build:
-      context: ../
-      dockerfile: docker/Dockerfile_local
-    volumes:
-      - type: bind
-        source: ${PWD}/vmpooler.yaml
-        target: /etc/vmpooler/vmpooler.yaml
-    ports:
-      - "4567:4567"
-    networks:
-      - redis-net
-    environment:
-      - VMPOOLER_DEBUG=true # for use of dummy auth
-      - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
-      - REDIS_SERVER=redislocal
-    image: vmpooler-local
-    depends_on:
-      - redislocal
-  redislocal:
-    image: redis
-    ports:
-      - "6379:6379"
-    networks:
-      - redis-net
-
-networks:
-  redis-net:
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
deleted file mode 100644
index 5da89df..0000000
--- a/docker/docker-entrypoint.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-set -e
-
-set -- vmpooler "$@"
-
-exec "$@"
diff --git a/docs/API.md b/docs/API-v3.md
similarity index 64%
rename from docs/API.md
rename to docs/API-v3.md
index 560d4b1..ec4feaf 100644
--- a/docs/API.md
+++ b/docs/API-v3.md
@@ -6,11 +6,36 @@
 5. [VM snapshots](#vmsnapshots)
 6. [Status and metrics](#statusmetrics)
 7. [Pool configuration](#poolconfig)
+8. [Ondemand VM provisioning](#ondemandvm)
 
 ### API <a name="API"></a>
 
 vmpooler provides a REST API for VM management.  The following examples use `curl` for communication.
 
+## Major change in V3 versus V2
+
+The api/v1 and api/v2 endpoints have been removed. Additionally, the generic api endpoint that reroutes to a versioned endpoint has been removed.
+
+The api/v3 endpoint removes the deprecated "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each pool instead of only a single domain for all pools, or a domain restricted to a particular provider.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will now now need to specify the versioned endpoint (api/v3).
+
+## Major change in V2 versus V1
+
+The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each provider instead of only one.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will still be redirected to v1, but this behavior is notw deprecated and will be changed to v2 in the next major version.
+
+### updavig clients from v1 to v2
+
+Clients need to update their paths to using api/v2 instead of api/v1. Please note the API responses that used to return a "domain" key, will no longer have a separate "domain" key and now return
+the FQDN (includes the domain in the hostname).
+
+One way to support both v1 and v2 in the client logic is to look for a "domain" and append it to the hostname if it exists (existing v1 behavior). If the "domain" key does not exist, you can use the hostname 
+as is since it is a FQDN (v2 behavor).
+
 #### Token operations <a name="token"></a>
 
 Token-based authentication can be used when requesting or modifying VMs.  The `/token` route can be used to create, query, or delete tokens.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations.
@@ -25,7 +50,7 @@ Return codes:
 * 404  when config:auth not found or other error
 
 ```
-$ curl -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -47,7 +72,7 @@ Return codes:
 * 404  when config:auth not found
 
 ```
-$ curl -X POST -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -X POST -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -66,7 +91,7 @@ Return codes:
 * 404  when config:auth or token not found
 
 ```
-$ curl --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 ```
 ```json
 {
@@ -95,7 +120,7 @@ Return codes:
 * 404  when config:auth not found
 
 ```
-$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -114,7 +139,7 @@ Return codes:
 * 200  OK
 
 ```
-$ curl --url vmpooler.example.com/api/v1/vm
+$ curl --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 [
@@ -135,21 +160,20 @@ Return codes:
 * 503  when the vm failed to allocate a vm, or the pool is empty
 
 ```
-$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v1/vm
+$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "o41xtodlvnvu5cw",
-      "khirruvwfjlmx3y"
+      "o41xtodlvnvu5cw.example.com",
+      "khirruvwfjlmx3y.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "y91qbrpbfj6d13q"
-  },
-  "domain": "example.com"
+    "hostname": "y91qbrpbfj6d13q.example.com"
+  }
 }
 ```
 
@@ -165,22 +189,21 @@ Return codes:
 * 503  when the vm failed to allocate a vm, or the pool is empty
 
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
-    "hostname": "fq6qlpjlsskycq6"
-  },
-  "domain": "example.com"
+    "hostname": "fq6qlpjlsskycq6.example.com"
+  }
 }
 ```
 
 Multiple VMs can be requested by using multiple query parameters in the URL:
 
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
 ```
 
 ```json
@@ -188,14 +211,13 @@ $ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debia
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "sc0o4xqtodlul5w",
-      "4m4dkhqiufnjmxy"
+      "sc0o4xqtodlul5w.example.com",
+      "4m4dkhqiufnjmxy.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "zb91y9qbrbf6d3q"
-  },
-  "domain": "example.com"
+    "hostname": "zb91y9qbrbf6d3q.example.com"
+  }
 }
 ```
 
@@ -210,7 +232,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ```
-$ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
+$ curl --url vmpooler.example.com/api/v3/vm/pxpmtoonx7fiqg6
 ```
 ```json
 {
@@ -226,7 +248,6 @@ $ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
       "user": "jdoe"
     },
     "ip": "192.168.0.1",
-    "domain": "example.com",
     "host": "host1.example.com",
     "migrated": "true"
   }
@@ -255,7 +276,7 @@ Return codes:
 * 400  when supplied PUT parameters fail validation
 
 ```
-$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -264,7 +285,7 @@ $ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpj
 ```
 
 ```
-$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -282,7 +303,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ```
-$ curl -X DELETE --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X DELETE --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -302,7 +323,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname or size is not an integer
 
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/disk/8
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/disk/8
 ````
 ````json
 {
@@ -316,7 +337,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -328,8 +349,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "disk": [
       "+8gb"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 
@@ -347,7 +367,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot
 ````
 ````json
 {
@@ -361,7 +381,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -373,8 +393,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "snapshots": [
       "n4eb4kdtp7rwv4x158366vd9jhac8btq"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 ````
@@ -389,7 +408,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname or snapshot is not valid
 
 ````
-$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
+$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
 ````
 ````json
 {
@@ -404,7 +423,7 @@ $ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.ex
 A "live" status endpoint, representing the current state of the service.
 
 ```
-$ curl --url vmpooler.example.com/api/v1/status
+$ curl --url vmpooler.example.com/api/v3/status
 ```
 ```json
 {
@@ -456,7 +475,7 @@ The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "s
 If the query parameter 'view' is provided, it will be used to select which top level
 element to compute and return. Select them by specifying which one you want in a comma 
 separated list.
-For example `vmpooler.example.com/api/v1/status?view=capacity,boot`
+For example `vmpooler.example.com/api/v3/status?view=capacity,boot`
 
 ##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]]
 
@@ -473,7 +492,7 @@ Return codes:
 
 
 ```
-$ curl --url vmpooler.example.com/api/v1/summary
+$ curl --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -565,7 +584,7 @@ $ curl --url vmpooler.example.com/api/v1/summary
 
 
 ```
-$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v1/summary
+$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -629,9 +648,9 @@ $ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api
 ```
 
 You can also query only the specific top level section you want by including it after `summary/`.
-The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v1/summary/boot/`.
+The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v3/summary/boot/`.
 You can further drill-down the data by specifying the second level parameter to query eg
-`vmpooler.example.com/api/v1/summary/tag/created_by`
+`vmpooler.example.com/api/v3/summary/tag/created_by`
 
 ##### GET /poolstat?pool=FOO
 
@@ -643,7 +662,7 @@ Return codes
 * 200  OK 
 
 ```
-$ curl https://vmpooler.example.com/api/v1/poolstat?pool=centos-6-x86_64
+$ curl https://vmpooler.example.com/api/v3/poolstat?pool=centos-6-x86_64
 ```
 ```json
 {
@@ -668,7 +687,7 @@ Return codes
 * 200  OK 
 
 ```
-$ curl https://vmpooler.example.com/api/v1/totalrunning
+$ curl https://vmpooler.example.com/api/v3/totalrunning
 ```
 
 ```json
@@ -690,7 +709,7 @@ Return codes
 * 400  No configuration found  
 
 ```
-$ curl https://vmpooler.example.com/api/v1/config
+$ curl https://vmpooler.example.com/api/v3/config
 ```
 ```json
 {
@@ -734,7 +753,7 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v1/config/poolsize
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v3/config/poolsize
 ```
 ```json
 {
@@ -742,6 +761,28 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","deb
 }
 ```
 
+##### DELETE /config/poolsize/&lt;pool&gt;
+
+Delete an overridden pool size. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - size reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/poolsize/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "pool_size_before_overrides": 2,
+    "pool_size_before_reset": 4
+}
+```
+
 ##### POST /config/pooltemplate
 
 Change the template configured for a pool, and replenish the pool with instances built from the new template.
@@ -766,7 +807,145 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v1/config/pooltemplate
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v3/config/pooltemplate
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### DELETE /config/pooltemplate/&lt;pool&gt;
+
+Delete an overridden pool template. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - template reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/pooltemplate/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2",
+    "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta"
+}
+```
+
+##### POST /poolreset
+
+Clear all pending and ready instances in a pool, and deploy replacements
+
+All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned.
+
+When a pool reset is requested a 201 status will be returned.
+
+A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances.
+
+For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured.
+
+Responses:
+* 201 - Pool reset requested received
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v3/poolreset
+```
+```json
+{
+  "ok": true
+}
+```
+
+#### Ondemand VM operations <a name="ondemandvm"></a>
+
+Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated.
+
+##### POST /ondemandvm
+
+All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned
+
+When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status.
+
+An authentication token is required in order to request instances on demand when authentication is configured.
+
+Responses:
+* 201 - Provisioning request accepted
+* 400 - Payload contains invalid JSON and cannot be parsed
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 403 - Request exceeds the configured per pool maximum
+* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred.
+* 409 - A request of the matching ID has already been created
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v3/ondemandvm
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a"
+}
+```
+
+##### GET /ondemandvm
+
+Get the status of an ondemandvm request that has already been posted.
+
+When the request is ready the ready status will change to 'true'.
+
+The number of instances pending vs ready will be reflected in the API response.
+
+Responses:
+* 200 - The API request was successful and the status is ok
+* 202 - The request is not ready yet
+* 404 - The request can not be found, or an unknown error occurred
+```
+$ curl https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": false,
+  "debian-7-i386": {
+    "ready": "3",
+    "pending": "1"
+  }
+}
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": true,
+  "debian-7-i386": {
+    "hostname": [
+      "vm1",
+      "vm2",
+      "vm3",
+      "vm4"
+    ]
+  }
+}
+```
+
+##### DELETE /ondemandvm
+
+Delete a ondemand request
+
+Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted.
+
+Responses:
+* 200 - The API request was sucessful. A message will indicate if the request has already been deleted.
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 404 - The request can not be found, or an unknown error occurred.
+```
+$ curl -X DELETE https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
 ```
 ```json
 {
diff --git a/docs/PROVIDER_API.md b/docs/PROVIDER_API.md
new file mode 100644
index 0000000..fb0bcdb
--- /dev/null
+++ b/docs/PROVIDER_API.md
@@ -0,0 +1,100 @@
+# Provider API
+
+Providers facilitate VMPooler interacting with some other system that can create virtual machines. A single VMPooler instance can utilize one or more providers and can have multiple instances of the same provider. An example of having multiple instances of the same provider is when you need to interact with multiple vCenters from the same VMPooler instance.
+
+## Known Providers
+
+- `vmpooler-provider-vsphere` provides the ability to use VMware as a source of VMs. Its code can be found in the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository.
+
+Know of others? Please submit a pull request to update this list or reach out to us on the Puppet community Slack.
+
+Want to create a new one? See below!
+
+## Create a new provider gem from scratch
+
+### Requirements
+
+1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
+2. the main provider code file should be named the same at the name of the provider. For example, the `vpshere` provider's main file is `lib/vmpooler/providers/vsphere.rb`.
+3. The gem must be installed on the same machine as VMPooler
+4. The provider name must be referenced in the VMPooler config file in order for it to be loaded.
+5. Your gem name and repository name should be `vmpooler-provider-<provider name>` so the community can easily search provider plugins.
+
+The resulting directory structure should resemble this:
+
+```bash
+lib/
+├── vmpooler/
+│   └── providers/
+│       └── <provider name>.rb
+└── vmpooler-provider-<provider name>/
+    └── version.rb
+```
+
+### 1. Use bundler to create the provider scaffolding
+
+```bash
+bundler gem --test=rspec --no-exe --no-ext vmpooler-provider-spoof
+cd vmpooler-providers-spoof/
+mkdir -p ./lib/vmpooler/providers
+touch ./lib/vmpooler/providers/spoof.rb
+mkdir ./lib/vmpooler-providers-spoof
+touch ./lib/vmpooler-providers-spoof/version.rb
+```
+
+There may be some boilerplate files generated, just delete those.
+
+### 2. Create the main provider file
+
+Ensure the main provider file uses the following code.
+
+```ruby
+# lib/vmpooler/providers/spoof.rb
+require 'yaml'
+require 'vmpooler/providers/base'
+
+module Vmpooler
+  class PoolManager
+    class Provider
+      class Spoof < Vmpooler::PoolManager::Provider::Base
+        # At this time it is not documented which methods should be implemented
+        # have a look at the https://github.com/puppetlabs/vmpooler-provider-vsphere
+        #for an example
+      end
+    end
+  end
+end
+```
+
+### 3. Create the version file
+
+Ensure you have a version file similar this:
+
+```ruby
+# frozen_string_literal: true
+# lib/vmpooler-provider-vsphere/version.rb 
+
+module VmpoolerProviderSpoof
+  VERSION = '1.0.0'
+end
+```
+
+### 4. Fill out your gemspec
+
+Ensure you fill out your gemspec file to your specifications.  If you need a dependency, please make sure you require it.
+
+`spec.add_dependency "foo", "~> 1.15"`.
+
+At a minimum you may want to add the `vmpooler` gem as a dev dependency so you can use it during testing.
+
+`spec.add_dev_dependency "vmpooler", "~> 2.0"`
+
+Also make sure this dependency can be loaded by JRuby.  If the dependency cannot be used by JRuby don't use it.
+
+### 5. Create some tests
+
+Your provider code should be tested before releasing. Copy and refactor some tests from the `vmpooler` gem under `spec/unit/providers/dummy_spec.rb`.
+
+### 6. Publish
+
+Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index de378a1..0000000
--- a/docs/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Documentation for vmpooler
-
-
-* [Setting up a Development Environment](dev-setup.md)
-* [API Documentation](API.md)
diff --git a/docs/configuration.md b/docs/configuration.md
index 3ce13de..560c328 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -19,11 +19,6 @@ Provide the entire configuration as a blob of yaml. Individual parameters passed
 
 Path to a the file to use when loading the vmpooler configuration. This is only evaluated if `VMPOOLER_CONFIG` has not been specified.
 
-### DOMAIN
-
-If set, returns a top-level 'domain' JSON key in POST requests
-(optional)
-
 ### REDIS\_SERVER
 
 The redis server to use for vmpooler.
@@ -74,6 +69,16 @@ The prefix to use while storing Graphite data.
 The TCP port to communicate with the graphite server.
 (optional; default: 2003)
 
+### MAX\_ONDEMAND\_INSTANCES\_PER\_REQUEST
+
+The maximum number of instances any individual ondemand request may contain per pool.
+(default: 10)
+
+### ONDEMAND\_REQUEST\_TTL
+
+The amount of time (in minutes) to give for a ondemand request to be fulfilled before considering it to have failed.
+(default: 5)
+
 ## Manager options <a name="manager"></a>
 
 ### TASK\_LIMIT
@@ -123,6 +128,11 @@ The target cluster VMs are cloned into (host with least VMs chosen)
 How long (in minutes) before marking a clone as 'failed' and retrying.
 (optional; default: 15)
 
+### READY\_TTL
+
+How long (in minutes) a ready VM should stay in the ready queue.
+(default: 60)
+
 ### MAX\_TRIES
 
 Set the max number of times a connection should retry in VM providers. This optional setting allows a user to dial in retry limits to suit your environment.
@@ -130,7 +140,7 @@ Set the max number of times a connection should retry in VM providers. This opti
 
 ### RETRY\_FACTOR
 
-When retrying, each attempt sleeps for the try count * retry_factor.
+When retrying, each attempt sleeps for the try count * retry\_factor.
 Increase this number to lengthen the delay between retry attempts.
 This is particularly useful for instances with a large number of pools
 to prevent a thundering herd when retrying connections.
@@ -155,9 +165,12 @@ This can also be set per pool.
 
 ### PURGE\_UNCONFIGURED\_FOLDERS
 
-Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+Deprecated, see PURGE\_UNCONFIGURED\_RESOURCES
+
+### PURGE\_UNCONFIGURED\_RESOURCES
+
+Enable purging of VMs (and other resources) detected within the provider that are not explicitly configured.
+Implementation is provider-dependent
 When enabled in the global configuration then purging is enabled for all providers
 Expects a boolean value
 (optional; default: false)
@@ -176,6 +189,35 @@ https://jenkins.example.com/job/platform\_puppet-agent-extra\_puppet-agent-integ
 Expects a boolean value
 (optional; default: false)
 
+### REQUEST\_LOGGER
+
+Enable API request logging to the logger
+When enabled all requests to the API are written to the standard logger.
+Expects a boolean value
+(optional; default: false)
+
+### EXTRA\_CONFIG
+
+Specify additional application configuration files
+The argument can accept a full path to a file, or multiple files comma separated.
+Expects a string value
+(optional)
+
+### ONDEMAND\_CLONE\_LIMIT
+
+Maximum number of simultaneous clones to perform for ondemand provisioning requests.
+(default: 10)
+
+### REDIS\_CONNECTION\_POOL\_SIZE
+
+Maximum number of connections to utilize for the redis connection pool.
+(default: 10)
+
+### REDIS\_CONNECTION\_POOL\_TIMEOUT
+
+How long a task should wait (in seconds) for a redis connection when all connections are in use.
+(default: 5)
+
 ## API options <a name="API"></a>
 
 ### AUTH\_PROVIDER
@@ -204,6 +246,18 @@ This can be a string providing a single DN. For multiple DNs please specify the
 The LDAP object-type used to designate a user object.
 (optional)
 
+### LDAP\_SERVICE_ACCOUNT\_HASH
+
+A hash containing the following parameters for a service account to perform the
+initial bind. After the initial bind, then a search query is performed using the
+'base' and 'user_object', then re-binds as the returned user.
+
+- :user_dn: The full distinguished name (DN) of the service account used to bind.
+
+- :password: The password for the service account used to bind.
+
+(optional)
+
 ### SITE\_NAME
 
 The name of your deployment.
@@ -214,3 +268,8 @@ The name of your deployment.
 Enable experimental API capabilities such as changing pool template and size without application restart
 Expects a boolean value
 (optional; default: false)
+
+### MAX\_LIFETIME\_UPPER\_LIMIT
+
+Specify a maximum lifetime that a VM may be extended to in hours.
+(optional)
diff --git a/docs/dev-setup.md b/docs/dev-setup.md
deleted file mode 100644
index 3a128b0..0000000
--- a/docs/dev-setup.md
+++ /dev/null
@@ -1,234 +0,0 @@
-# Setting up a vmpooler development environment
-
-## Requirements
-
-* Supported on OSX, Windows and Linux
-
-* Ruby or JRuby
-
-  Note - Ruby 1.x support will be removed so it is best to use more modern ruby versions
-
-  Note - It is recommended to user Bundler instead of installing gems into the system repository
-
-* A local Redis server
-
-  Either a containerized instance of Redis or a local version is fine.
-
-## Setup source and ruby
-
-* Clone repository, either from your own fork or the original source
-
-* Perform a bundle install
-
-```
-~/ > git clone https://github.com/puppetlabs/vmpooler.git
-Cloning into 'vmpooler'...
-remote: Counting objects: 3411, done.
-...
-
-~/ > cd vmpooler
-
-~/vmpooler/ > bundle install
-Fetching gem metadata from https://rubygems.org/.........
-Fetching version metadata from https://rubygems.org/..
-Resolving dependencies...
-Installing rake 12.0.0
-...
-Bundle complete! 16 Gemfile dependencies, 37 gems now installed.
-```
-
-## Setup environment variables
-
-### `VMPOOLER_DEBUG`
-
-Setting the `VMPOOLER_DEBUG` environment variable will instruct vmpooler to:
-
-* Output log messages to STDOUT
-
-* Allow the use of the dummy authentication method
-
-* Add interrupt traps so you can stop vmpooler when run interactively
-
-Linux, OSX
-```bash
-~/vmpooler/ > export VMPOOLER_DEBUG=true
-```
-
-Windows (PowerShell)
-```powershell
-C:\vmpooler > $ENV:VMPOOLER_DEBUG = 'true'
-```
-
-
-### `VMPOOLER_CONFIG`
-
-When `VMPOOLER_CONFIG` is set, vmpooler will read its configuration from the content of the environment variable.
-
-Note that this variable does not point to a different configuration file, but stores the contents of a configuration file. You may use `VMPOOLER_CONFIG_FILE` instead to specify a filename.
-
-
-### `VMPOOLER_CONFIG_FILE`
-
-When `VMPOOLER_CONFIG_FILE` is set, vmpooler will read its configuration from the file specified in the environment variable.
-
-Note that this variable points to a different configuration file, unlike `VMPOOLER_CONFIG`.
-
-
-## Setup vmpooler Configuration
-
-You can create a `vmpooler.yaml` file, set the `VMPOOLER_CONFIG` environment variable with the equivalent content, or set the `VMPOOLER_CONFIG_FILE` environment variable with the name of another configuration file to use. `VMPOOLER_CONFIG` takes precedence over `VMPOOLER_CONFIG_FILE`.
-
-Example minimal configuration file:
-```yaml
-
----
-:providers:
-  :dummy:
-
-:redis:
-  server: 'localhost'
-
-:auth:
-   provider: dummy
-
-:tagfilter:
-  url: '(.*)\/'
-
-:config:
-  site_name: 'vmpooler'
-  # Need to change this on Windows
-  logfile: '/var/log/vmpooler.log'
-  task_limit: 10
-  timeout: 15
-  vm_lifetime: 12
-  vm_lifetime_auth: 24
-  allowed_tags:
-    - 'created_by'
-    - 'project'
-  domain: 'example.com'
-  prefix: 'poolvm-'
-
-# Uncomment the lines below to suppress metrics to STDOUT
-# :statsd:
-#   server: 'localhost'
-#   prefix: 'vmpooler'
-#   port: 8125
-
-:pools:
-  - name: 'pool01'
-    size: 5
-    provider: dummy
-  - name: 'pool02'
-    size: 5
-    provider: dummy
-```
-
-## Running vmpooler locally
-
-* Run `bundle exec ruby vmpooler`
-
-  If using JRuby, you may need to use `bundle exec jruby vmpooler`
-
-You should see output similar to:
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-[2017-06-16 14:50:31] starting vmpooler
-[2017-06-16 14:50:31] [!] Creating provider 'dummy'
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection pool of size 1 with timeout 10
-[2017-06-16 14:50:31] [*] [disk_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [snapshot_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [pool01] starting worker thread
-[2017-06-16 14:50:31] [*] [pool02] starting worker thread
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection object ID 1784
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-[2017-06-16 14:50:31] [!] [pool02] is empty
-[2017-06-16 14:50:31] [!] [pool01] is empty
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nexs1w50m4djap5'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-r543eibo4b6tjer'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-neqmu7wj7aukyjy'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nsdnrhhy22lnemo'
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-r543eibo4b6tjer' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-neqmu7wj7aukyjy' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool02] 'poolvm-nexs1w50m4djap5' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-edzlp954lyiozli'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nb0uci0yrwbxr6x'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-y2yxgnovaneymvy'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nur59d25s1y8jko'
-...
-```
-
-### Common Errors
-
-* Forget to set VMPOOLER_DEBUG environment variable
-
-vmpooler will fail to start with an error similar to below
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-
-~/vmpooler/lib/vmpooler.rb:44:in `config': Dummy authentication should not be used outside of debug mode; please set environment variable VMPOOLER_DEBUG to 'true' if you want to use dummy authentication (RuntimeError)
-        from vmpooler:8:in `<main>'
-...
-```
-
-* Error in vmpooler configuration
-
-If there is an error in the vmpooler configuration file, or any other fatal error in the Pool Manager, vmpooler will appear to be running but no log information is displayed.  This is due to the error not being displayed until you press `Ctrl-C` and then suddenly you can see the cause of the issue.
-
-For example, when running vmpooler on Windows, but with a unix style filename for the vmpooler log
-
-```powershell
-C:\vmpooler > bundle exec ruby vmpooler
-[2017-06-16 14:49:57] starting vmpooler
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-
-# [ NOTHING ELSE IS LOGGED ]
-```
-
-Once `Ctrl-C` is pressed the error is shown
-
-```powershell
-...
-== Sinatra has ended his set (crowd applauds)
-Shutting down.
-C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `initialize': No such file or directory @ rb_sysopen - /var/log/vmpooler.log (Errno::ENOENT)
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/vmpooler/lib/vmpooler/logger.rb:17:in `log'
-        from C:/vmpooler/lib/vmpooler/pool_manager.rb:709:in `execute!'
-        from vmpooler:26:in `block in <main>'
-```
-
-## Default vmpooler URLs
-
-| Endpoint  | URL                                                                  |
-|-----------|----------------------------------------------------------------------|
-| Dashboard | [http://localhost:4567/dashboard/](http://localhost:4567/dashboard/) |
-| API       | [http://localhost:4567/api/v1]([http://localhost:4567/api/v1)        |
-
-## Use the vmpooler API locally
-
-Once a local vmpooler instance is running you can use any tool you need to interact with the API.  The dummy authentication provider will allow a user to connect if the username and password are not the same:
-
-* Authentication is successful for username `Alice` with password `foo`
-
-* Authentication will fail for username `Alice` with password `Alice`
-
-Like normal vmpooler, tokens will be created for the user and can be used for regular vmpooler operations.
diff --git a/docs/vagrant.md b/docs/vagrant.md
deleted file mode 100644
index 58332b9..0000000
--- a/docs/vagrant.md
+++ /dev/null
@@ -1,45 +0,0 @@
-A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer.
-To use it run:
-
-```
-vagrant up
-vagrant ssh
-docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler
-```
-
-To run vmpooler with the example dummy provider you can replace the above docker command with this:
-
-```
-docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler
-```
-
-Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/).
-
-### Running directly in Vagrant
-
-You can also run vmpooler directly in the Vagrant box. To do so run this:
-
-```
-vagrant up
-vagrant ssh
-cd /vagrant
-
-# Do this if using the dummy provider
-export VMPOOLER_DEBUG=true
-cp vmpooler.yaml.dummy-example vmpooler.yaml
-
-# vmpooler needs a redis server.
-sudo yum -y install redis
-sudo systemctl start redis
-
-# Optional: Choose your ruby version or use jruby
-# ruby 2.4.x is used by default
-rvm list
-rvm use jruby-9.1.7.0
-
-gem install bundler
-bundle install
-bundle exec ruby vmpooler
-```
-
-When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/).
diff --git a/examples/vmpooler.yaml.dummy-example.aliasedpools b/examples/vmpooler.yaml.dummy-example.aliasedpools
index efe0ce2..55bf9ff 100644
--- a/examples/vmpooler.yaml.dummy-example.aliasedpools
+++ b/examples/vmpooler.yaml.dummy-example.aliasedpools
@@ -17,15 +17,20 @@
   logfile: '/Users/samuel/workspace/vmpooler/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 :pools:
   - name: 'debian-7-i386'
     alias: [ 'debian-7-32' ]
@@ -34,8 +39,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-stringalias'
     alias: 'debian-7-32-stringalias'
     template: 'Templates/debian-7-i386'
@@ -43,8 +50,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64'
     alias: [ 'debian-7-64', 'debian-7-amd64' ]
     template: 'Templates/debian-7-x86_64'
@@ -52,16 +61,20 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-noalias'
     template: 'Templates/debian-7-i386'
     folder: 'Pooled VMs/debian-7-i386'
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64-alias-otherpool-extended'
     alias: [ 'debian-7-x86_64' ]
     template: 'Templates/debian-7-x86_64'
@@ -69,6 +82,7 @@
     datastore: 'other-vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
-
+    dns_plugin: 'example'
diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb
index e94f1c2..2fcde30 100644
--- a/lib/vmpooler.rb
+++ b/lib/vmpooler.rb
@@ -1,10 +1,13 @@
+# frozen_string_literal: true
+
 module Vmpooler
+  require 'concurrent'
   require 'date'
+  require 'deep_merge'
   require 'json'
   require 'net/ldap'
   require 'open-uri'
   require 'pickup'
-  require 'rbvmomi'
   require 'redis'
   require 'set'
   require 'sinatra/base'
@@ -12,7 +15,16 @@ module Vmpooler
   require 'timeout'
   require 'yaml'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
+  # Dependencies for tracing
+  require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-http_client'
+  require 'opentelemetry-instrumentation-redis'
+  require 'opentelemetry-instrumentation-sinatra'
+  require 'opentelemetry-sdk'
+  require 'opentelemetry/exporter/jaeger'
+  require 'opentelemetry/resource/detectors'
+
+  %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
 
@@ -21,111 +33,148 @@ module Vmpooler
     if ENV['VMPOOLER_CONFIG']
       config_string = ENV['VMPOOLER_CONFIG']
       # Parse the YAML config into a Hash
-      # Whitelist the Symbol class
-      parsed_config = YAML.safe_load(config_string, [Symbol])
+      # Allow the Symbol class
+      parsed_config = YAML.safe_load(config_string, permitted_classes: [Symbol])
     else
       # Take the name of the config file either from an ENV variable or from the filepath argument
       config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
       parsed_config = YAML.load_file(config_file) if File.exist? config_file
+      parsed_config[:config]['extra_config'] = ENV['EXTRA_CONFIG'] if ENV['EXTRA_CONFIG']
+      if parsed_config[:config]['extra_config']
+        extra_configs = parsed_config[:config]['extra_config'].split(',')
+        extra_configs.each do |config|
+          puts "loading extra_config file #{config}"
+          extra_config = YAML.load_file(config)
+          parsed_config.deep_merge(extra_config)
+        end
+      end
     end
 
     parsed_config ||= { config: {} }
 
     # Bail out if someone attempts to start vmpooler with dummy authentication
     # without enbaling debug mode.
-    if parsed_config.has_key? :auth
-      if parsed_config[:auth]['provider'] == 'dummy'
-        unless ENV['VMPOOLER_DEBUG']
-          warning = [
-            'Dummy authentication should not be used outside of debug mode',
-            'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
-          ]
+    if parsed_config.key?(:auth) && parsed_config[:auth]['provider'] == 'dummy' && !ENV['VMPOOLER_DEBUG']
+      warning = [
+        'Dummy authentication should not be used outside of debug mode',
+        'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
+      ]
 
-          raise warning.join(";\s")
-        end
-      end
+      raise warning.join(";\s")
     end
 
     # Set some configuration defaults
-    parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
-    parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
-    parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
-    parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime']  || 24
-    parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
-
-    parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
-
-    parsed_config[:config]['site_name'] = ENV['SITE_NAME'] if ENV['SITE_NAME']
-    parsed_config[:config]['domain'] = ENV['DOMAIN'] if ENV['DOMAIN']
-    parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
-    parsed_config[:config]['timeout'] = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
-    parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
-    parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
-    parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
-    parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['task_limit']                         = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit']               = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
+    parsed_config[:config]['migration_limit']                    = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
+    parsed_config[:config]['vm_checktime']                       = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
+    parsed_config[:config]['vm_lifetime']                        = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit']           = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl']                          = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl']               = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
+    parsed_config[:config]['prefix']                             = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
+    parsed_config[:config]['logfile']                            = ENV['LOGFILE'] if ENV['LOGFILE']
+    parsed_config[:config]['site_name']                          = ENV['SITE_NAME'] if ENV['SITE_NAME']
+    if !parsed_config[:config]['domain'].nil? || !ENV['DOMAIN'].nil?
+      puts '[!] [error] The "domain" config setting has been removed in v3. Please see the docs for migrating the domain config to use a dns plugin at https://github.com/puppetlabs/vmpooler/blob/main/README.md#migrating-to-v3'
+      exit 1
+    end
+    parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
+    parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['timeout_notification']               = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
+    parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
+    parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
+    parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
+    parsed_config[:config]['create_folders']                     = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['experimental_features']              = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+    parsed_config[:config]['usage_stats']                        = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
+    parsed_config[:config]['request_logger']                     = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
+    parsed_config[:config]['create_template_delta_disks']        = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_RESOURCES'] if ENV['PURGE_UNCONFIGURED_RESOURCES']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    # ENV PURGE_UNCONFIGURED_FOLDERS deprecated, will be removed in version 3
+    puts '[!] [deprecation] rename ENV var \'PURGE_UNCONFIGURED_FOLDERS\' to \'PURGE_UNCONFIGURED_RESOURCES\'' if ENV['PURGE_UNCONFIGURED_FOLDERS']
     set_linked_clone(parsed_config)
-    parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
-    parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
-    parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
 
-    parsed_config[:redis] = parsed_config[:redis] || {}
-    parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
-    parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
-    parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
-    parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]                            = parsed_config[:redis] || {}
+    parsed_config[:redis]['server']                  = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
+    parsed_config[:redis]['port']                    = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
+    parsed_config[:redis]['password']                = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
+    parsed_config[:redis]['data_ttl']                = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size']    = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+    parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
+    parsed_config[:redis]['reconnect_attempts']      = string_array_to_array(ENV['REDIS_RECONNECT_ATTEMPTS']) || parsed_config[:redis]['reconnect_attempts'] || 10
 
-    parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
+    parsed_config[:statsd]           = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
     parsed_config[:statsd]['prefix'] = ENV['STATSD_PREFIX'] if ENV['STATSD_PREFIX']
-    parsed_config[:statsd]['port'] = string_to_int(ENV['STATSD_PORT']) if ENV['STATSD_PORT']
+    parsed_config[:statsd]['port']   = string_to_int(ENV['STATSD_PORT']) if ENV['STATSD_PORT']
 
-    parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]           = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
-    parsed_config[:graphite]['port'] = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
+    parsed_config[:graphite]['port']   = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
+
+    parsed_config[:tracing]                = parsed_config[:tracing] || {}
+    parsed_config[:tracing]['enabled']     = ENV['VMPOOLER_TRACING_ENABLED'] || parsed_config[:tracing]['enabled'] || 'false'
+    parsed_config[:tracing]['jaeger_host'] = ENV['VMPOOLER_TRACING_JAEGER_HOST'] || parsed_config[:tracing]['jaeger_host'] || 'http://localhost:14268/api/traces'
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
-    if parsed_config.has_key? :auth
-      parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
-      parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
-      parsed_config[:auth][:ldap]['host'] = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
-      parsed_config[:auth][:ldap]['port'] = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
-      parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
+    if parsed_config.key? :auth
+      parsed_config[:auth]['provider']           = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
+      parsed_config[:auth][:ldap]                = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
+      parsed_config[:auth][:ldap]['host']        = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
+      parsed_config[:auth][:ldap]['port']        = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
+      parsed_config[:auth][:ldap]['base']        = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
+      if parsed_config[:auth]['provider'] == 'ldap' && parsed_config[:auth][:ldap].key?('encryption')
+        parsed_config[:auth][:ldap]['encryption'] = parsed_config[:auth][:ldap]['encryption']
+      elsif parsed_config[:auth]['provider'] == 'ldap'
+        parsed_config[:auth][:ldap]['encryption'] = {}
+      end
     end
 
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
     unless parsed_config[:pools]
+      puts 'loading pools configuration from redis, since the config[:pools] is empty'
       redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
       parsed_config[:pools] = load_pools_from_redis(redis)
     end
 
+    # Marshal.dump is paired with Marshal.load to create a copy that has its own memory space
+    # so that each can be edited independently
+    # rubocop:disable Security/MarshalLoad
+
+    # retain a copy of the pools that were observed at startup
+    serialized_pools = Marshal.dump(parsed_config[:pools])
+    parsed_config[:pools_at_startup] = Marshal.load(serialized_pools)
+
     # Create an index of pools by title
     parsed_config[:pool_index] = pool_index(parsed_config[:pools])
+    # rubocop:enable Security/MarshalLoad
 
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
+      pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
-        if pool['alias'].is_a?(Array)
+        if pool['alias'].instance_of?(Array)
           pool['alias'].each do |pool_alias|
             parsed_config[:alias] ||= {}
             parsed_config[:alias][pool_alias] = [pool['name']] unless parsed_config[:alias].key? pool_alias
             parsed_config[:alias][pool_alias] << pool['name'] unless parsed_config[:alias][pool_alias].include? pool['name']
             parsed_config[:pool_names] << pool_alias
           end
-        elsif pool['alias'].is_a?(String)
+        elsif pool['alias'].instance_of?(String)
           parsed_config[:alias][pool['alias']] = pool['name']
           parsed_config[:pool_names] << pool['alias']
         end
       end
     end
 
-    if parsed_config[:tagfilter]
-      parsed_config[:tagfilter].keys.each do |tag|
-        parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
-      end
+    parsed_config[:tagfilter]&.keys&.each do |tag|
+      parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
     end
 
     parsed_config[:uptime] = Time.now
@@ -146,24 +195,30 @@ module Vmpooler
     pools
   end
 
-  def self.new_redis(host = 'localhost', port = nil, password = nil)
-    Redis.new(host: host, port: port, password: password)
-  end
-
-  def self.new_logger(logfile)
-    Vmpooler::Logger.new logfile
-  end
-
-  def self.new_metrics(params)
-    if params[:statsd]
-      Vmpooler::Statsd.new(params[:statsd])
-    elsif params[:graphite]
-      Vmpooler::Graphite.new(params[:graphite])
-    else
-      Vmpooler::DummyStatsd.new
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics, redis_reconnect_attempts = 0)
+    Vmpooler::PoolManager::GenericConnectionPool.new(
+      metrics: metrics,
+      connpool_type: 'redis_connection_pool',
+      connpool_provider: 'manager',
+      size: size,
+      timeout: timeout
+    ) do
+      connection = Concurrent::Hash.new
+      redis = new_redis(host, port, password, redis_reconnect_attempts)
+      connection['connection'] = redis
     end
   end
 
+  def self.new_redis(host = 'localhost', port = nil, password = nil, redis_reconnect_attempts = 10)
+    Redis.new(
+      host: host,
+      port: port,
+      password: password,
+      reconnect_attempts: redis_reconnect_attempts,
+      connect_timeout: 300
+    )
+  end
+
   def self.pools(conf)
     conf[:pools]
   end
@@ -171,7 +226,7 @@ module Vmpooler
   def self.pool_index(pools)
     pools_hash = {}
     index = 0
-    for pool in pools
+    pools.each do |pool|
       pools_hash[pool['name']] = index
       index += 1
     end
@@ -182,16 +237,62 @@ module Vmpooler
     # Returns a integer if input is a string
     return if s.nil?
     return unless s =~ /\d/
-    return Integer(s)
+
+    Integer(s)
+  end
+
+  def self.string_array_to_array(s)
+    # Returns an array from an array like string
+    return if s.nil?
+
+    JSON.parse(s)
   end
 
   def self.true?(obj)
-    obj.to_s.downcase == "true"
+    obj.to_s.downcase == 'true'
   end
 
-  def self.set_linked_clone(parsed_config)
+  def self.set_linked_clone(parsed_config) # rubocop:disable Naming/AccessorMethodName
     parsed_config[:config]['create_linked_clones'] = parsed_config[:config]['create_linked_clones'] || true
     parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/
     parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones']
   end
+
+  def self.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+    if startup_args.length == 1 && startup_args.include?('api')
+      service_name = 'vmpooler-api'
+    elsif startup_args.length == 1 && startup_args.include?('manager')
+      service_name = 'vmpooler-manager'
+    else
+      service_name = 'vmpooler'
+    end
+
+    service_name += "-#{prefix}" unless prefix.empty?
+
+    if tracing_enabled.eql?('false')
+      puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
+      )
+    else
+      puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
+      )
+    end
+
+    OpenTelemetry::SDK.configure do |c|
+      c.use 'OpenTelemetry::Instrumentation::Sinatra'
+      c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::HttpClient'
+      c.use 'OpenTelemetry::Instrumentation::Redis'
+
+      c.add_span_processor(span_processor)
+
+      c.service_name = service_name
+      c.service_version = version
+
+      c.resource = OpenTelemetry::Resource::Detectors::AutoDetector.detect
+    end
+  end
 end
diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb
index 239155b..3b0d9de 100644
--- a/lib/vmpooler/api.rb
+++ b/lib/vmpooler/api.rb
@@ -1,49 +1,62 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API < Sinatra::Base
-    def initialize
-      super
-    end
-
-    not_found do
-      content_type :json
-
-      result = {
-        ok: false
-      }
-
-      JSON.pretty_generate(result)
-    end
-
-    # Load dashboard components
-    begin
-      require 'dashboard'
-    rescue LoadError
-      require File.expand_path(File.join(File.dirname(__FILE__), 'dashboard'))
-    end
-
-    use Vmpooler::Dashboard
-
     # Load API components
-    %w[helpers dashboard reroute v1].each do |lib|
-      begin
-        require "api/#{lib}"
-      rescue LoadError
-        require File.expand_path(File.join(File.dirname(__FILE__), 'api', lib))
-      end
+    %w[helpers dashboard v3 request_logger healthcheck].each do |lib|
+      require "vmpooler/api/#{lib}"
     end
+    # Load dashboard components
+    require 'vmpooler/dashboard'
 
-    use Vmpooler::API::Dashboard
-    use Vmpooler::API::Reroute
-    use Vmpooler::API::V1
-
-    def configure(config, redis, metrics)
+    def self.execute(torun, config, redis, metrics, logger)
       self.settings.set :config, config
-      self.settings.set :redis, redis
+      self.settings.set :redis, redis unless redis.nil?
       self.settings.set :metrics, metrics
-    end
+      self.settings.set :checkoutlock, Mutex.new
 
-    def execute!
-      self.settings.run!
+      # Deflating in all situations
+      # https://www.schneems.com/2017/11/08/80-smaller-rails-footprint-with-rack-deflate/
+      use Rack::Deflater
+
+      # not_found clause placed here to fix rspec test issue.
+      not_found do
+        content_type :json
+
+        result = {
+          ok: false
+        }
+
+        JSON.pretty_generate(result)
+      end
+
+      if metrics.respond_to?(:setup_prometheus_metrics)
+        # Prometheus metrics are only setup if actually specified
+        # in the config file.
+        metrics.setup_prometheus_metrics(torun)
+
+        # Using customised collector that filters out hostnames on API paths
+        require 'vmpooler/metrics/promstats/collector_middleware'
+        require 'prometheus/middleware/exporter'
+        use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.prometheus_prefix}_http"
+        use Prometheus::Middleware::Exporter, path: metrics.prometheus_endpoint
+        # Note that a user may want to use this check without prometheus
+        # However, prometheus setup includes the web server which is required for this check
+        # At this time prometheus is a requirement of using the health check on manager
+        use Vmpooler::API::Healthcheck
+      end
+
+      if torun.include? :api
+        # Enable API request logging only if required
+        use Vmpooler::API::RequestLogger, logger: logger if config[:config]['request_logger']
+
+        use Vmpooler::Dashboard
+        use Vmpooler::API::Dashboard
+        use Vmpooler::API::V3
+      end
+
+      # Get thee started O WebServer
+      self.run!
     end
   end
 end
diff --git a/lib/vmpooler/api/dashboard.rb b/lib/vmpooler/api/dashboard.rb
index 2288b79..f5e65cf 100644
--- a/lib/vmpooler/api/dashboard.rb
+++ b/lib/vmpooler/api/dashboard.rb
@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API
     class Dashboard < Sinatra::Base
-
       helpers do
         include Vmpooler::API::Helpers
       end
@@ -21,9 +22,11 @@ module Vmpooler
 
         if config[:graphs]
           return false unless config[:graphs]['server']
+
           @graph_server = config[:graphs]['server']
         elsif config[:graphite]
           return false unless config[:graphite]['server']
+
           @graph_server = config[:graphite]['server']
         else
           false
@@ -36,9 +39,11 @@ module Vmpooler
 
         if config[:graphs]
           return 'vmpooler' unless config[:graphs]['prefix']
+
           @graph_prefix = config[:graphs]['prefix']
         elsif config[:graphite]
           return false unless config[:graphite]['prefix']
+
           @graph_prefix = config[:graphite]['prefix']
         else
           false
@@ -48,12 +53,14 @@ module Vmpooler
       # what is the base URL for viewable graphs?
       def graph_url
         return false unless graph_server && graph_prefix
+
         @graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
       end
 
       # return a full URL to a viewable graph for a given metrics target (graphite syntax)
       def graph_link(target = '')
         return '' unless graph_url
+
         graph_url + target
       end
 
@@ -76,7 +83,7 @@ module Vmpooler
             history ||= {}
 
             begin
-              buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
+              buffer = URI.parse(graph_link('.ready.*&from=-1hour&format=json')).read
               history = JSON.parse(buffer)
 
               history.each do |pool|
@@ -100,7 +107,7 @@ module Vmpooler
                   end
                 end
               end
-            rescue
+            rescue StandardError
             end
           else
             pools.each do |pool|
@@ -121,34 +128,32 @@ module Vmpooler
 
         pools.each do |pool|
           running = running_hash[pool['name']]
-          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
           result[pool['major']] ||= {}
           result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
         end
 
-        if params[:history]
-          if graph_url
-            begin
-              buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
-              JSON.parse(buffer).each do |pool|
-                if pool['target'] =~ /.*\.(.*)$/
-                  pool['name'] = Regexp.last_match[1]
-                  pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
-                  result[pool['major']]['history'] ||= []
+        if params[:history] && graph_url
+          begin
+            buffer = URI.parse(graph_link('.running.*&from=-1hour&format=json')).read
+            JSON.parse(buffer).each do |pool|
+              if pool['target'] =~ /.*\.(.*)$/
+                pool['name'] = Regexp.last_match[1]
+                pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
+                result[pool['major']]['history'] ||= []
 
-                  for i in 0..pool['datapoints'].length
-                    if pool['datapoints'][i] && pool['datapoints'][i][0]
-                      pool['last'] = pool['datapoints'][i][0]
-                      result[pool['major']]['history'][i] ||= 0
-                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
-                    else
-                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
-                    end
+                for i in 0..pool['datapoints'].length
+                  if pool['datapoints'][i] && pool['datapoints'][i][0]
+                    pool['last'] = pool['datapoints'][i][0]
+                    result[pool['major']]['history'][i] ||= 0
+                    result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
+                  else
+                    result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
                   end
                 end
               end
-            rescue
             end
+          rescue StandardError
           end
         end
         JSON.pretty_generate(result)
diff --git a/lib/vmpooler/api/healthcheck.rb b/lib/vmpooler/api/healthcheck.rb
new file mode 100644
index 0000000..50da734
--- /dev/null
+++ b/lib/vmpooler/api/healthcheck.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    class Healthcheck < Sinatra::Base
+      get '/healthcheck/?' do
+        content_type :json
+
+        status 200
+        JSON.pretty_generate({ 'ok' => true })
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb
index 54636ca..747640d 100644
--- a/lib/vmpooler/api/helpers.rb
+++ b/lib/vmpooler/api/helpers.rb
@@ -1,133 +1,198 @@
+# frozen_string_literal: true
+
+require 'vmpooler/api/input_validator'
+
 module Vmpooler
 
   class API
 
     module Helpers
+      include InputValidator
+
+      def tracer
+        @tracer ||= OpenTelemetry.tracer_provider.tracer('api', Vmpooler::VERSION)
+      end
 
       def has_token?
         request.env['HTTP_X_AUTH_TOKEN'].nil? ? false : true
       end
 
       def valid_token?(backend)
-        return false unless has_token?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return false unless has_token?
 
-        backend.exists('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
+          backend.exists?("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}") ? true : false
+        end
       end
 
       def validate_token(backend)
-        if valid_token?(backend)
-          backend.hset('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'last', Time.now)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          if valid_token?(backend)
+            backend.hset("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'last', Time.now.to_s)
 
-          return true
+            return true
+          end
+
+          content_type :json
+
+          result = { 'ok' => false }
+
+          headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+          halt 401, JSON.pretty_generate(result)
         end
-
-        content_type :json
-
-        result = { 'ok' => false }
-
-        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
-        halt 401, JSON.pretty_generate(result)
       end
 
       def validate_auth(backend)
-        return if authorized?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return if authorized?
 
-        content_type :json
+          content_type :json
 
-        result = { 'ok' => false }
+          result = { 'ok' => false }
 
-        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
-        halt 401, JSON.pretty_generate(result)
+          headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+          halt 401, JSON.pretty_generate(result)
+        end
       end
 
       def authorized?
-        @auth ||= Rack::Auth::Basic::Request.new(request.env)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          @auth ||= Rack::Auth::Basic::Request.new(request.env)
 
-        if @auth.provided? and @auth.basic? and @auth.credentials
-          username, password = @auth.credentials
+          if @auth.provided? and @auth.basic? and @auth.credentials
+            username, password = @auth.credentials
 
-          if authenticate(Vmpooler::API.settings.config[:auth], username, password)
-            return true
+            if authenticate(Vmpooler::API.settings.config[:auth], username, password)
+              return true
+            end
           end
-        end
 
-        return false
+          return false
+        end
       end
 
-      def authenticate_ldap(port, host, user_object, base, username_str, password_str)
-        ldap = Net::LDAP.new(
-          :host => host,
-          :port => port,
-          :encryption => {
-            :method => :start_tls,
-            :tls_options => { :ssl_version => 'TLSv1' }
+      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.name' => host,
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp',
+            'enduser.id' => username_str
           },
-          :base => base,
-          :auth => {
-            :method => :simple,
-            :username => "#{user_object}=#{username_str},#{base}",
-            :password => password_str
-          }
-        )
+          kind: :client
+        ) do
+          if service_account_hash
+            username = service_account_hash[:user_dn]
+            password = service_account_hash[:password]
+          else
+            username = "#{user_object}=#{username_str},#{base}"
+            password = password_str
+          end
 
-        return true if ldap.bind
-        return false
+          ldap = Net::LDAP.new(
+            :host => host,
+            :port => port,
+            :encryption => encryption_hash,
+            :base => base,
+            :auth => {
+              :method => :simple,
+              :username => username,
+              :password => password
+            }
+          )
+
+          if service_account_hash
+            return true if ldap.bind_as(
+              :base => base,
+              :filter => "(#{user_object}=#{username_str})",
+              :password => password_str
+            )
+          elsif ldap.bind
+            return true
+          else
+            return false
+          end
+
+          return false
+        end
       end
 
       def authenticate(auth, username_str, password_str)
-        case auth['provider']
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'enduser.id' => username_str
+          }
+        ) do
+          case auth['provider']
           when 'dummy'
             return (username_str != password_str)
           when 'ldap'
             ldap_base = auth[:ldap]['base']
             ldap_port = auth[:ldap]['port'] || 389
+            ldap_user_obj = auth[:ldap]['user_object']
+            ldap_host = auth[:ldap]['host']
+            ldap_encryption_hash = auth[:ldap]['encryption'] || {
+              :method => :start_tls,
+              :tls_options => { :ssl_version => 'TLSv1' }
+            }
+            service_account_hash = auth[:ldap]['service_account_hash']
 
-            if ldap_base.is_a? Array
-              ldap_base.each do |search_base|
+            unless ldap_base.is_a? Array
+              ldap_base = ldap_base.split
+            end
+
+            unless ldap_user_obj.is_a? Array
+              ldap_user_obj = ldap_user_obj.split
+            end
+
+            ldap_base.each do |search_base|
+              ldap_user_obj.each do |search_user_obj|
                 result = authenticate_ldap(
                   ldap_port,
-                  auth[:ldap]['host'],
-                  auth[:ldap]['user_object'],
+                  ldap_host,
+                  ldap_encryption_hash,
+                  search_user_obj,
                   search_base,
                   username_str,
                   password_str,
+                  service_account_hash
                 )
-                return true if result == true
+                return true if result
               end
-            else
-              result = authenticate_ldap(
-                ldap_port,
-                auth[:ldap]['host'],
-                auth[:ldap]['user_object'],
-                ldap_base,
-                username_str,
-                password_str,
-              )
-              return result
             end
 
             return false
           end
+        end
       end
 
       def export_tags(backend, hostname, tags)
-        tags.each_pair do |tag, value|
-          next if value.nil? or value.empty?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          backend.pipelined do |pipeline|
+            tags.each_pair do |tag, value|
+              next if value.nil? or value.empty?
 
-          backend.hset('vmpooler__vm__' + hostname, 'tag:' + tag, value)
-          backend.hset('vmpooler__tag__' + Date.today.to_s, hostname + ':' + tag, value)
+              pipeline.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value)
+              pipeline.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value)
+            end
+          end
         end
       end
 
       def filter_tags(tags)
-        return unless Vmpooler::API.settings.config[:tagfilter]
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return unless Vmpooler::API.settings.config[:tagfilter]
 
-        tags.each_pair do |tag, value|
-          next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
-          tags[tag] = value.match(filter).captures.join if value.match(filter)
+          tags.each_pair do |tag, value|
+            next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
+
+            tags[tag] = value.match(filter).captures.join if value.match(filter)
+          end
+
+          tags
         end
-
-        tags
       end
 
       def mean(list)
@@ -139,318 +204,353 @@ module Vmpooler
         /^\d{4}-\d{2}-\d{2}$/ === date_str
       end
 
-      def hostname_shorten(hostname, domain=nil)
-        if domain && hostname =~ /^\w+\.#{domain}$/
-          hostname = hostname[/[^\.]+/]
-        end
-
-        hostname
+      def hostname_shorten(hostname)
+        hostname[/[^.]+/]
       end
 
       def get_task_times(backend, task, date_str)
-        backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
+        end
       end
 
       # Takes the pools and a key to run scard on
       # returns an integer for the total count
       def get_total_across_pools_redis_scard(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.scard(key + pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
           end
+          res.inject(0) { |m, x| m + x }.to_i
         end
-        res.inject(0){ |m, x| m+x }.to_i
       end
 
       # Takes the pools and a key to run scard on
       # returns a hash with each pool name as key and the value being the count as integer
       def get_list_across_pools_redis_scard(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        temp_hash = {}
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.scard(key + pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
           end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_i
+          end
+          temp_hash
         end
-        pools.each_with_index do |pool, i|
-          temp_hash[pool['name']] = res[i].to_i
-        end
-        temp_hash
       end
 
       # Takes the pools and a key to run hget on
       # returns a hash with each pool name as key and the value as string
       def get_list_across_pools_redis_hget(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        temp_hash = {}
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.hget(key, pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.hget(key, pool['name'])
+            end
           end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_s
+          end
+          temp_hash
         end
-        pools.each_with_index do |pool, i|
-          temp_hash[pool['name']] = res[i].to_s
-        end
-        temp_hash
       end
 
       def get_capacity_metrics(pools, backend)
-        capacity = {
-            current: 0,
-            total:   0,
-            percent: 0
-        }
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          capacity = {
+              current: 0,
+              total: 0,
+              percent: 0
+          }
 
-        pools.each do |pool|
-          capacity[:total]   += pool['size'].to_i
+          pools.each do |pool|
+            capacity[:total] += pool['size'].to_i
+          end
+
+          capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
+          if capacity[:total] > 0
+            capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1)
+          end
+
+          capacity
         end
-
-        capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-
-        if capacity[:total] > 0
-          capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1)
-        end
-
-        capacity
       end
 
       def get_queue_metrics(pools, backend)
-        queue = {
-            pending:   0,
-            cloning:   0,
-            booting:   0,
-            ready:     0,
-            running:   0,
-            completed: 0,
-            total:     0
-        }
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          queue = {
+              requested: 0,
+              pending: 0,
+              cloning: 0,
+              booting: 0,
+              ready: 0,
+              running: 0,
+              completed: 0,
+              total: 0
+          }
 
-        queue[:pending]   = get_total_across_pools_redis_scard(pools,'vmpooler__pending__', backend)
-        queue[:ready]     = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-        queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-        queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
+          # Use a single pipeline to fetch all queue counts at once for better performance
+          results = backend.pipelined do |pipeline|
+            # Order matters - we'll use indices to extract values
+            pools.each do |pool|
+              pipeline.scard("vmpooler__provisioning__request#{pool['name']}")    # 0..n-1
+              pipeline.scard("vmpooler__provisioning__processing#{pool['name']}") # n..2n-1
+              pipeline.scard("vmpooler__odcreate__task#{pool['name']}")           # 2n..3n-1
+              pipeline.scard("vmpooler__pending__#{pool['name']}")                # 3n..4n-1
+              pipeline.scard("vmpooler__ready__#{pool['name']}")                  # 4n..5n-1
+              pipeline.scard("vmpooler__running__#{pool['name']}")                # 5n..6n-1
+              pipeline.scard("vmpooler__completed__#{pool['name']}")              # 6n..7n-1
+            end
+            pipeline.get('vmpooler__tasks__clone')           # 7n
+            pipeline.get('vmpooler__tasks__ondemandclone')   # 7n+1
+          end
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
-        queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
-        queue[:booting] = 0 if queue[:booting] < 0
-        queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
+          n = pools.length
+          # Safely extract results with default to empty array if slice returns nil
+          queue[:requested] = (results[0...n] || []).sum(&:to_i) +
+                              (results[n...(2 * n)] || []).sum(&:to_i) +
+                              (results[(2 * n)...(3 * n)] || []).sum(&:to_i)
+          queue[:pending]   = (results[(3 * n)...(4 * n)] || []).sum(&:to_i)
+          queue[:ready]     = (results[(4 * n)...(5 * n)] || []).sum(&:to_i)
+          queue[:running]   = (results[(5 * n)...(6 * n)] || []).sum(&:to_i)
+          queue[:completed] = (results[(6 * n)...(7 * n)] || []).sum(&:to_i)
+          queue[:cloning]   = (results[7 * n] || 0).to_i + (results[7 * n + 1] || 0).to_i
+          queue[:booting]   = queue[:pending].to_i - queue[:cloning].to_i
+          queue[:booting]   = 0 if queue[:booting] < 0
+          queue[:total]     = queue[:requested] + queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
 
-        queue
+          queue
+        end
       end
 
       def get_tag_metrics(backend, date_str, opts = {})
-        opts = {:only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:only => false}.merge(opts)
 
-        tags = {}
+          tags = {}
 
-        backend.hgetall('vmpooler__tag__' + date_str).each do |key, value|
-          hostname = 'unknown'
-          tag = 'unknown'
+          backend.hgetall("vmpooler__tag__#{date_str}").each do |key, value|
+            hostname = 'unknown'
+            tag = 'unknown'
 
-          if key =~ /\:/
-            hostname, tag = key.split(':', 2)
+            if key =~ /:/
+              hostname, tag = key.split(':', 2)
+            end
+
+            next if opts[:only] && tag != opts[:only]
+
+            tags[tag] ||= {}
+            tags[tag][value] ||= 0
+            tags[tag][value] += 1
+
+            tags[tag]['total'] ||= 0
+            tags[tag]['total'] += 1
           end
 
-          if opts[:only]
-            next unless tag == opts[:only]
-          end
-
-          tags[tag] ||= {}
-          tags[tag][value] ||= 0
-          tags[tag][value] += 1
-
-          tags[tag]['total'] ||= 0
-          tags[tag]['total'] += 1
+          tags
         end
-
-        tags
       end
 
       def get_tag_summary(backend, from_date, to_date, opts = {})
-        opts = {:only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:only => false}.merge(opts)
 
-        result = {
-          tag: {},
-          daily: []
-        }
-
-        (from_date..to_date).each do |date|
-          daily = {
-            date: date.to_s,
-            tag: get_tag_metrics(backend, date.to_s, opts)
+          result = {
+            tag: {},
+            daily: []
           }
-          result[:daily].push(daily)
-        end
 
-        result[:daily].each do |daily|
-          daily[:tag].each_key do |tag|
-            result[:tag][tag] ||= {}
+          (from_date..to_date).each do |date|
+            daily = {
+              date: date.to_s,
+              tag: get_tag_metrics(backend, date.to_s, opts)
+            }
+            result[:daily].push(daily)
+          end
 
-            daily[:tag][tag].each do |key, value|
-              result[:tag][tag][key] ||= 0
-              result[:tag][tag][key] += value
+          result[:daily].each do |daily|
+            daily[:tag].each_key do |tag|
+              result[:tag][tag] ||= {}
+
+              daily[:tag][tag].each do |key, value|
+                result[:tag][tag][key] ||= 0
+                result[:tag][tag][key] += value
+              end
             end
           end
-        end
 
-        result
+          result
+        end
       end
 
       def get_task_metrics(backend, task_str, date_str, opts = {})
-        opts = {:bypool => false, :only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:bypool => false, :only => false}.merge(opts)
 
-        task = {
-            duration: {
-                average: 0,
-                min:     0,
-                max:     0,
-                total:   0
-            },
-            count:    {
-                total: 0
-            }
-        }
+          task = {
+              duration: {
+                  average: 0,
+                  min: 0,
+                  max: 0,
+                  total: 0
+              },
+              count: {
+                  total: 0
+              }
+          }
 
-        task[:count][:total] = backend.hlen('vmpooler__' + task_str + '__' + date_str).to_i
+          task[:count][:total] = backend.hlen("vmpooler__#{task_str}__#{date_str}").to_i
 
-        if task[:count][:total] > 0
-          if opts[:bypool] == true
-            task_times_bypool = {}
+          if task[:count][:total] > 0
+            if opts[:bypool] == true
+              task_times_bypool = {}
 
-            task[:count][:pool]    = {}
-            task[:duration][:pool] = {}
+              task[:count][:pool]    = {}
+              task[:duration][:pool] = {}
 
-            backend.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
-              pool     = 'unknown'
-              hostname = 'unknown'
+              backend.hgetall("vmpooler__#{task_str}__#{date_str}").each do |key, value|
+                pool     = 'unknown'
+                hostname = 'unknown'
 
-              if key =~ /\:/
-                pool, hostname = key.split(':')
-              else
-                hostname = key
+                if key =~ /:/
+                  pool, hostname = key.split(':')
+                else
+                  hostname = key
+                end
+
+                task[:count][:pool][pool]    ||= {}
+                task[:duration][:pool][pool] ||= {}
+
+                task_times_bypool[pool] ||= []
+                task_times_bypool[pool].push(value.to_f)
               end
 
-              task[:count][:pool][pool]    ||= {}
-              task[:duration][:pool][pool] ||= {}
+              task_times_bypool.each_key do |pool|
+                task[:count][:pool][pool][:total] = task_times_bypool[pool].length
 
-              task_times_bypool[pool] ||= []
-              task_times_bypool[pool].push(value.to_f)
+                task[:duration][:pool][pool][:total]                                   = task_times_bypool[pool].reduce(:+).to_f
+                task[:duration][:pool][pool][:average]                                 = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1)
+                task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax
+              end
             end
 
-            task_times_bypool.each_key do |pool|
-              task[:count][:pool][pool][:total] = task_times_bypool[pool].length
+            task_times = get_task_times(backend, task_str, date_str)
 
-              task[:duration][:pool][pool][:total]                                   = task_times_bypool[pool].reduce(:+).to_f
-              task[:duration][:pool][pool][:average]                                 = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1)
-              task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax
+            task[:duration][:total]                      = task_times.reduce(:+).to_f
+            task[:duration][:average]                    = (task[:duration][:total] / task[:count][:total]).round(1)
+            task[:duration][:min], task[:duration][:max] = task_times.minmax
+          end
+
+          if opts[:only]
+            task.each_key do |key|
+              task.delete(key) unless key.to_s == opts[:only]
             end
           end
 
-          task_times = get_task_times(backend, task_str, date_str)
-
-          task[:duration][:total]                      = task_times.reduce(:+).to_f
-          task[:duration][:average]                    = (task[:duration][:total] / task[:count][:total]).round(1)
-          task[:duration][:min], task[:duration][:max] = task_times.minmax
+          task
         end
-
-        if opts[:only]
-          task.each_key do |key|
-            task.delete(key) unless key.to_s == opts[:only]
-          end
-        end
-
-        task
       end
 
       def get_task_summary(backend, task_str, from_date, to_date, opts = {})
-        opts = {:bypool => false, :only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:bypool => false, :only => false}.merge(opts)
 
-        task_sym = task_str.to_sym
+          task_sym = task_str.to_sym
 
-        result = {
-          task_sym => {},
-          daily: []
-        }
-
-        (from_date..to_date).each do |date|
-          daily = {
-            date: date.to_s,
-            task_sym => get_task_metrics(backend, task_str, date.to_s, opts)
+          result = {
+            task_sym => {},
+            daily: []
           }
-          result[:daily].push(daily)
-        end
 
-        daily_task = {}
-        daily_task_bypool = {} if opts[:bypool] == true
+          (from_date..to_date).each do |date|
+            daily = {
+              date: date.to_s,
+              task_sym => get_task_metrics(backend, task_str, date.to_s, opts)
+            }
+            result[:daily].push(daily)
+          end
 
-        result[:daily].each do |daily|
-          daily[task_sym].each_key do |type|
-            result[task_sym][type] ||= {}
-            daily_task[type] ||= {}
+          daily_task = {}
+          daily_task_bypool = {} if opts[:bypool] == true
 
-            ['min', 'max'].each do |key|
-              if daily[task_sym][type][key]
-                daily_task[type][:data] ||= []
-                daily_task[type][:data].push(daily[task_sym][type][key])
+          result[:daily].each do |daily|
+            daily[task_sym].each_key do |type|
+              result[task_sym][type] ||= {}
+              daily_task[type] ||= {}
+
+              ['min', 'max'].each do |key|
+                if daily[task_sym][type][key]
+                  daily_task[type][:data] ||= []
+                  daily_task[type][:data].push(daily[task_sym][type][key])
+                end
+              end
+
+              result[task_sym][type][:total] ||= 0
+              result[task_sym][type][:total] += daily[task_sym][type][:total]
+
+              if opts[:bypool] == true
+                result[task_sym][type][:pool] ||= {}
+                daily_task_bypool[type] ||= {}
+
+                next unless daily[task_sym][type][:pool]
+
+                daily[task_sym][type][:pool].each_key do |pool|
+                  result[task_sym][type][:pool][pool] ||= {}
+                  daily_task_bypool[type][pool] ||= {}
+
+                  ['min', 'max'].each do |key|
+                    if daily[task_sym][type][:pool][pool][key.to_sym]
+                      daily_task_bypool[type][pool][:data] ||= []
+                      daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym])
+                    end
+                  end
+
+                  result[task_sym][type][:pool][pool][:total] ||= 0
+                  result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total]
+                end
               end
             end
+          end
 
-            result[task_sym][type][:total] ||= 0
-            result[task_sym][type][:total] += daily[task_sym][type][:total]
+          result[task_sym].each_key do |type|
+            if daily_task[type][:data]
+              result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax
+              result[task_sym][type][:average] = mean(daily_task[type][:data])
+            end
 
             if opts[:bypool] == true
-              result[task_sym][type][:pool] ||= {}
-              daily_task_bypool[type] ||= {}
-
-              next unless daily[task_sym][type][:pool]
-
-              daily[task_sym][type][:pool].each_key do |pool|
-                result[task_sym][type][:pool][pool] ||= {}
-                daily_task_bypool[type][pool] ||= {}
-
-                ['min', 'max'].each do |key|
-                  if daily[task_sym][type][:pool][pool][key.to_sym]
-                    daily_task_bypool[type][pool][:data] ||= []
-                    daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym])
+              result[task_sym].each_key do |type|
+                result[task_sym][type][:pool].each_key do |pool|
+                  if daily_task_bypool[type][pool][:data]
+                    result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax
+                    result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data])
                   end
                 end
-
-                result[task_sym][type][:pool][pool][:total] ||= 0
-                result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total]
               end
             end
           end
+
+          result
         end
-
-        result[task_sym].each_key do |type|
-          if daily_task[type][:data]
-            result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax
-            result[task_sym][type][:average] = mean(daily_task[type][:data])
-          end
-
-          if opts[:bypool] == true
-            result[task_sym].each_key do |type|
-              result[task_sym][type][:pool].each_key do |pool|
-                if daily_task_bypool[type][pool][:data]
-                  result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax
-                  result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data])
-                end
-              end
-            end
-          end
-        end
-
-        result
       end
 
       def pool_index(pools)
         pools_hash = {}
         index = 0
-        for pool in pools
+        pools.each do |pool|
           pools_hash[pool['name']] = index
           index += 1
         end
@@ -458,24 +558,36 @@ module Vmpooler
       end
 
       def template_ready?(pool, backend)
-        prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
-        return false if prepared_template.nil?
-        return true if pool['template'] == prepared_template
-        return false
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
+          return false if prepared_template.nil?
+          return true if pool['template'] == prepared_template
+
+          return false
+        end
       end
 
       def is_integer?(x)
         Integer(x)
         true
-      rescue
+      rescue StandardError
         false
       end
 
       def open_socket(host, domain = nil, timeout = 1, port = 22, &_block)
-        Timeout.timeout(timeout) do
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp'
+          },
+          kind: :client
+        ) do
           target_host = host
           target_host = "#{host}.#{domain}" if domain
-          sock = TCPSocket.new target_host, port
+          span = OpenTelemetry::Trace.current_span
+          span.set_attribute('net.peer.name', target_host)
+          sock = TCPSocket.new(target_host, port, connect_timeout: timeout)
           begin
             yield sock if block_given?
           ensure
@@ -483,16 +595,6 @@ module Vmpooler
           end
         end
       end
-
-      def vm_ready?(vm_name, domain = nil)
-        begin
-          open_socket(vm_name, domain)
-        rescue => _err
-          return false
-        end
-
-        true
-      end
     end
   end
 end
diff --git a/lib/vmpooler/api/input_validator.rb b/lib/vmpooler/api/input_validator.rb
new file mode 100644
index 0000000..add4d6a
--- /dev/null
+++ b/lib/vmpooler/api/input_validator.rb
@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Input validation helpers to enhance security
+    module InputValidator
+      # Maximum lengths to prevent abuse
+      MAX_HOSTNAME_LENGTH = 253
+      MAX_TAG_KEY_LENGTH = 50
+      MAX_TAG_VALUE_LENGTH = 255
+      MAX_REASON_LENGTH = 500
+      MAX_POOL_NAME_LENGTH = 100
+      MAX_TOKEN_LENGTH = 64
+
+      # Valid patterns
+      HOSTNAME_PATTERN = /\A[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)* \z/ix.freeze
+      POOL_NAME_PATTERN = /\A[a-zA-Z0-9_-]+\z/.freeze
+      TAG_KEY_PATTERN = /\A[a-zA-Z0-9_\-.]+\z/.freeze
+      TOKEN_PATTERN = /\A[a-zA-Z0-9\-_]+\z/.freeze
+      INTEGER_PATTERN = /\A\d+\z/.freeze
+
+      class ValidationError < StandardError; end
+
+      # Validate hostname format and length
+      def validate_hostname(hostname)
+        return error_response('Hostname is required') if hostname.nil? || hostname.empty?
+        return error_response('Hostname too long') if hostname.length > MAX_HOSTNAME_LENGTH
+        return error_response('Invalid hostname format') unless hostname.match?(HOSTNAME_PATTERN)
+
+        true
+      end
+
+      # Validate pool/template name
+      def validate_pool_name(pool_name)
+        return error_response('Pool name is required') if pool_name.nil? || pool_name.empty?
+        return error_response('Pool name too long') if pool_name.length > MAX_POOL_NAME_LENGTH
+        return error_response('Invalid pool name format') unless pool_name.match?(POOL_NAME_PATTERN)
+
+        true
+      end
+
+      # Validate tag key and value
+      def validate_tag(key, value)
+        return error_response('Tag key is required') if key.nil? || key.empty?
+        return error_response('Tag key too long') if key.length > MAX_TAG_KEY_LENGTH
+        return error_response('Invalid tag key format') unless key.match?(TAG_KEY_PATTERN)
+
+        if value
+          return error_response('Tag value too long') if value.length > MAX_TAG_VALUE_LENGTH
+
+          # Sanitize value to prevent injection attacks
+          sanitized_value = value.gsub(/[^\w\s\-.@:\/]/, '')
+          return error_response('Tag value contains invalid characters') if sanitized_value != value
+        end
+
+        true
+      end
+
+      # Validate token format
+      def validate_token_format(token)
+        return error_response('Token is required') if token.nil? || token.empty?
+        return error_response('Token too long') if token.length > MAX_TOKEN_LENGTH
+        return error_response('Invalid token format') unless token.match?(TOKEN_PATTERN)
+
+        true
+      end
+
+      # Validate integer parameter
+      def validate_integer(value, name = 'value', min: nil, max: nil)
+        return error_response("#{name} is required") if value.nil?
+
+        value_str = value.to_s
+        return error_response("#{name} must be a valid integer") unless value_str.match?(INTEGER_PATTERN)
+
+        int_value = value.to_i
+        return error_response("#{name} must be at least #{min}") if min && int_value < min
+        return error_response("#{name} must be at most #{max}") if max && int_value > max
+
+        int_value
+      end
+
+      # Validate VM request count
+      def validate_vm_count(count)
+        validated = validate_integer(count, 'VM count', min: 1, max: 100)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate disk size
+      def validate_disk_size(size)
+        validated = validate_integer(size, 'Disk size', min: 1, max: 2048)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate lifetime (TTL) in hours
+      def validate_lifetime(lifetime)
+        validated = validate_integer(lifetime, 'Lifetime', min: 1, max: 168) # max 1 week
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate reason text
+      def validate_reason(reason)
+        return true if reason.nil? || reason.empty?
+        return error_response('Reason too long') if reason.length > MAX_REASON_LENGTH
+
+        # Sanitize to prevent XSS/injection
+        sanitized = reason.gsub(/[<>"']/, '')
+        return error_response('Reason contains invalid characters') if sanitized != reason
+
+        true
+      end
+
+      # Sanitize JSON body to prevent injection
+      def sanitize_json_body(body)
+        return {} if body.nil? || body.empty?
+
+        begin
+          parsed = JSON.parse(body)
+          return error_response('Request body must be a JSON object') unless parsed.is_a?(Hash)
+
+          # Limit depth and size to prevent DoS
+          return error_response('Request body too complex') if json_depth(parsed) > 5
+          return error_response('Request body too large') if body.length > 10_240 # 10KB max
+
+          parsed
+        rescue JSON::ParserError => e
+          error_response("Invalid JSON: #{e.message}")
+        end
+      end
+
+      # Check if validation result is an error
+      def validation_error?(result)
+        result.is_a?(Hash) && result['ok'] == false
+      end
+
+      private
+
+      def error_response(message)
+        { 'ok' => false, 'error' => message }
+      end
+
+      def json_depth(obj, depth = 0)
+        return depth unless obj.is_a?(Hash) || obj.is_a?(Array)
+        return depth + 1 if obj.empty?
+
+        if obj.is_a?(Hash)
+          depth + 1 + obj.values.map { |v| json_depth(v, 0) }.max
+        else
+          depth + 1 + obj.map { |v| json_depth(v, 0) }.max
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/rate_limiter.rb b/lib/vmpooler/api/rate_limiter.rb
new file mode 100644
index 0000000..8ecfb62
--- /dev/null
+++ b/lib/vmpooler/api/rate_limiter.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Rate limiter middleware to protect against abuse
+    # Uses Redis to track request counts per IP and token
+    class RateLimiter
+      DEFAULT_LIMITS = {
+        global_per_ip: { limit: 100, period: 60 }, # 100 requests per minute per IP
+        authenticated: { limit: 500, period: 60 }, # 500 requests per minute with token
+        vm_creation: { limit: 20, period: 60 },    # 20 VM creations per minute
+        vm_deletion: { limit: 50, period: 60 }     # 50 VM deletions per minute
+      }.freeze
+
+      def initialize(app, redis, config = {})
+        @app = app
+        @redis = redis
+        @config = DEFAULT_LIMITS.merge(config[:rate_limits] || {})
+        @enabled = config.fetch(:rate_limiting_enabled, true)
+      end
+
+      def call(env)
+        return @app.call(env) unless @enabled
+
+        request = Rack::Request.new(env)
+        client_id = identify_client(request)
+        endpoint_type = classify_endpoint(request)
+
+        # Check rate limits
+        return rate_limit_response(client_id, endpoint_type) if rate_limit_exceeded?(client_id, endpoint_type, request)
+
+        # Track the request
+        increment_request_count(client_id, endpoint_type)
+
+        @app.call(env)
+      end
+
+      private
+
+      def identify_client(request)
+        # Prioritize token-based identification for authenticated requests
+        token = request.env['HTTP_X_AUTH_TOKEN']
+        return "token:#{token}" if token && !token.empty?
+
+        # Fall back to IP address
+        ip = request.ip || request.env['REMOTE_ADDR'] || 'unknown'
+        "ip:#{ip}"
+      end
+
+      def classify_endpoint(request)
+        path = request.path
+        method = request.request_method
+
+        return :vm_creation if method == 'POST' && path.include?('/vm')
+        return :vm_deletion if method == 'DELETE' && path.include?('/vm')
+        return :authenticated if request.env['HTTP_X_AUTH_TOKEN']
+
+        :global_per_ip
+      end
+
+      def rate_limit_exceeded?(client_id, endpoint_type, _request)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        current_count = @redis.get(key).to_i
+        current_count >= limit_config[:limit]
+      rescue StandardError => e
+        # If Redis fails, allow the request through (fail open)
+        warn "Rate limiter Redis error: #{e.message}"
+        false
+      end
+
+      def increment_request_count(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        @redis.pipelined do |pipeline|
+          pipeline.incr(key)
+          pipeline.expire(key, limit_config[:period])
+        end
+      rescue StandardError => e
+        # Log error but don't fail the request
+        warn "Rate limiter increment error: #{e.message}"
+      end
+
+      def rate_limit_response(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        begin
+          ttl = @redis.ttl(key)
+        rescue StandardError
+          ttl = limit_config[:period]
+        end
+
+        headers = {
+          'Content-Type' => 'application/json',
+          'X-RateLimit-Limit' => limit_config[:limit].to_s,
+          'X-RateLimit-Remaining' => '0',
+          'X-RateLimit-Reset' => (Time.now.to_i + ttl).to_s,
+          'Retry-After' => ttl.to_s
+        }
+
+        body = JSON.pretty_generate({
+                                      'ok' => false,
+                                      'error' => 'Rate limit exceeded',
+                                      'limit' => limit_config[:limit],
+                                      'period' => limit_config[:period],
+                                      'retry_after' => ttl
+                                    })
+
+        [429, headers, [body]]
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/request_logger.rb b/lib/vmpooler/api/request_logger.rb
new file mode 100644
index 0000000..d31c190
--- /dev/null
+++ b/lib/vmpooler/api/request_logger.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    class RequestLogger
+      attr_reader :app
+
+      def initialize(app, options = {})
+        @app = app
+        @logger = options[:logger]
+      end
+
+      def call(env)
+        status, headers, body = @app.call(env)
+        @logger.log('s', "[ ] API: Method: #{env['REQUEST_METHOD']}, Status: #{status}, Path: #{env['PATH_INFO']}, Body: #{body}")
+        [status, headers, body]
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/reroute.rb b/lib/vmpooler/api/reroute.rb
deleted file mode 100644
index 9c68663..0000000
--- a/lib/vmpooler/api/reroute.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-module Vmpooler
-  class API
-    class Reroute < Sinatra::Base
-      api_version = '1'
-
-      get '/status/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/status")
-      end
-
-      get '/summary/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary")
-      end
-
-      get '/summary/:route/?:key?/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}")
-      end
-
-      get '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      post '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      get '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      delete '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      get '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/:template/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}")
-      end
-
-      get '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      delete '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      put '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      post '/vm/:hostname/snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot")
-      end
-
-      post '/vm/:hostname/snapshot/:snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}")
-      end
-
-      put '/vm/:hostname/disk/:size/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}")
-      end
-    end
-  end
-end
diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb
deleted file mode 100644
index 4b84e3e..0000000
--- a/lib/vmpooler/api/v1.rb
+++ /dev/null
@@ -1,1119 +0,0 @@
-module Vmpooler
-  class API
-  class V1 < Sinatra::Base
-    api_version = '1'
-    api_prefix  = "/api/v#{api_version}"
-
-    helpers do
-      include Vmpooler::API::Helpers
-    end
-
-    def backend
-      Vmpooler::API.settings.redis
-    end
-
-    def metrics
-      Vmpooler::API.settings.metrics
-    end
-
-    def config
-      Vmpooler::API.settings.config[:config]
-    end
-
-    def pools
-      Vmpooler::API.settings.config[:pools]
-    end
-
-    def pool_exists?(template)
-      Vmpooler::API.settings.config[:pool_names].include?(template)
-    end
-
-    def need_auth!
-      validate_auth(backend)
-    end
-
-    def need_token!
-      validate_token(backend)
-    end
-
-    def fetch_single_vm(template)
-      template_backends = [template]
-      aliases = Vmpooler::API.settings.config[:alias]
-      if aliases
-        template_backends = template_backends + aliases[template] if aliases[template].is_a?(Array)
-        template_backends << aliases[template] if aliases[template].is_a?(String)
-        pool_index = pool_index(pools)
-        weighted_pools = {}
-        template_backends.each do |t|
-          next unless pool_index.key? t
-          index = pool_index[t]
-          clone_target = pools[index]['clone_target'] || config['clone_target']
-          next unless config.key?('backend_weight')
-          weight = config['backend_weight'][clone_target]
-          if weight
-            weighted_pools[t] = weight
-          end
-        end
-
-        if weighted_pools.count == template_backends.count
-          pickup = Pickup.new(weighted_pools)
-          selection = pickup.pick
-          template_backends.delete(selection)
-          template_backends.unshift(selection)
-        else
-          first = template_backends.sample
-          template_backends.delete(first)
-          template_backends.unshift(first)
-        end
-      end
-
-      template_backends.each do |template_backend|
-        vms = backend.smembers("vmpooler__ready__#{template_backend}")
-        next if vms.empty?
-        vms.reverse.each do |vm|
-          ready = vm_ready?(vm, config['domain'])
-          if ready
-            backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
-            return [vm, template_backend, template]
-          else
-            backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
-            metrics.increment("checkout.nonresponsive.#{template_backend}")
-          end
-        end
-      end
-      [nil, nil, nil]
-    end
-
-    def return_vm_to_ready_state(template, vm)
-      backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
-    end
-
-    def account_for_starting_vm(template, vm)
-      backend.sadd("vmpooler__migrating__#{template}", vm)
-      backend.hset("vmpooler__active__#{template}", vm, Time.now)
-      backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
-
-      if Vmpooler::API.settings.config[:auth] and has_token?
-        validate_token(backend)
-
-        backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
-        backend.hset('vmpooler__vm__' + vm, 'token:user',
-          backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')
-        )
-
-        if config['vm_lifetime_auth'].to_i > 0
-          backend.hset('vmpooler__vm__' + vm, 'lifetime', config['vm_lifetime_auth'].to_i)
-        end
-      end
-    end
-
-    def update_result_hosts(result, template, vm)
-      result[template] ||= {}
-      if result[template]['hostname']
-        result[template]['hostname'] = Array(result[template]['hostname'])
-        result[template]['hostname'].push(vm)
-      else
-        result[template]['hostname'] = vm
-      end
-    end
-
-    def atomically_allocate_vms(payload)
-      result = { 'ok' => false }
-      failed = false
-      vms = []
-
-      payload.each do |requested, count|
-        count.to_i.times do |_i|
-          vmname, vmpool, vmtemplate = fetch_single_vm(requested)
-          if !vmname
-            failed = true
-            metrics.increment('checkout.empty.' + requested)
-            break
-          else
-            vms << [ vmpool, vmname, vmtemplate ]
-            metrics.increment('checkout.success.' + vmtemplate)
-          end
-        end
-      end
-
-      if failed
-        vms.each do |(vmpool, vmname, vmtemplate)|
-          return_vm_to_ready_state(vmpool, vmname)
-        end
-        status 503
-      else
-        vms.each do |(vmpool, vmname, vmtemplate)|
-          account_for_starting_vm(vmpool, vmname)
-          update_result_hosts(result, vmtemplate, vmname)
-        end
-
-        result['ok'] = true
-        result['domain'] = config['domain'] if config['domain']
-      end
-
-      result
-    end
-
-    def update_pool_size(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_sizes
-
-      payload.each do |poolname, size|
-        unless pools[pool_index[poolname]]['size'] == size.to_i
-          pools[pool_index[poolname]]['size'] = size.to_i
-          backend.hset('vmpooler__config__poolsize', poolname, size)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def update_pool_template(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_templates
-
-      payload.each do |poolname, template|
-        unless pools[pool_index[poolname]]['template'] == template
-          pools[pool_index[poolname]]['template'] = template
-          backend.hset('vmpooler__config__template', poolname, template)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def update_clone_target(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_clone_targets
-
-      payload.each do |poolname, clone_target|
-        unless pools[pool_index[poolname]]['clone_target'] == clone_target
-          pools[pool_index[poolname]]['clone_target'] == clone_target
-          backend.hset('vmpooler__config__clone_target', poolname, clone_target)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def sync_pool_templates
-      pool_index = pool_index(pools)
-      template_configs = backend.hgetall('vmpooler__config__template')
-      unless template_configs.nil?
-        template_configs.each do |poolname, template|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['template'] == template
-              pools[pool_index[poolname]]['template'] = template
-            end
-          end
-        end
-      end
-    end
-
-    def sync_pool_sizes
-      pool_index = pool_index(pools)
-      poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
-      unless poolsize_configs.nil?
-        poolsize_configs.each do |poolname, size|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['size'] == size.to_i
-              pools[pool_index[poolname]]['size'] == size.to_i
-            end
-          end
-        end
-      end
-    end
-
-    def sync_clone_targets
-      pool_index = pool_index(pools)
-      clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
-      unless clone_target_configs.nil?
-        clone_target_configs.each do |poolname, clone_target|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['clone_target'] == clone_target
-              pools[pool_index[poolname]]['clone_target'] == clone_target
-              end
-          end
-        end
-      end
-    end
-
-    get '/' do
-      sync_pool_sizes
-      redirect to('/dashboard/')
-    end
-
-    # Provide run-time statistics
-    #
-    # Example:
-    #
-    # {
-    #   "boot": {
-    #     "duration": {
-    #       "average": 163.6,
-    #       "min": 65.49,
-    #       "max": 830.07,
-    #       "total": 247744.71000000002
-    #     },
-    #     "count": {
-    #       "total": 1514
-    #     }
-    #   },
-    #   "capacity": {
-    #     "current": 968,
-    #     "total": 975,
-    #     "percent": 99.3
-    #   },
-    #   "clone": {
-    #     "duration": {
-    #       "average": 17.0,
-    #       "min": 4.66,
-    #       "max": 637.96,
-    #       "total": 25634.15
-    #     },
-    #     "count": {
-    #       "total": 1507
-    #     }
-    #   },
-    #   "queue": {
-    #     "pending": 12,
-    #     "cloning": 0,
-    #     "booting": 12,
-    #     "ready": 968,
-    #     "running": 367,
-    #     "completed": 0,
-    #     "total": 1347
-    #   },
-    #   "pools": {
-    #     "ready": 100,
-    #     "running": 120,
-    #     "pending": 5,
-    #     "max": 250,
-    #   }
-    #   "status": {
-    #     "ok": true,
-    #     "message": "Battle station fully armed and operational.",
-    #     "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools
-    #       "redhat-7-x86_64",
-    #       "ubuntu-1404-i386"
-    #     ],
-    #     "uptime": 179585.9
-    # }
-    #
-    # If the query parameter 'view' is provided, it will be used to select which top level
-    # element to compute and return. Select them by specifying them in a comma separated list.
-    # For example /status?view=capacity,boot
-    # would return only the "capacity" and "boot" statistics. "status" is always returned
-
-    get "#{api_prefix}/status/?" do
-      content_type :json
-
-      if params[:view]
-        views = params[:view].split(",")
-      end
-
-      result = {
-        status: {
-          ok: true,
-          message: 'Battle station fully armed and operational.'
-        }
-      }
-
-      sync_pool_sizes
-
-      result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
-      result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
-      result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
-      result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot")
-
-      # Check for empty pools
-      result[:pools] = {} unless views and not views.include?("pools")
-      ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-      running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-      pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
-      lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
-
-      pools.each do |pool|
-        # REMIND: move this out of the API and into the back-end
-        ready    = ready_hash[pool['name']]
-        running  = running_hash[pool['name']]
-        pending  = pending_hash[pool['name']]
-        max      = pool['size']
-        lastBoot = lastBoot_hash[pool['name']]
-        aka      = pool['alias']
-
-        result[:pools][pool['name']] = {
-          ready:    ready,
-          running:  running,
-          pending:  pending,
-          max:      max,
-          lastBoot: lastBoot
-        }
-
-        if aka
-          result[:pools][pool['name']][:alias] = aka
-        end
-
-        # for backwards compatibility, include separate "empty" stats in "status" block
-        if ready == 0
-          result[:status][:empty] ||= []
-          result[:status][:empty].push(pool['name'])
-
-          result[:status][:ok] = false
-          result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
-        end
-      end unless views and not views.include?("pools")
-
-      result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
-
-      JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
-    end
-
-    # request statistics for specific pools by passing parameter 'pool'
-    # with a coma separated list of pools we want to query ?pool=ABC,DEF
-    # returns the ready, max numbers and the aliases (if set)
-    get "#{api_prefix}/poolstat/?" do
-      content_type :json
-
-      result = {}
-
-      poolscopy = []
-
-      if params[:pool]
-        subpool = params[:pool].split(",")
-        poolscopy = pools.select do |p|
-          if subpool.include?(p['name'])
-            true
-          elsif !p['alias'].nil?
-            if p['alias'].is_a?(Array)
-              (p['alias'] & subpool).any?
-            elsif p['alias'].is_a?(String)
-              subpool.include?(p['alias'])
-            end
-          end
-        end
-      end
-
-      result[:pools] = {}
-
-      poolscopy.each do |pool|
-        result[:pools][pool['name']] = {}
-
-        max      = pool['size']
-        aka      = pool['alias']
-
-        result[:pools][pool['name']][:max] = max
-
-        if aka
-          result[:pools][pool['name']][:alias] = aka
-        end
-
-      end
-
-      ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
-
-      ready_hash.each { |k, v| result[:pools][k][:ready] = v }
-
-      JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
-    end
-
-    # requests the total number of running VMs
-    get "#{api_prefix}/totalrunning/?" do
-      content_type :json
-      queue = {
-          running:   0,
-      }
-
-      queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-
-      JSON.pretty_generate(queue)
-    end
-
-    get "#{api_prefix}/summary/?" do
-      content_type :json
-
-      result = {
-        daily: []
-      }
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true)
-      clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true)
-      tag = get_tag_summary(backend, from_date, to_date)
-
-      result[:boot] = boot[:boot]
-      result[:clone] = clone[:clone]
-      result[:tag] = tag[:tag]
-
-      daily = {}
-
-      boot[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:boot] = day[:boot]
-      end
-
-      clone[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:clone] = day[:clone]
-      end
-
-      tag[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:tag] = day[:tag]
-      end
-
-      daily.each_key do |day|
-        result[:daily].push({
-          date: day,
-          boot: daily[day][:boot],
-          clone: daily[day][:clone],
-          tag: daily[day][:tag]
-        })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/summary/:route/?:key?/?" do
-      content_type :json
-
-      result = {}
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      case params[:route]
-        when 'boot'
-          result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'clone'
-          result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'tag'
-          result = get_tag_summary(backend, from_date, to_date, :only => params[:key])
-        else
-          halt 404, JSON.pretty_generate({ 'ok' => false })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        backend.keys('vmpooler__token__*').each do |key|
-          data = backend.hgetall(key)
-
-          if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
-            token = key.split('__').last
-
-            result[token] ||= {}
-
-            result[token]['created'] = data['created']
-            result[token]['last'] = data['last'] || 'never'
-
-            result['ok'] = true
-          end
-        end
-
-        if result['ok']
-          status 200
-        else
-          status 404
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        token = backend.hgetall('vmpooler__token__' + params[:token])
-
-        if not token.nil? and not token.empty?
-          status 200
-
-          pools.each do |pool|
-            backend.smembers('vmpooler__running__' + pool['name']).each do |vm|
-              if backend.hget('vmpooler__vm__' + vm, 'token:token') == params[:token]
-                token['vms'] ||= {}
-                token['vms']['running'] ||= []
-                token['vms']['running'].push(vm)
-              end
-            end
-          end
-
-          result = { 'ok' => true, params[:token] => token }
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        if backend.del('vmpooler__token__' + params[:token]).to_i > 0
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/token" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.hset('vmpooler__token__' + result['token'], 'user', @auth.username)
-        backend.hset('vmpooler__token__' + result['token'], 'created', Time.now)
-
-        status 200
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/?" do
-      content_type :json
-
-      result = []
-
-      pools.each do |pool|
-        result.push(pool['name'])
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      payload = JSON.parse(request.body.read)
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    def extract_templates_from_query_params(params)
-      payload = {}
-
-      params.split('+').each do |template|
-        payload[template] ||= 0
-        payload[template] += 1
-      end
-
-      payload
-    end
-
-    def invalid_templates(payload)
-      invalid = []
-      payload.keys.each do |template|
-        invalid << template unless pool_exists?(template)
-      end
-      invalid
-    end
-
-    def invalid_template_or_size(payload)
-      invalid = []
-      payload.each do |pool, size|
-        invalid << pool unless pool_exists?(pool)
-        unless is_integer?(size)
-          invalid << pool
-          next
-        end
-        invalid << pool unless Integer(size) >= 0
-      end
-      invalid
-    end
-
-    def invalid_template_or_path(payload)
-      invalid = []
-      payload.each do |pool, template|
-        invalid << pool unless pool_exists?(pool)
-        invalid << pool unless template.include? '/'
-        invalid << pool if template[0] == '/'
-        invalid << pool if template[-1] == '/'
-      end
-      invalid
-    end
-
-    def invalid_pool(payload)
-      invalid = []
-      payload.each do |pool, clone_target|
-        invalid << pool unless pool_exists?(pool)
-      end
-      invalid
-    end
-
-    post "#{api_prefix}/vm/:template/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      payload = extract_templates_from_query_params(params[:template])
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        status 200
-        result['ok'] = true
-
-        result[params[:hostname]] = {}
-
-        result[params[:hostname]]['template'] = rdata['template']
-        result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
-
-        if rdata['destroy']
-          result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['state'] = 'destroyed'
-        elsif rdata['checkout']
-          result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
-          result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
-          result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
-          result[params[:hostname]]['state'] = 'running'
-        elsif rdata['check']
-          result[params[:hostname]]['state'] = 'ready'
-        else
-          result[params[:hostname]]['state'] = 'pending'
-        end
-
-        rdata.keys.each do |key|
-          if key.match('^tag\:(.+?)$')
-            result[params[:hostname]]['tags'] ||= {}
-            result[params[:hostname]]['tags'][$1] = rdata[key]
-          end
-
-          if key.match('^snapshot\:(.+?)$')
-            result[params[:hostname]]['snapshots'] ||= []
-            result[params[:hostname]]['snapshots'].push($1)
-          end
-        end
-
-        if rdata['disk']
-          result[params[:hostname]]['disk'] = rdata['disk'].split(':')
-        end
-
-        # Look up IP address of the hostname
-        begin
-          ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
-        rescue
-          ipAddress = ""
-        end
-
-        result[params[:hostname]]['ip'] = ipAddress
-
-        if config['domain']
-          result[params[:hostname]]['domain'] = config['domain']
-        end
-
-        result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
-        result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
-
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        need_token! if rdata['token:token']
-
-        if backend.srem('vmpooler__running__' + rdata['template'], params[:hostname])
-          backend.sadd('vmpooler__completed__' + rdata['template'], params[:hostname])
-
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    put "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      failure = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists('vmpooler__vm__' + params[:hostname])
-        begin
-          jdata = JSON.parse(request.body.read)
-        rescue
-          halt 400, JSON.pretty_generate(result)
-        end
-
-        # Validate data payload
-        jdata.each do |param, arg|
-          case param
-            when 'lifetime'
-              need_token! if Vmpooler::API.settings.config[:auth]
-
-              unless arg.to_i > 0
-                failure = true
-              end
-            when 'tags'
-              unless arg.is_a?(Hash)
-                failure = true
-              end
-
-              if config['allowed_tags']
-                failure = true if not (arg.keys - config['allowed_tags']).empty?
-              end
-            else
-              failure = true
-          end
-        end
-
-        if failure
-          status 400
-        else
-          jdata.each do |param, arg|
-            case param
-              when 'lifetime'
-                need_token! if Vmpooler::API.settings.config[:auth]
-
-                arg = arg.to_i
-
-                backend.hset('vmpooler__vm__' + params[:hostname], param, arg)
-              when 'tags'
-                filter_tags(arg)
-                export_tags(backend, params[:hostname], arg)
-            end
-          end
-
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/disk/:size/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if ((params[:size].to_i > 0 )and (backend.exists('vmpooler__vm__' + params[:hostname])))
-        result[params[:hostname]] = {}
-        result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
-
-        backend.sadd('vmpooler__tasks__disk', params[:hostname] + ':' + params[:size])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists('vmpooler__vm__' + params[:hostname])
-        result[params[:hostname]] = {}
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.sadd('vmpooler__tasks__snapshot', params[:hostname] + ':' + result[params[:hostname]]['snapshot'])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
-      content_type :json
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      unless backend.hget('vmpooler__vm__' + params[:hostname], 'snapshot:' + params[:snapshot]).to_i.zero?
-        backend.sadd('vmpooler__tasks__snapshot-revert', params[:hostname] + ':' + params[:snapshot])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/poolsize/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_size(payload)
-          if invalid.empty?
-            result = update_pool_size(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/pooltemplate/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_path(payload)
-          if invalid.empty?
-            result = update_pool_template(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/clonetarget/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_pool(payload)
-          if invalid.empty?
-            result = update_clone_target(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/config/?" do
-      content_type :json
-      result = { 'ok' => false }
-      status 404
-
-      if pools
-        sync_pool_sizes
-        sync_pool_templates
-
-        pool_configuration = []
-        pools.each do |pool|
-          pool['template_ready'] = template_ready?(pool, backend)
-          pool_configuration << pool
-        end
-
-        result = {
-          pool_configuration: pool_configuration,
-          status: {
-            ok: true
-          }
-        }
-
-        status 200
-      end
-      JSON.pretty_generate(result)
-    end
-  end
-  end
-end
diff --git a/lib/vmpooler/api/v3.rb b/lib/vmpooler/api/v3.rb
new file mode 100644
index 0000000..21bc4e3
--- /dev/null
+++ b/lib/vmpooler/api/v3.rb
@@ -0,0 +1,1876 @@
+# frozen_string_literal: true
+
+require 'vmpooler/util/parsing'
+require 'vmpooler/dns'
+
+module Vmpooler
+  class API
+    class V3 < Sinatra::Base
+      api_version = '3'
+      api_prefix  = "/api/v#{api_version}"
+
+      # Simple in-memory cache for status endpoint
+      # rubocop:disable Style/ClassVars
+      @@status_cache = {}
+      @@status_cache_mutex = Mutex.new
+      # rubocop:enable Style/ClassVars
+      STATUS_CACHE_TTL = 30 # seconds
+
+      # Clear cache (useful for testing)
+      def self.clear_status_cache
+        @@status_cache_mutex.synchronize do
+          @@status_cache.clear
+        end
+      end
+
+      helpers do
+        include Vmpooler::API::Helpers
+      end
+
+      def backend
+        Vmpooler::API.settings.redis
+      end
+
+      def metrics
+        Vmpooler::API.settings.metrics
+      end
+
+      def config
+        Vmpooler::API.settings.config[:config]
+      end
+
+      def full_config
+        Vmpooler::API.settings.config
+      end
+
+      def pools
+        Vmpooler::API.settings.config[:pools]
+      end
+
+      def pools_at_startup
+        Vmpooler::API.settings.config[:pools_at_startup]
+      end
+
+      def pool_exists?(template)
+        Vmpooler::API.settings.config[:pool_names].include?(template)
+      end
+
+      def need_auth!
+        validate_auth(backend)
+      end
+
+      def need_token!
+        validate_token(backend)
+      end
+
+      def checkoutlock
+        Vmpooler::API.settings.checkoutlock
+      end
+
+      def get_template_aliases(template)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = []
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            result += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+          end
+          result
+        end
+      end
+
+      def get_pool_weights(template_backends)
+        pool_index = pool_index(pools)
+        weighted_pools = {}
+        template_backends.each do |t|
+          next unless pool_index.key? t
+
+          index = pool_index[t]
+          clone_target = pools[index]['clone_target'] || config['clone_target']
+          next unless config.key?('backend_weight')
+
+          weight = config['backend_weight'][clone_target]
+          if weight
+            weighted_pools[t] = weight
+          end
+        end
+        weighted_pools
+      end
+
+      def count_selection(selection)
+        result = {}
+        selection.uniq.each do |poolname|
+          result[poolname] = selection.count(poolname)
+        end
+        result
+      end
+
+      def evaluate_template_aliases(template, count)
+        template_backends = []
+        template_backends << template if backend.sismember('vmpooler__pools', template)
+        selection = []
+        aliases = get_template_aliases(template)
+        if aliases
+          template_backends += aliases
+          weighted_pools = get_pool_weights(template_backends)
+
+          if weighted_pools.count > 1 && weighted_pools.count == template_backends.count
+            pickup = Pickup.new(weighted_pools)
+            count.to_i.times do
+              selection << pickup.pick
+            end
+          else
+            count.to_i.times do
+              selection << template_backends.sample
+            end
+          end
+        end
+
+        count_selection(selection)
+      end
+
+      # Fetch a single vm from a pool
+      #
+      # @param [String] template
+      #   The template that the vm should be created from
+      #
+      # @return [Tuple] vmname, vmpool, vmtemplate
+      #   Returns a tuple containing the vm's name, the pool it came from, and
+      #   what template was used, if successful. Otherwise the tuple contains.
+      #   nil values.
+      def fetch_single_vm(template)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          template_backends = [template]
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            template_backends += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+            pool_index = pool_index(pools)
+            weighted_pools = {}
+            template_backends.each do |t|
+              next unless pool_index.key? t
+
+              index = pool_index[t]
+              clone_target = pools[index]['clone_target'] || config['clone_target']
+              next unless config.key?('backend_weight')
+
+              weight = config['backend_weight'][clone_target]
+              if weight
+                weighted_pools[t] = weight
+              end
+            end
+
+            if weighted_pools.count == template_backends.count
+              pickup = Pickup.new(weighted_pools)
+              selection = pickup.pick
+              template_backends.delete(selection)
+              template_backends.unshift(selection)
+            else
+              first = template_backends.sample
+              template_backends.delete(first)
+              template_backends.unshift(first)
+            end
+          end
+
+          checkoutlock.synchronize do
+            template_backends.each do |template_backend|
+              vms = backend.smembers("vmpooler__ready__#{template_backend}")
+              next if vms.empty?
+
+              vm = vms.pop
+              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+              if smoved
+                return [vm, template_backend, template]
+              end
+            end
+            [nil, nil, nil]
+          end
+        end
+      end
+
+      def return_vm_to_ready_state(template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          backend.srem("vmpooler__migrating__#{template}", vm)
+          backend.hdel("vmpooler__active__#{template}", vm)
+          backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
+          backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
+        end
+      end
+
+      def account_for_starting_vm(template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
+          span.set_attribute('enduser.id', user)
+          has_token_result = has_token?
+          backend.sadd("vmpooler__migrating__#{template}", vm)
+          backend.hset("vmpooler__active__#{template}", vm, Time.now.to_s)
+          backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now.to_s)
+
+          if Vmpooler::API.settings.config[:auth] and has_token_result
+            backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
+            backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
+
+            if config['vm_lifetime_auth'].to_i > 0
+              backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
+            end
+          end
+        end
+      end
+
+      def update_result_hosts(result, template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result[template] ||= {}
+          if result[template]['hostname']
+            result[template]['hostname'] = Array(result[template]['hostname'])
+            result[template]['hostname'].push(vm)
+          else
+            result[template]['hostname'] = vm
+          end
+        end
+      end
+
+      def atomically_allocate_vms(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          result = { 'ok' => false }
+          failed = false
+          vms = [] # vmpool, vmname, vmtemplate
+
+          validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
+
+          payload.each do |requested, count|
+            count.to_i.times do |_i|
+              vmname, vmpool, vmtemplate = fetch_single_vm(requested)
+              if vmname
+                account_for_starting_vm(vmpool, vmname)
+                vms << [vmpool, vmname, vmtemplate]
+                metrics.increment("checkout.success.#{vmpool}")
+                update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
+              else
+                failed = true
+                metrics.increment("checkout.empty.#{requested}")
+                break
+              end
+            end
+          end
+
+          if failed
+            vms.each do |(vmpool, vmname, _vmtemplate)|
+              return_vm_to_ready_state(vmpool, vmname)
+            end
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.atomically_allocate_vms',
+              'error.message' => '503 due to failing to allocate one or more vms'
+            })
+            status 503
+          else
+            vm_names = []
+            vms.each do |(vmpool, vmname, vmtemplate)|
+              vmdomain = Dns.get_domain_for_pool(full_config, vmpool)
+              vmfqdn = "#{vmname}.#{vmdomain}"
+              update_result_hosts(result, vmtemplate, vmfqdn)
+              vm_names.append(vmfqdn)
+            end
+
+            span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
+
+            result['ok'] = true
+          end
+
+          result
+        end
+      end
+
+      def component_to_test(match, labels_string)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          return if labels_string.nil?
+
+          labels_string_parts = labels_string.split(',')
+          labels_string_parts.each do |part|
+            key, value = part.split('=')
+            next if value.nil?
+            return value if key == match
+          end
+          'none'
+        end
+      end
+
+      def update_user_metrics(operation, vmname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          begin
+            jenkins_build_url = backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
+            user = backend.hget("vmpooler__vm__#{vmname}", 'token:user')
+            poolname = backend.hget("vmpooler__vm__#{vmname}", 'template')
+            poolname = poolname.gsub('.', '_')
+
+            if user
+              user = user.gsub('.', '_')
+            else
+              user = 'unauthenticated'
+            end
+            metrics.increment("user.#{user}.#{operation}.#{poolname}")
+
+            if jenkins_build_url
+              if jenkins_build_url.include? 'litmus'
+                # Very simple filter for Litmus jobs - just count them coming through for the moment.
+                metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
+              else
+                url_parts = jenkins_build_url.split('/')[2..]
+                jenkins_instance = url_parts[0].gsub('.', '_')
+                value_stream_parts = url_parts[2].split('_')
+                value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
+                value_stream = value_stream_parts.shift
+                branch = value_stream_parts.pop
+                project = value_stream_parts.shift
+                job_name = value_stream_parts.join('_')
+                build_metadata_parts = url_parts[3]
+                component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
+
+                metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
+                metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
+                metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
+              end
+            end
+          rescue StandardError => e
+            puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
+            span.record_exception(e)
+            span.status = OpenTelemetry::Trace::Status.error(e.to_s)
+            span.add_event('log', attributes: {
+              'log.severity' => 'debug',
+              'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
+            })
+          end
+        end
+      end
+
+      def reset_pool_size(poolname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+
+          pools_updated = 0
+          sync_pool_sizes
+
+          pool_size_now = pools[pool_index[poolname]]['size'].to_i
+          pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
+          result['pool_size_before_reset'] = pool_size_now
+          result['pool_size_before_overrides'] = pool_size_original
+
+          unless pool_size_now == pool_size_original
+            pools[pool_index[poolname]]['size'] = pool_size_original
+            backend.hdel('vmpooler__config__poolsize', poolname)
+            backend.sadd('vmpooler__pool__undo_size_override', poolname)
+            pools_updated += 1
+            status 201
+          end
+
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_pool_size(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_pool_sizes
+
+          payload.each do |poolname, size|
+            unless pools[pool_index[poolname]]['size'] == size.to_i
+              pools[pool_index[poolname]]['size'] = size.to_i
+              backend.hset('vmpooler__config__poolsize', poolname, size)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def reset_pool_template(poolname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index_live = pool_index(pools)
+          pool_index_original = pool_index(pools_at_startup)
+
+          pools_updated = 0
+          sync_pool_templates
+
+          template_now = pools[pool_index_live[poolname]]['template']
+          template_original = pools_at_startup[pool_index_original[poolname]]['template']
+          result['template_before_reset'] = template_now
+          result['template_before_overrides'] = template_original
+
+          unless template_now == template_original
+            pools[pool_index_live[poolname]]['template'] = template_original
+            backend.hdel('vmpooler__config__template', poolname)
+            backend.sadd('vmpooler__pool__undo_template_override', poolname)
+            pools_updated += 1
+            status 201
+          end
+
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_pool_template(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_pool_templates
+
+          payload.each do |poolname, template|
+            unless pools[pool_index[poolname]]['template'] == template
+              pools[pool_index[poolname]]['template'] = template
+              backend.hset('vmpooler__config__template', poolname, template)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def reset_pool(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          payload.each do |poolname, _count|
+            backend.sadd('vmpooler__poolreset', poolname)
+          end
+          status 201
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_clone_target(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_clone_targets
+
+          payload.each do |poolname, clone_target|
+            unless pools[pool_index[poolname]]['clone_target'] == clone_target
+              pools[pool_index[poolname]]['clone_target'] = clone_target
+              backend.hset('vmpooler__config__clone_target', poolname, clone_target)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      # Cache helper methods for status endpoint
+      def get_cached_status(cache_key)
+        @@status_cache_mutex.synchronize do
+          cached = @@status_cache[cache_key]
+          if cached && (Time.now - cached[:timestamp]) < STATUS_CACHE_TTL
+            return cached[:data]
+          end
+
+          nil
+        end
+      end
+
+      def set_cached_status(cache_key, data)
+        @@status_cache_mutex.synchronize do
+          @@status_cache[cache_key] = {
+            data: data,
+            timestamp: Time.now
+          }
+          # Cleanup old cache entries (keep only last 10 unique view combinations)
+          if @@status_cache.size > 10
+            oldest = @@status_cache.min_by { |_k, v| v[:timestamp] }
+            @@status_cache.delete(oldest[0])
+          end
+        end
+      end
+
+      def sync_pool_templates
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          template_configs = backend.hgetall('vmpooler__config__template')
+          template_configs&.each do |poolname, template|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['template'] = template
+          end
+        end
+      end
+
+      def sync_pool_sizes
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
+          poolsize_configs&.each do |poolname, size|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['size'] = size.to_i
+          end
+        end
+      end
+
+      def sync_clone_targets
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
+          clone_target_configs&.each do |poolname, clone_target|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['clone_target'] = clone_target
+          end
+        end
+      end
+
+      def too_many_requested?(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          payload&.each do |poolname, count|
+            next unless count.to_i > config['max_ondemand_instances_per_request']
+
+            metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
+            return true
+          end
+          false
+        end
+      end
+
+      def generate_ondemand_request(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          result = { 'ok': false }
+
+          requested_instances = payload.reject { |k, _v| k == 'request_id' }
+          if too_many_requested?(requested_instances)
+            e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+            result['message'] = e_message
+            status 403
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.generate_ondemand_request',
+              'error.message' => "403 due to #{e_message}"
+            })
+            return result
+          end
+
+          score = Time.now.to_i
+          request_id = payload['request_id']
+          request_id ||= generate_request_id
+          result['request_id'] = request_id
+          span.set_attribute('vmpooler.request_id', request_id)
+
+          if backend.exists?("vmpooler__odrequest__#{request_id}")
+            e_message = "request_id '#{request_id}' has already been created"
+            result['message'] = e_message
+            status 409
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.generate_ondemand_request',
+              'error.message' => "409 due to #{e_message}"
+            })
+            metrics.increment('ondemandrequest_generate.duplicaterequests')
+            return result
+          end
+
+          status 201
+
+          platforms_with_aliases = []
+          requested_instances.each do |poolname, count|
+            selection = evaluate_template_aliases(poolname, count)
+            selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+          end
+          platforms_string = platforms_with_aliases.join(',')
+
+          return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+          backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+          if Vmpooler::API.settings.config[:auth] and has_token?
+            token_token = request.env['HTTP_X_AUTH_TOKEN']
+            token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
+            span.set_attribute('enduser.id', token_user)
+          end
+
+          result[:ok] = true
+          metrics.increment('ondemandrequest_generate.success')
+          result
+        end
+      end
+
+      def generate_request_id
+        SecureRandom.uuid
+      end
+
+      get '/' do
+        sync_pool_sizes
+        redirect to('/dashboard/')
+      end
+
+      # Provide run-time statistics
+      #
+      # Example:
+      #
+      # {
+      #   "boot": {
+      #     "duration": {
+      #       "average": 163.6,
+      #       "min": 65.49,
+      #       "max": 830.07,
+      #       "total": 247744.71000000002
+      #     },
+      #     "count": {
+      #       "total": 1514
+      #     }
+      #   },
+      #   "capacity": {
+      #     "current": 968,
+      #     "total": 975,
+      #     "percent": 99.3
+      #   },
+      #   "clone": {
+      #     "duration": {
+      #       "average": 17.0,
+      #       "min": 4.66,
+      #       "max": 637.96,
+      #       "total": 25634.15
+      #     },
+      #     "count": {
+      #       "total": 1507
+      #     }
+      #   },
+      #   "queue": {
+      #     "pending": 12,
+      #     "cloning": 0,
+      #     "booting": 12,
+      #     "ready": 968,
+      #     "running": 367,
+      #     "completed": 0,
+      #     "total": 1347
+      #   },
+      #   "pools": {
+      #     "ready": 100,
+      #     "running": 120,
+      #     "pending": 5,
+      #     "max": 250,
+      #   }
+      #   "status": {
+      #     "ok": true,
+      #     "message": "Battle station fully armed and operational.",
+      #     "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools
+      #       "redhat-7-x86_64",
+      #       "ubuntu-1404-i386"
+      #     ],
+      #     "uptime": 179585.9
+      # }
+      #
+      # If the query parameter 'view' is provided, it will be used to select which top level
+      # element to compute and return. Select them by specifying them in a comma separated list.
+      # For example /status?view=capacity,boot
+      # would return only the "capacity" and "boot" statistics. "status" is always returned
+
+      get "#{api_prefix}/status/?" do
+        content_type :json
+
+        # Create cache key based on view parameters
+        cache_key = params[:view] ? "status_#{params[:view]}" : "status_all"
+
+        # Try to get cached response
+        cached_response = get_cached_status(cache_key)
+        return cached_response if cached_response
+
+        if params[:view]
+          views = params[:view].split(",")
+        end
+
+        result = {
+          status: {
+            ok: true,
+            message: 'Battle station fully armed and operational.'
+          }
+        }
+
+        sync_pool_sizes
+
+        result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
+        result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
+        result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
+        result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot")
+
+        # Check for empty pools
+        result[:pools] = {} unless views and not views.include?("pools")
+        ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+        running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+        pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
+        lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
+
+        unless views and not views.include?("pools")
+          pools.each do |pool|
+            # REMIND: move this out of the API and into the back-end
+            ready    = ready_hash[pool['name']]
+            running  = running_hash[pool['name']]
+            pending  = pending_hash[pool['name']]
+            max      = pool['size']
+            lastBoot = lastBoot_hash[pool['name']]
+            aka      = pool['alias']
+
+            result[:pools][pool['name']] = {
+              ready: ready,
+              running: running,
+              pending: pending,
+              max: max,
+              lastBoot: lastBoot
+            }
+
+            if aka
+              result[:pools][pool['name']][:alias] = aka
+            end
+
+            # for backwards compatibility, include separate "empty" stats in "status" block
+            if ready == 0 && max != 0
+              result[:status][:empty] ||= []
+              result[:status][:empty].push(pool['name'])
+
+              result[:status][:ok] = false
+              result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
+            end
+          end
+        end
+
+        result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
+
+        response = JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+
+        # Cache the response
+        set_cached_status(cache_key, response)
+
+        response
+      end
+
+      # request statistics for specific pools by passing parameter 'pool'
+      # with a coma separated list of pools we want to query ?pool=ABC,DEF
+      # returns the ready, max numbers and the aliases (if set)
+      get "#{api_prefix}/poolstat/?" do
+        content_type :json
+
+        result = {}
+
+        poolscopy = []
+
+        if params[:pool]
+          subpool = params[:pool].split(",")
+          poolscopy = pools.select do |p|
+            if subpool.include?(p['name'])
+              true
+            elsif !p['alias'].nil?
+              if p['alias'].instance_of?(Array)
+                (p['alias'] & subpool).any?
+              elsif p['alias'].instance_of?(String)
+                subpool.include?(p['alias'])
+              end
+            end
+          end
+        end
+
+        result[:pools] = {}
+
+        poolscopy.each do |pool|
+          result[:pools][pool['name']] = {}
+
+          max      = pool['size']
+          aka      = pool['alias']
+
+          result[:pools][pool['name']][:max] = max
+
+          if aka
+            result[:pools][pool['name']][:alias] = aka
+          end
+        end
+
+        ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
+
+        ready_hash.each { |k, v| result[:pools][k][:ready] = v }
+
+        JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+      end
+
+      # requests the total number of running VMs
+      get "#{api_prefix}/totalrunning/?" do
+        content_type :json
+        queue = {
+            running: 0
+        }
+
+        queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+
+        JSON.pretty_generate(queue)
+      end
+
+      get "#{api_prefix}/summary/?" do
+        content_type :json
+
+        result = {
+          daily: []
+        }
+
+        from_param = params[:from] || Date.today.to_s
+        to_param   = params[:to]   || Date.today.to_s
+
+        # Validate date formats
+        [from_param, to_param].each do |param|
+          if !validate_date_str(param.to_s)
+            halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
+          end
+        end
+
+        from_date, to_date = Date.parse(from_param), Date.parse(to_param)
+
+        if to_date < from_date
+          halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
+        elsif from_date > Date.today
+          halt 400, 'Date range is invalid, \'from\' must be in the past.'
+        end
+
+        boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true)
+        clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true)
+        tag = get_tag_summary(backend, from_date, to_date)
+
+        result[:boot] = boot[:boot]
+        result[:clone] = clone[:clone]
+        result[:tag] = tag[:tag]
+
+        daily = {}
+
+        boot[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:boot] = day[:boot]
+        end
+
+        clone[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:clone] = day[:clone]
+        end
+
+        tag[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:tag] = day[:tag]
+        end
+
+        daily.each_key do |day|
+          result[:daily].push({
+            date: day,
+            boot: daily[day][:boot],
+            clone: daily[day][:clone],
+            tag: daily[day][:tag]
+          })
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/summary/:route/?:key?/?" do
+        content_type :json
+
+        result = {}
+
+        from_param = params[:from] || Date.today.to_s
+        to_param   = params[:to]   || Date.today.to_s
+
+        # Validate date formats
+        [from_param, to_param].each do |param|
+          if !validate_date_str(param.to_s)
+            halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
+          end
+        end
+
+        from_date, to_date = Date.parse(from_param), Date.parse(to_param)
+
+        if to_date < from_date
+          halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
+        elsif from_date > Date.today
+          halt 400, 'Date range is invalid, \'from\' must be in the past.'
+        end
+
+        case params[:route]
+          when 'boot'
+            result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key])
+          when 'clone'
+            result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key])
+          when 'tag'
+            result = get_tag_summary(backend, from_date, to_date, :only => params[:key])
+          else
+            halt 404, JSON.pretty_generate({ 'ok' => false })
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          backend.keys('vmpooler__token__*').each do |key|
+            data = backend.hgetall(key)
+
+            if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
+              span = OpenTelemetry::Trace.current_span
+              span.set_attribute('enduser.id', data['user'])
+              token = key.split('__').last
+
+              result[token] ||= {}
+
+              result[token]['created'] = data['created']
+              result[token]['last'] = data['last'] || 'never'
+
+              result['ok'] = true
+            end
+          end
+
+          if result['ok']
+            status 200
+          else
+            status 404
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/token/:token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          token = backend.hgetall("vmpooler__token__#{params[:token]}")
+
+          if not token.nil? and not token.empty?
+            status 200
+
+            pools.each do |pool|
+              backend.smembers("vmpooler__running__#{pool['name']}").each do |vm|
+                if backend.hget("vmpooler__vm__#{vm}", 'token:token') == params[:token]
+                  token['vms'] ||= {}
+                  token['vms']['running'] ||= []
+                  token['vms']['running'].push(vm)
+                end
+              end
+            end
+
+            result = { 'ok' => true, params[:token] => token }
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/token/:token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          if backend.del("vmpooler__token__#{params[:token]}").to_i > 0
+            status 200
+            result['ok'] = true
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/token" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
+          result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
+
+          backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username)
+          backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now.to_s)
+          span = OpenTelemetry::Trace.current_span
+          span.set_attribute('enduser.id', @auth.username)
+
+          status 200
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/vm/?" do
+        content_type :json
+
+        result = []
+
+        pools.each do |pool|
+          result.push(pool['name'])
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.ondemand.requestid')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        result = { 'ok' => false }
+
+        begin
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+            if invalid.empty?
+              result = generate_ondemand_request(payload)
+            else
+              result[:bad_templates] = invalid
+              invalid.each do |bad_template|
+                metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+              end
+              status 404
+            end
+          else
+            metrics.increment('ondemandrequest_fail.invalid.unknown')
+            status 404
+          end
+        rescue JSON::ParserError
+          span = OpenTelemetry::Trace.current_span
+          span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
+          status 400
+          result = {
+            'ok' => false,
+            'message' => 'JSON payload could not be parsed'
+          }
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.delete.ondemand.template')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest_fail.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/ondemandvm/:requestid/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.ondemand.request')
+
+        status 404
+        result = check_ondemand_request(params[:requestid])
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/ondemandvm/:requestid/?" do
+        content_type :json
+        need_token! if Vmpooler::API.settings.config[:auth]
+        metrics.increment('http_requests_vm_total.delete.ondemand.request')
+
+        status 404
+        result = delete_ondemand_request(params[:requestid])
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.post.vm.checkout')
+
+        # Validate and sanitize JSON body
+        payload = sanitize_json_body(request.body.read)
+        if validation_error?(payload)
+          status 400
+          return JSON.pretty_generate(payload)
+        end
+
+        # Validate each template and count
+        payload.each do |template, count|
+          validation = validate_pool_name(template)
+          if validation_error?(validation)
+            status 400
+            return JSON.pretty_generate(validation)
+          end
+
+          validated_count = validate_vm_count(count)
+          if validation_error?(validated_count)
+            status 400
+            return JSON.pretty_generate(validated_count)
+          end
+        end
+
+        if payload && !payload.empty?
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      def extract_templates_from_query_params(params)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          payload = {}
+
+          params.split('+').each do |template|
+            payload[template] ||= 0
+            payload[template] += 1
+          end
+
+          payload
+        end
+      end
+
+      def invalid_templates(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.keys.each do |template|
+            invalid << template unless pool_exists?(template)
+          end
+          invalid
+        end
+      end
+
+      def invalid_template_or_size(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, size|
+            invalid << pool unless pool_exists?(pool)
+            unless is_integer?(size)
+              invalid << pool
+              next
+            end
+            invalid << pool unless Integer(size) >= 0
+          end
+          invalid
+        end
+      end
+
+      def invalid_template_or_path(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, template|
+            invalid << pool unless pool_exists?(pool)
+            invalid << pool unless template.include? '/'
+            invalid << pool if template[0] == '/'
+            invalid << pool if template[-1] == '/'
+          end
+          invalid
+        end
+      end
+
+      def invalid_pool(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, _clone_target|
+            invalid << pool unless pool_exists?(pool)
+          end
+          invalid
+        end
+      end
+
+      def delete_ondemand_request(request_id)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          span.set_attribute('vmpooler.request_id', request_id)
+          result = { 'ok' => false }
+
+          platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
+          unless platforms
+            e_message = "no request found for request_id '#{request_id}'"
+            result['message'] = e_message
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.delete_ondemand_request',
+              'error.message' => e_message
+            })
+            return result
+          end
+
+          if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
+            result['message'] = 'the request has already been deleted'
+          else
+            backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+
+            Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
+              backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
+                backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
+              end
+              backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+            end
+            backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
+          end
+          status 200
+          result['ok'] = true
+          result
+        end
+      end
+
+      post "#{api_prefix}/vm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.get.vm.template')
+
+        # Template can contain multiple pools separated by +, so validate after parsing
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.vm.hostname')
+
+        result = {}
+
+        status 404
+        result['ok'] = false
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
+        unless rdata.empty?
+          status 200
+          result['ok'] = true
+
+          result[params[:hostname]] = {}
+
+          result[params[:hostname]]['template'] = rdata['template']
+          result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
+
+          if rdata['destroy']
+            result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout']
+            result[params[:hostname]]['state'] = 'destroyed'
+          elsif rdata['checkout']
+            result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
+            result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
+            result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
+            result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
+            result[params[:hostname]]['state'] = 'running'
+          elsif rdata['check']
+            result[params[:hostname]]['state'] = 'ready'
+          else
+            result[params[:hostname]]['state'] = 'pending'
+          end
+
+          rdata.keys.each do |key|
+            if key.match('^tag\:(.+?)$')
+              result[params[:hostname]]['tags'] ||= {}
+              result[params[:hostname]]['tags'][$1] = rdata[key]
+            end
+
+            if key.match('^snapshot\:(.+?)$')
+              result[params[:hostname]]['snapshots'] ||= []
+              result[params[:hostname]]['snapshots'].push($1)
+            end
+          end
+
+          if rdata['disk']
+            result[params[:hostname]]['disk'] = rdata['disk'].split(':')
+          end
+
+          # Look up IP address of the hostname
+          begin
+            ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
+          rescue StandardError
+            ipAddress = ""
+          end
+
+          result[params[:hostname]]['ip'] = ipAddress
+
+          if rdata['pool']
+            vmdomain = Dns.get_domain_for_pool(full_config, rdata['pool'])
+            if vmdomain
+              result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}"
+            end
+          end
+
+          result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
+          result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
+
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      def check_ondemand_request(request_id)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          span.set_attribute('vmpooler.request_id', request_id)
+          result = { 'ok' => false }
+          request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+          if request_hash.empty?
+            e_message = "no request found for request_id '#{request_id}'"
+            result['message'] = e_message
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.check_ondemand_request',
+              'error.message' => e_message
+            })
+            return result
+          end
+
+          result['request_id'] = request_id
+          result['ready'] = false
+          result['ok'] = true
+          status 202
+
+          case request_hash['status']
+          when 'ready'
+            result['ready'] = true
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+              instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              domain = Dns.get_domain_for_pool(full_config, pool)
+              instances.map! { |instance| instance.concat(".#{domain}") }
+
+              if result.key?(platform_alias)
+                result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
+              else
+                result[platform_alias] = { 'hostname': instances }
+              end
+            end
+            status 200
+          when 'failed'
+            result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+            status 200
+          when 'deleted'
+            result['message'] = 'The request has been deleted'
+            status 200
+          else
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+              instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              instances_pending = count.to_i - instance_count.to_i
+
+              if result.key?(platform_alias) && result[platform_alias].key?(:ready)
+                result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
+                result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
+              else
+                result[platform_alias] = {
+                  'ready': instance_count.to_s,
+                  'pending': instances_pending.to_s
+                }
+              end
+            end
+          end
+
+          result
+        end
+      end
+
+      delete "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.delete.vm.hostname')
+
+        result = {}
+
+        status 404
+        result['ok'] = false
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
+        unless rdata.empty?
+          need_token! if rdata['token:token']
+
+          if backend.srem("vmpooler__running__#{rdata['template']}", params[:hostname])
+            backend.sadd("vmpooler__completed__#{rdata['template']}", params[:hostname])
+
+            status 200
+            result['ok'] = true
+            metrics.increment('delete.success')
+            update_user_metrics('destroy', params[:hostname]) if Vmpooler::API.settings.config[:config]['usage_stats']
+          else
+            metrics.increment('delete.failed')
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      put "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.put.vm.modify')
+
+        status 404
+        result = { 'ok' => false }
+
+        failure = []
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          # Validate and sanitize JSON body
+          jdata = sanitize_json_body(request.body.read)
+          if validation_error?(jdata)
+            status 400
+            return JSON.pretty_generate(jdata)
+          end
+
+          # Validate data payload
+          jdata.each do |param, arg|
+            case param
+              when 'lifetime'
+                need_token! if Vmpooler::API.settings.config[:auth]
+
+                # Validate lifetime is a positive integer
+                lifetime_int = arg.to_i
+                if lifetime_int <= 0
+                  failure.push("Lifetime must be a positive integer (got #{arg})")
+                  next
+                end
+
+                # in hours, defaults to one week
+                max_lifetime_upper_limit = config['max_lifetime_upper_limit']
+                if max_lifetime_upper_limit
+                  max_lifetime_upper_limit = max_lifetime_upper_limit.to_i
+                  if arg.to_i >= max_lifetime_upper_limit
+                    failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.")
+                  end
+                end
+
+              when 'tags'
+                failure.push("You provided tags (#{arg}) as something other than a hash.") unless arg.is_a?(Hash)
+
+                # Validate each tag key and value
+                arg.each do |key, value|
+                  tag_validation = validate_tag(key, value)
+                  if validation_error?(tag_validation)
+                    failure.push(tag_validation['error'])
+                  end
+                end
+
+                failure.push("You provided unsuppored tags (#{arg}).") if config['allowed_tags'] && !(arg.keys - config['allowed_tags']).empty?
+              else
+                failure.push("Unknown argument #{arg}.")
+            end
+          end
+
+          if !failure.empty?
+            status 400
+            result['failure'] = failure
+          else
+            jdata.each do |param, arg|
+              case param
+                when 'lifetime'
+                  need_token! if Vmpooler::API.settings.config[:auth]
+
+                  arg = arg.to_i
+
+                  backend.hset("vmpooler__vm__#{params[:hostname]}", param, arg)
+                when 'tags'
+                  filter_tags(arg)
+                  export_tags(backend, params[:hostname], arg)
+              end
+            end
+
+            status 200
+            result['ok'] = true
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/disk/:size/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.disksize')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        # Validate disk size
+        validated_size = validate_disk_size(params[:size])
+        if validation_error?(validated_size)
+          status 400
+          return JSON.pretty_generate(validated_size)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          result[params[:hostname]] = {}
+          result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
+
+          backend.sadd('vmpooler__tasks__disk', "#{params[:hostname]}:#{params[:size]}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/snapshot/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.snapshot')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          result[params[:hostname]] = {}
+
+          o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
+          result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
+
+          backend.sadd('vmpooler__tasks__snapshot', "#{params[:hostname]}:#{result[params[:hostname]]['snapshot']}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.snapshot')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        unless backend.hget("vmpooler__vm__#{params[:hostname]}", "snapshot:#{params[:snapshot]}").to_i.zero?
+          backend.sadd('vmpooler__tasks__snapshot-revert', "#{params[:hostname]}:#{params[:snapshot]}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/config/poolsize/:pool/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          if pool_exists?(params[:pool])
+            result = reset_pool_size(params[:pool])
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/poolsize/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_template_or_size(payload)
+            if invalid.empty?
+              result = update_pool_size(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:not_configured] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/config/pooltemplate/:pool/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          if pool_exists?(params[:pool])
+            result = reset_pool_template(params[:pool])
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/pooltemplate/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_template_or_path(payload)
+            if invalid.empty?
+              result = update_pool_template(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:bad_templates] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/poolreset/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          begin
+            payload = JSON.parse(request.body.read)
+            if payload
+              invalid = invalid_templates(payload)
+              if invalid.empty?
+                result = reset_pool(payload)
+              else
+                invalid.each do |bad_pool|
+                  metrics.increment("poolreset.invalid.#{bad_pool}")
+                end
+                result[:bad_pools] = invalid
+                status 400
+              end
+            else
+              metrics.increment('poolreset.invalid.unknown')
+              status 404
+            end
+          rescue JSON::ParserError
+            span = OpenTelemetry::Trace.current_span
+            span.record_exception(e)
+            span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
+            status 400
+            result = {
+              'ok' => false,
+              'message' => 'JSON payload could not be parsed'
+            }
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/clonetarget/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_pool(payload)
+            if invalid.empty?
+              result = update_clone_target(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:bad_templates] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/config/?" do
+        content_type :json
+        result = { 'ok' => false }
+        status 404
+
+        if pools
+          sync_pool_sizes
+          sync_pool_templates
+
+          pool_configuration = []
+          pools.each do |pool|
+            pool['template_ready'] = template_ready?(pool, backend)
+            pool_configuration << pool
+          end
+
+          result = {
+            pool_configuration: pool_configuration,
+            status: {
+              ok: true
+            }
+          }
+
+          status 200
+        end
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/full_config/?" do
+        content_type :json
+
+        result = {
+          full_config: full_config,
+          status: {
+            ok: true
+          }
+        }
+
+        status 200
+        JSON.pretty_generate(result)
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/dashboard.rb b/lib/vmpooler/dashboard.rb
index b875465..8182409 100644
--- a/lib/vmpooler/dashboard.rb
+++ b/lib/vmpooler/dashboard.rb
@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class Dashboard < Sinatra::Base
-
     def config
       Vmpooler.config
     end
diff --git a/lib/vmpooler/dns.rb b/lib/vmpooler/dns.rb
new file mode 100644
index 0000000..823fe17
--- /dev/null
+++ b/lib/vmpooler/dns.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Vmpooler
+  class Dns
+    # Load one or more VMPooler DNS plugin gems by name
+    #
+    # @param names [Array<String>] The list of gem names to load
+    def self.load_by_name(names)
+      names = Array(names)
+      instance = new
+      names.map { |name| instance.load_from_gems(name) }.flatten
+    end
+
+    # Returns the plugin class for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns class
+    # @return [String] The plugin class for the specifid dns config
+    def self.get_dns_plugin_class_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      plugin_class = ''
+
+      dns_configs.map do |dns_config_name|
+        plugin_class = config[:dns_configs][dns_config_name]['dns_class'] if dns_config_name.to_s == name
+      end
+
+      plugin_class
+    end
+
+    # Returns the domain for the specified pool
+    #
+    # @param config [String] - the full config structure
+    # @param pool_name [String] - the name of the pool
+    # @return [String] - domain name for pool, which is set via reference to the dns_configs block
+    def self.get_domain_for_pool(config, pool_name)
+      pool = config[:pools].find { |p| p['name'] == pool_name }
+      pool_dns_config = pool['dns_plugin']
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == pool_dns_config
+      end
+    end
+
+    # Returns the plugin domain for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns domain
+    # @return [String] The domain for the specifid dns config
+    def self.get_dns_plugin_domain_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == name
+      end
+    end
+
+    # Returns a list of DNS plugin classes specified in the vmpooler configuration
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @return nil || [Array<String>] A list of DNS plugin classes
+    def self.get_dns_plugin_config_classes(config)
+      return nil unless config[:dns_configs]
+
+      dns_configs = config[:dns_configs].keys
+      dns_plugins = dns_configs.map do |dns_config_name|
+        if config[:dns_configs][dns_config_name] && config[:dns_configs][dns_config_name]['dns_class']
+          config[:dns_configs][dns_config_name]['dns_class'].to_s
+        else
+          dns_config_name.to_s
+        end
+      end.compact.uniq
+
+      # dynamic-dns is not actually a class, it's just used as a value to denote
+      # that dynamic dns is used so no loading or record management is needed
+      dns_plugins.delete('dynamic-dns')
+
+      dns_plugins
+    end
+
+    # Load a single DNS plugin gem by name
+    #
+    # @param name [String] The name of the DNS plugin gem to load
+    # @return [String] The full require path to the specified gem
+    def load_from_gems(name = nil)
+      require_path = "vmpooler/dns/#{name.gsub('-', '/')}"
+      require require_path
+      require_path
+    end
+  end
+end
diff --git a/lib/vmpooler/dns/base.rb b/lib/vmpooler/dns/base.rb
new file mode 100644
index 0000000..61403ef
--- /dev/null
+++ b/lib/vmpooler/dns/base.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class PoolManager
+    class Dns
+      class Base
+        # These defs must be overidden in child classes
+
+        # Helper Methods
+        # Global Logger object
+        attr_reader :logger
+        # Global Metrics object
+        attr_reader :metrics
+        # Provider options passed in during initialization
+        attr_reader :dns_options
+
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          @config = config
+          @logger = logger
+          @metrics = metrics
+          @redis = redis_connection_pool
+          @dns_plugin_name = name
+
+          @dns_options = options
+
+          logger.log('s', "[!] Creating dns plugin '#{name}'")
+        end
+
+        def pool_config(pool_name)
+          # Get the configuration of a specific pool
+          @config[:pools].each do |pool|
+            return pool if pool['name'] == pool_name
+          end
+
+          nil
+        end
+
+        # Returns this dns plugin's configuration
+        #
+        # @returns [Hashtable] This dns plugins's configuration from the config file.  Returns nil if the dns plugin config does not exist
+        def dns_config
+          @config[:dns_configs].each do |dns|
+            # Convert the symbol from the config into a string for comparison
+            return (dns[1].nil? ? {} : dns[1]) if dns[0].to_s == @dns_plugin_name
+          end
+
+          nil
+        end
+
+        def global_config
+          # This entire VM Pooler config
+          @config
+        end
+
+        def name
+          @dns_plugin_name
+        end
+
+        def get_ip(vm_name)
+          @redis.with_metrics do |redis|
+            redis.hget("vmpooler__vm__#{vm_name}", 'ip')
+          end
+        end
+
+        # returns
+        #   Array[String] : Array of pool names this provider services
+        def provided_pools
+          @config[:pools].select { |pool| pool['dns_config'] == name }.map { |pool| pool['name'] }
+        end
+
+        def create_or_replace_record(hostname)
+          raise("#{self.class.name} does not implement create_or_replace_record #{hostname}")
+        end
+
+        def delete_record(hostname)
+          raise("#{self.class.name} does not implement delete_record for #{hostname}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/dummy_statsd.rb b/lib/vmpooler/dummy_statsd.rb
deleted file mode 100644
index a555268..0000000
--- a/lib/vmpooler/dummy_statsd.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-module Vmpooler
-  class DummyStatsd
-    attr_reader :server, :port, :prefix
-
-    def initialize(*)
-    end
-
-    def increment(*)
-      true
-    end
-
-    def gauge(*)
-      true
-    end
-
-    def timing(*)
-      true
-    end
-  end
-end
diff --git a/lib/vmpooler/generic_connection_pool.rb b/lib/vmpooler/generic_connection_pool.rb
index ca18576..5299b23 100644
--- a/lib/vmpooler/generic_connection_pool.rb
+++ b/lib/vmpooler/generic_connection_pool.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'connection_pool'
 
 module Vmpooler
@@ -9,42 +11,26 @@ module Vmpooler
       def initialize(options = {}, &block)
         super(options, &block)
         @metrics = options[:metrics]
-        @metric_prefix = options[:metric_prefix]
-        @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
+        @connpool_type = options[:connpool_type]
+        @connpool_type = 'connectionpool' if @connpool_type.nil? || @connpool_type == ''
+        @connpool_provider = options[:connpool_provider]
+        @connpool_provider = 'unknown' if @connpool_provider.nil? || @connpool_provider == ''
       end
 
-      if Thread.respond_to?(:handle_interrupt)
-        # MRI
-        def with_metrics(options = {})
-          Thread.handle_interrupt(Exception => :never) do
-            start = Time.now
-            conn = checkout(options)
-            timespan_ms = ((Time.now - start) * 1000).to_i
-            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-            @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
-            begin
-              Thread.handle_interrupt(Exception => :immediate) do
-                yield conn
-              end
-            ensure
-              checkin
-              @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-            end
-          end
-        end
-      else
-        # jruby 1.7.x
-        def with_metrics(options = {})
+      def with_metrics(options = {})
+        Thread.handle_interrupt(Exception => :never) do
           start = Time.now
           conn = checkout(options)
           timespan_ms = ((Time.now - start) * 1000).to_i
-          @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
-          @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
+          @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
+          @metrics&.timing("connection_waited.#{@connpool_type}.#{@connpool_provider}", timespan_ms)
           begin
-            yield conn
+            Thread.handle_interrupt(Exception => :immediate) do
+              yield conn
+            end
           ensure
             checkin
-            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+            @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
           end
         end
       end
diff --git a/lib/vmpooler/graphite.rb b/lib/vmpooler/graphite.rb
deleted file mode 100644
index e7f28c3..0000000
--- a/lib/vmpooler/graphite.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-module Vmpooler
-  class Graphite
-    attr_reader :server, :port, :prefix
-
-    def initialize(params = {})
-      if params['server'].nil? || params['server'].empty?
-        raise ArgumentError, "Graphite server is required. Config: #{params.inspect}"
-      end
-
-      @server = params['server']
-      @port   = params['port'] || 2003
-      @prefix = params['prefix'] || 'vmpooler'
-    end
-
-    def increment(label)
-      log label, 1
-    end
-
-    def gauge(label, value)
-      log label, value
-    end
-
-    def timing(label, duration)
-      log label, duration
-    end
-
-    def log(path, value)
-      Thread.new do
-        socket = TCPSocket.new(server, port)
-        begin
-          socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
-        ensure
-          socket.close
-        end
-      end
-    rescue => err
-      $stderr.puts "Failure logging #{path} to graphite server [#{server}:#{port}]: #{err}"
-    end
-  end
-end
diff --git a/lib/vmpooler/logger.rb b/lib/vmpooler/logger.rb
index f481246..218ec4c 100644
--- a/lib/vmpooler/logger.rb
+++ b/lib/vmpooler/logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems' unless defined?(Gem)
 
 module Vmpooler
@@ -14,7 +16,7 @@ module Vmpooler
 
       puts "[#{stamp}] #{string}" if ENV['VMPOOLER_DEBUG']
 
-      open(@file, 'a') do |f|
+      File.open(@file, 'a') do |f|
         f.puts "[#{stamp}] #{string}"
       end
     end
diff --git a/lib/vmpooler/metrics.rb b/lib/vmpooler/metrics.rb
new file mode 100644
index 0000000..98c17f9
--- /dev/null
+++ b/lib/vmpooler/metrics.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'vmpooler/metrics/statsd'
+require 'vmpooler/metrics/graphite'
+require 'vmpooler/metrics/promstats'
+require 'vmpooler/metrics/dummy_statsd'
+
+module Vmpooler
+  class Metrics
+    # static class instantiate appropriate metrics object.
+    def self.init(logger, params)
+      if params[:statsd]
+        metrics = Vmpooler::Metrics::Statsd.new(logger, params[:statsd])
+      elsif params[:graphite]
+        metrics = Vmpooler::Metrics::Graphite.new(logger, params[:graphite])
+      elsif params[:prometheus]
+        metrics = Vmpooler::Metrics::Promstats.new(logger, params[:prometheus])
+      else
+        metrics = Vmpooler::Metrics::DummyStatsd.new
+      end
+      metrics
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/dummy_statsd.rb b/lib/vmpooler/metrics/dummy_statsd.rb
new file mode 100644
index 0000000..ad866b0
--- /dev/null
+++ b/lib/vmpooler/metrics/dummy_statsd.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class Metrics
+    class DummyStatsd < Metrics
+      attr_reader :server, :port, :prefix
+
+      def increment(*)
+        true
+      end
+
+      def gauge(*)
+        true
+      end
+
+      def timing(*)
+        true
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/graphite.rb b/lib/vmpooler/metrics/graphite.rb
new file mode 100644
index 0000000..88e66e3
--- /dev/null
+++ b/lib/vmpooler/metrics/graphite.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Metrics
+    class Graphite < Metrics
+      attr_reader :server, :port, :prefix
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
+
+        @server = params['server']
+        @port   = params['port'] || 2003
+        @prefix = params['prefix'] || 'vmpooler'
+        @logger = logger
+      end
+      # rubocop:enable Lint/MissingSuper
+
+      def increment(label)
+        log label, 1
+      end
+
+      def gauge(label, value)
+        log label, value
+      end
+
+      def timing(label, duration)
+        log label, duration
+      end
+
+      def log(path, value)
+        Thread.new do
+          socket = TCPSocket.new(server, port)
+          begin
+            socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
+          ensure
+            socket.close
+          end
+        end
+      rescue Errno::EADDRNOTAVAIL => e
+        warn "Could not assign address to graphite server #{server}: #{e}"
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}")
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/promstats.rb b/lib/vmpooler/metrics/promstats.rb
new file mode 100644
index 0000000..d0e1ab9
--- /dev/null
+++ b/lib/vmpooler/metrics/promstats.rb
@@ -0,0 +1,501 @@
+# frozen_string_literal: true
+
+require 'prometheus/client'
+
+module Vmpooler
+  class Metrics
+    class Promstats < Metrics
+      attr_reader :prefix, :prometheus_endpoint, :prometheus_prefix
+
+      # Constants for Metric Types
+      M_COUNTER   = 1
+      M_GAUGE     = 2
+      M_SUMMARY   = 3
+      M_HISTOGRAM = 4
+
+      # Customised Bucket set to use for the Pooler clone times set to more appropriate intervals.
+      POOLER_CLONE_TIME_BUCKETS = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 120.0, 180.0, 240.0, 300.0, 600.0].freeze
+      POOLER_READY_TIME_BUCKETS = [30.0, 60.0, 120.0, 180.0, 240.0, 300.0, 500.0, 800.0, 1200.0, 1600.0].freeze
+      # Same for redis connection times - this is the same as the current Prometheus Default.
+      # https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14
+      REDIS_CONNECT_BUCKETS = [1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 18.0, 23.0].freeze
+
+      @p_metrics = {}
+      @torun = []
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        @prefix = params['prefix'] || 'vmpooler'
+        @prometheus_prefix = params['prometheus_prefix'] || 'vmpooler'
+        @prometheus_endpoint = params['prometheus_endpoint'] || '/prometheus'
+        @logger = logger
+
+        # Setup up prometheus registry and data structures
+        @prometheus = Prometheus::Client.registry
+      end
+# rubocop:enable Lint/MissingSuper
+
+=begin # rubocop:disable Style/BlockComments
+      The Metrics table is used to register metrics and translate/interpret the incoming metrics.
+
+      This table describes all of the prometheus metrics that are recognised by the application.
+      The background documentation for defining metrics is at: https://prometheus.io/docs/introduction/
+      In particular, the naming practices should be adhered to: https://prometheus.io/docs/practices/naming/
+      The Ruby Client docs are also useful: https://github.com/prometheus/client_ruby
+
+      The table here allows the currently used stats definitions to be translated correctly for Prometheus.
+      The current format is of the form A.B.C, where the final fields may be actual values (e.g. poolname).
+      Prometheus metrics cannot use the '.' as a character, so this is either translated into '_' or
+      variable parameters are expressed as labels accompanying the metric.
+
+      Sample statistics are:
+          # Example showing hostnames (FQDN)
+          migrate_from.pix-jj26-chassis1-2.ops.puppetlabs.net
+          migrate_to.pix-jj26-chassis1-8.ops.puppetlabs.net
+
+          # Example showing poolname as a parameter
+          poolreset.invalid.centos-8-x86_64
+
+          # Examples showing similar sub-typed checkout stats
+          checkout.empty.centos-8-x86_64
+          checkout.invalid.centos-8-x86_64
+          checkout.invalid.unknown
+          checkout.success.centos-8-x86_64
+
+          # Stats without any final parameter.
+          connect.fail
+          connect.open
+          delete.failed
+          delete.success
+
+          # Stats with multiple param_labels
+          vmpooler_user.debian-8-x86_64-pixa4.john
+
+        The metrics implementation here preserves the existing framework which will continue to support
+        graphite and statsd (since vmpooler is used outside of puppet). Some rationalisation and renaming
+        of the actual metrics was done to get a more usable model to fit within the prometheus framework.
+        This particularly applies to the user stats collected once individual machines are terminated as
+        this would have challenged prometheus' ability due to multiple (8) parameters being collected
+        in a single measure (which has a very high cardinality).
+
+        Prometheus requires all metrics to be pre-registered (which is the primary reason for this
+        table) and also uses labels to differentiate the characteristics of the measurement. This
+        is used throughout to capture information such as poolnames. So for example, this is a sample
+        of the prometheus metrics generated for the "vmpooler_ready" measurement:
+
+          # TYPE vmpooler_ready gauge
+          # HELP vmpooler_ready vmpooler number of machines in ready State
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="win-10-ent-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="debian-8-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="centos-8-x86_64-pixa4"} 2.0
+
+        Prometheus supports the following metric types:
+        (see https://prometheus.io/docs/concepts/metric_types/)
+
+          Counter (increment):
+            A counter is a cumulative metric that represents a single monotonically increasing counter whose
+            value can only increase or be reset to zero on restart
+
+          Gauge:
+            A gauge is a metric that represents a single numerical value that can arbitrarily go up and down.
+
+          Histogram:
+            A histogram samples observations (usually things like request durations or response sizes) and
+            counts them in configurable buckets. It also provides a sum of all observed values.
+            This replaces the timer metric supported by statsd
+
+          Summary :
+            Summary provides a total count of observations and a sum of all observed values, it calculates
+            configurable quantiles over a sliding time window.
+            (Summary is not used in vmpooler)
+
+        vmpooler_metrics_table is a table of hashes, where the hash key represents the first part of the
+        metric name, e.g. for the metric 'delete.*' (see above) the key would be 'delete:'. "Sub-metrics",
+        are supported, again for the 'delete.*' example, this can be subbed into '.failed' and '.success'
+
+        The entries within the hash as are follows:
+
+          mtype:
+            Metric type, which is one of the following constants:
+              M_COUNTER   = 1
+              M_GAUGE     = 2
+              M_SUMMARY   = 3
+              M_HISTOGRAM = 4
+
+          torun:
+            Indicates which process the metric is for - within vmpooler this is either ':api' or ':manager'
+            (there is a suggestion that we change this to two separate tables).
+
+          docstring:
+            Documentation string for the metric - this is displayed as HELP text by the endpoint.
+
+          metric_suffixes:
+            Array of sub-metrics of the form 'sub-metric: "doc-string for sub-metric"'. This supports
+            the generation of individual sub-metrics for all elements in the array.
+
+          param_labels:
+            This is an optional array of symbols for the final labels in a metric. It should not be
+            specified if there are no additional parameters.
+
+            If it specified, it can either be a single symbol, or two or more symbols. The treatment
+            differs if there is only one symbol given as all of the remainder of the metric string
+            supplied is collected into a label with the symbol name. This allows the handling of
+            node names (FQDN).
+
+            To illustrate:
+            1. In the 'connect.*' or 'delete.*' example above, it should not be specified.
+            2. For the 'migrate_from.*' example above, the remainder of the measure is collected
+               as the 'host_name' label.
+            3. For the 'vmpooler_user' example above, the first parameter is treated as the pool
+               name, and the second as the username.
+
+=end
+      def vmpooler_metrics_table
+        {
+          errors: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'Count of errors for pool',
+            metric_suffixes: {
+              markedasfailed: 'timeout waiting for instance to initialise',
+              duplicatehostname: 'unable to create instance due to duplicate hostname',
+              staledns: 'unable to create instance due to duplicate DNS record'
+            },
+            param_labels: %i[template_name]
+          },
+          user: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by a user',
+            param_labels: %i[user operation poolname]
+          },
+          usage_litmus: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Litmus jobs',
+            param_labels: %i[user operation poolname]
+          },
+          usage_jenkins_instance: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins instances',
+            param_labels: %i[jenkins_instance value_stream operation poolname]
+          },
+          usage_branch_project: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins branch/project',
+            param_labels: %i[branch project operation poolname]
+          },
+          usage_job_component: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins job/component',
+            param_labels: %i[job_name component_to_test operation poolname]
+          },
+          checkout: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Pool checkout counts',
+            metric_suffixes: {
+              nonresponsive: 'checkout failed - non responsive machine',
+              empty: 'checkout failed - no machine',
+              success: 'successful checkout',
+              invalid: 'checkout failed - invalid template'
+            },
+            param_labels: %i[poolname]
+          },
+          delete: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Delete machine',
+            metric_suffixes: {
+              success: 'succeeded',
+              failed: 'failed'
+            },
+            param_labels: []
+          },
+          ondemandrequest_generate: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Ondemand request',
+            metric_suffixes: {
+              duplicaterequests: 'failed duplicate request',
+              success: 'succeeded'
+            },
+            param_labels: []
+          },
+          ondemandrequest_fail: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Ondemand request failure',
+            metric_suffixes: {
+              toomanyrequests: 'too many requests',
+              invalid: 'invalid poolname'
+            },
+            param_labels: %i[poolname]
+          },
+          config: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'vmpooler pool configuration request',
+            metric_suffixes: { invalid: 'Invalid' },
+            param_labels: %i[poolname]
+          },
+          poolreset: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Pool reset counter',
+            metric_suffixes: { invalid: 'Invalid Pool' },
+            param_labels: %i[poolname]
+          },
+          connect: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler connect (to vSphere)',
+            metric_suffixes: {
+              open: 'Connect Succeeded',
+              fail: 'Connect Failed'
+            },
+            param_labels: []
+          },
+          migrate_from: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler machine migrated from',
+            param_labels: %i[host_name]
+          },
+          migrate_to: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler machine migrated to',
+            param_labels: %i[host_name]
+          },
+          http_requests_vm_total: {
+            mtype: M_COUNTER,
+            torun: %i[api],
+            docstring: 'Total number of HTTP request/sub-operations handled by the Rack application under the /vm endpoint',
+            param_labels: %i[method subpath operation]
+          },
+          ready: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler number of machines in ready State',
+            param_labels: %i[poolname]
+          },
+          running: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler number of machines running',
+            param_labels: %i[poolname]
+          },
+          connection_available: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler redis connections available',
+            param_labels: %i[type provider]
+          },
+          time_to_ready_state: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_READY_TIME_BUCKETS,
+            docstring: 'Time taken for machine to read ready state for pool',
+            param_labels: %i[poolname]
+          },
+          migrate: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to migrate machine for pool',
+            param_labels: %i[poolname]
+          },
+          clone: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to clone machine',
+            param_labels: %i[poolname]
+          },
+          destroy: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: POOLER_CLONE_TIME_BUCKETS,
+            docstring: 'vmpooler time taken to destroy machine',
+            param_labels: %i[poolname]
+          },
+          connection_waited: {
+            mtype: M_HISTOGRAM,
+            torun: %i[manager],
+            buckets: REDIS_CONNECT_BUCKETS,
+            docstring: 'vmpooler redis connection wait time',
+            param_labels: %i[type provider]
+          },
+          vmpooler_health: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler health check metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_purge: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler purge metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_destroy: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler destroy metrics',
+            param_labels: %i[poolname]
+          },
+          vmpooler_clone: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler clone metrics',
+            param_labels: %i[poolname]
+          }
+        }
+      end
+
+      # Helper to add individual prom metric.
+      # Allow Histograms to specify the bucket size.
+      def add_prometheus_metric(metric_spec, name, docstring)
+        case metric_spec[:mtype]
+        when M_COUNTER
+          metric_class = Prometheus::Client::Counter
+        when M_GAUGE
+          metric_class = Prometheus::Client::Gauge
+        when M_SUMMARY
+          metric_class = Prometheus::Client::Summary
+        when M_HISTOGRAM
+          metric_class = Prometheus::Client::Histogram
+        else
+          raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}")
+        end
+
+        if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets)
+          prom_metric = metric_class.new(
+            name.to_sym,
+            docstring: docstring,
+            labels: metric_spec[:param_labels] + [:vmpooler_instance],
+            buckets: metric_spec[:buckets],
+            preset_labels: { vmpooler_instance: @prefix }
+          )
+        else
+          prom_metric = metric_class.new(
+            name.to_sym,
+            docstring: docstring,
+            labels: metric_spec[:param_labels] + [:vmpooler_instance],
+            preset_labels: { vmpooler_instance: @prefix }
+          )
+        end
+        @prometheus.register(prom_metric)
+      end
+
+      # Top level method to register all the prometheus metrics.
+
+      def setup_prometheus_metrics(torun)
+        @torun = torun
+        @p_metrics = vmpooler_metrics_table
+        @p_metrics.each do |name, metric_spec|
+          # Only register metrics appropriate to api or manager
+          next if (torun & metric_spec[:torun]).empty?
+
+          if metric_spec.key? :metric_suffixes
+            # Iterate thru the suffixes if provided to register multiple counters here.
+            metric_spec[:metric_suffixes].each do |metric_suffix|
+              add_prometheus_metric(
+                metric_spec,
+                "#{@prometheus_prefix}_#{name}_#{metric_suffix[0]}",
+                "#{metric_spec[:docstring]} #{metric_suffix[1]}"
+              )
+            end
+          else
+            # No Additional counter suffixes so register this as metric.
+            add_prometheus_metric(
+              metric_spec,
+              "#{@prometheus_prefix}_#{name}",
+              metric_spec[:docstring]
+            )
+          end
+        end
+      end
+
+      # locate a metric and check/interpet the sub-fields.
+      def find_metric(label)
+        sublabels = label.split('.')
+        metric_key = sublabels.shift.to_sym
+        raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key
+
+        metric_spec = @p_metrics[metric_key]
+        raise("Invalid Component #{component} for #{metric_key}") if (metric_spec[:torun] & @torun).nil?
+
+        metric = metric_spec.clone
+
+        if metric.key? :metric_suffixes
+          metric_subkey = sublabels.shift.to_sym
+          raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym
+
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}_#{metric_subkey}"
+        else
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}"
+        end
+
+        # Check if we are looking for a parameter value at last element.
+        if metric.key? :param_labels
+          metric[:labels] = {}
+          # Special case processing here - if there is only one parameter label then make sure
+          # we append all of the remaining contents of the metric with "." separators to ensure
+          # we get full nodenames (e.g. for Migration to node operations)
+          if metric[:param_labels].length == 1
+            metric[:labels][metric[:param_labels].first] = sublabels.join('.')
+          else
+            metric[:param_labels].reverse_each do |param_label|
+              metric[:labels][param_label] = sublabels.pop(1).first
+            end
+          end
+        end
+        metric
+      end
+
+      # Helper to get lab metrics.
+      def get(label)
+        metric = find_metric(label)
+        [metric, @prometheus.get(metric[:metric_name])]
+      end
+
+      # Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation.
+      def increment(label)
+        begin
+          counter_metric, c = get(label)
+          c.increment(labels: counter_metric[:labels])
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}")
+        end
+      end
+
+      def gauge(label, value)
+        begin
+          unless value.nil?
+            gauge_metric, g = get(label)
+            g.set(value.to_i, labels: gauge_metric[:labels])
+          end
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}")
+        end
+      end
+
+      def timing(label, duration)
+        begin
+          # https://prometheus.io/docs/practices/histograms/
+          unless duration.nil?
+            histogram_metric, hm = get(label)
+            hm.observe(duration.to_f, labels: histogram_metric[:labels])
+          end
+        rescue StandardError => e
+          @logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/promstats/collector_middleware.rb b/lib/vmpooler/metrics/promstats/collector_middleware.rb
new file mode 100644
index 0000000..c7e8325
--- /dev/null
+++ b/lib/vmpooler/metrics/promstats/collector_middleware.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# This is an adapted Collector module for vmpooler based on the sample implementation
+# available in the prometheus client_ruby library
+# https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/middleware/collector.rb
+#
+# The code was also failing Rubocop on PR check, so have addressed all the offenses.
+#
+# The method strip_hostnames_from_path (originally strip_ids_from_path) has been adapted
+# to add a match for hostnames in paths # to replace with a single ":hostname" string to
+# avoid # proliferation of stat lines for # each new vm hostname deleted, modified or
+# otherwise queried.
+
+require 'benchmark'
+require 'prometheus/client'
+require 'vmpooler/logger'
+
+module Vmpooler
+  class Metrics
+    class Promstats
+      # CollectorMiddleware is an implementation of Rack Middleware customised
+      # for vmpooler use.
+      #
+      # By default metrics are registered on the global registry. Set the
+      # `:registry` option to use a custom registry.
+      #
+      # By default metrics all have the prefix "http_server". Set to something
+      # else if you like.
+      #
+      # The request counter metric is broken down by code, method and path by
+      # default. Set the `:counter_label_builder` option to use a custom label
+      # builder.
+      #
+      # The request duration metric is broken down by method and path by default.
+      # Set the `:duration_label_builder` option to use a custom label builder.
+      #
+      # Label Builder functions will receive a Rack env and a status code, and must
+      # return a hash with the labels for that request. They must also accept an empty
+      # env, and return a hash with the correct keys. This is necessary to initialize
+      # the metrics with the correct set of labels.
+      class CollectorMiddleware
+        attr_reader :app, :registry
+
+        def initialize(app, options = {})
+          @app = app
+          @registry = options[:registry] || Prometheus::Client.registry
+          @metrics_prefix = options[:metrics_prefix] || 'http_server'
+
+          init_request_metrics
+          init_exception_metrics
+        end
+
+        def call(env) # :nodoc:
+          trace(env) { @app.call(env) }
+        end
+
+        protected
+
+        def init_request_metrics
+          @requests = @registry.counter(
+            :"#{@metrics_prefix}_requests_total",
+            docstring:
+              'The total number of HTTP requests handled by the Rack application.',
+            labels: %i[code method path]
+          )
+          @durations = @registry.histogram(
+            :"#{@metrics_prefix}_request_duration_seconds",
+            docstring: 'The HTTP response duration of the Rack application.',
+            labels: %i[method path]
+          )
+        end
+
+        def init_exception_metrics
+          @exceptions = @registry.counter(
+            :"#{@metrics_prefix}_exceptions_total",
+            docstring: 'The total number of exceptions raised by the Rack application.',
+            labels: [:exception]
+          )
+        end
+
+        def trace(env)
+          response = nil
+          duration = Benchmark.realtime { response = yield }
+          record(env, response.first.to_s, duration)
+          response
+        rescue StandardError => e
+          @exceptions.increment(labels: { exception: e.class.name })
+          raise
+        end
+
+        def record(env, code, duration)
+          counter_labels = {
+            code: code,
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          duration_labels = {
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          @requests.increment(labels: counter_labels)
+          @durations.observe(duration, labels: duration_labels)
+        rescue # rubocop:disable Style/RescueStandardError
+          nil
+        end
+
+        def strip_hostnames_from_path(path)
+          # Custom for /vm path - so we just collect aggrate stats for all usage along this one
+          # path. Custom counters are then added more specific endpoints in v1.rb
+          # Since we aren't parsing UID/GIDs as in the original example, these are removed.
+          # Similarly, request IDs are also stripped from the /ondemand path.
+          path
+            .gsub(%r{/vm/.+$}, '/vm')
+            .gsub(%r{/ondemandvm/.+$}, '/ondemandvm')
+            .gsub(%r{/token/.+$}, '/token')
+            .gsub(%r{/lib/.+$}, '/lib')
+            .gsub(%r{/img/.+$}, '/img')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/statsd.rb b/lib/vmpooler/metrics/statsd.rb
new file mode 100644
index 0000000..b469c38
--- /dev/null
+++ b/lib/vmpooler/metrics/statsd.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'rubygems' unless defined?(Gem)
+require 'statsd'
+
+module Vmpooler
+  class Metrics
+    class Statsd < Metrics
+      attr_reader :server, :port, :prefix
+
+      # rubocop:disable Lint/MissingSuper
+      def initialize(logger, params = {})
+        raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
+
+        host    = params['server']
+        @port   = params['port'] || 8125
+        @prefix = params['prefix'] || 'vmpooler'
+        @server = ::Statsd.new(host, @port)
+        @logger = logger
+      end
+      # rubocop:enable Lint/MissingSuper
+
+      def increment(label)
+        server.increment("#{prefix}.#{label}")
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def gauge(label, value)
+        server.gauge("#{prefix}.#{label}", value)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def timing(label, duration)
+        server.timing("#{prefix}.#{label}", duration)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
index f2242ff..9c6def6 100644
--- a/lib/vmpooler/pool_manager.rb
+++ b/lib/vmpooler/pool_manager.rb
@@ -1,4 +1,10 @@
+# frozen_string_literal: true
+
+require 'vmpooler/dns'
 require 'vmpooler/providers'
+require 'vmpooler/util/parsing'
+require 'spicy-proton'
+require 'resolv' # ruby standard lib
 
 module Vmpooler
   class PoolManager
@@ -6,7 +12,7 @@ module Vmpooler
     CHECK_LOOP_DELAY_MAX_DEFAULT = 60
     CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0
 
-    def initialize(config, logger, redis, metrics)
+    def initialize(config, logger, redis_connection_pool, metrics)
       $config = config
 
       # Load logger library
@@ -15,22 +21,31 @@ module Vmpooler
       # metrics logging handle
       $metrics = metrics
 
-      # Connect to Redis
-      $redis = redis
+      # Redis connection pool
+      @redis = redis_connection_pool
 
       # VM Provider objects
-      $providers = {}
+      $providers = Concurrent::Hash.new
+
+      # VM DNS objects
+      $dns_plugins = Concurrent::Hash.new
 
       # Our thread-tracker object
-      $threads = {}
+      $threads = Concurrent::Hash.new
 
       # Pool mutex
-      @reconfigure_pool = {}
+      @reconfigure_pool = Concurrent::Hash.new
 
-      @vm_mutex = {}
+      @vm_mutex = Concurrent::Hash.new
+
+      # Name generator for generating host names
+      @name_generator = Spicy::Proton.new
 
       # load specified providers from config file
       load_used_providers
+
+      # load specified dns plugins from config file
+      load_used_dns_plugins
     end
 
     def config
@@ -39,107 +54,258 @@ module Vmpooler
 
     # Place pool configuration in redis so an API instance can discover running pool configuration
     def load_pools_to_redis
-      previously_configured_pools = $redis.smembers('vmpooler__pools')
-      currently_configured_pools = []
-      config[:pools].each do |pool|
-        currently_configured_pools << pool['name']
-        $redis.sadd('vmpooler__pools', pool['name'])
-        pool_keys = pool.keys
-        pool_keys.delete('alias')
-        to_set = {}
-        pool_keys.each do |k|
-          to_set[k] = pool[k]
+      @redis.with_metrics do |redis|
+        previously_configured_pools = redis.smembers('vmpooler__pools')
+        currently_configured_pools = []
+        config[:pools].each do |pool|
+          currently_configured_pools << pool['name']
+          redis.sadd('vmpooler__pools', pool['name'].to_s)
+          pool_keys = pool.keys
+          pool_keys.delete('alias')
+          to_set = {}
+          pool_keys.each do |k|
+            to_set[k] = pool[k]
+          end
+          to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
+          to_set['domain'] = Vmpooler::Dns.get_domain_for_pool(config, pool['name'])
+
+          redis.hmset("vmpooler__pool__#{pool['name']}", *to_set.to_a.flatten) unless to_set.empty?
         end
-        to_set['alias'] = pool['alias'].join(',') if to_set.has_key?('alias')
-        $redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
-      end
-      previously_configured_pools.each do |pool|
-        unless currently_configured_pools.include? pool
-          $redis.srem('vmpooler__pools', pool)
-          $redis.del("vmpooler__pool__#{pool}")
+        previously_configured_pools.each do |pool|
+          unless currently_configured_pools.include? pool
+            redis.srem('vmpooler__pools', pool.to_s)
+            redis.del("vmpooler__pool__#{pool}")
+          end
         end
       end
-      return
+      nil
     end
 
     # Check the state of a VM
-    def check_pending_vm(vm, pool, timeout, provider)
+    def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       Thread.new do
         begin
-          _check_pending_vm(vm, pool, timeout, provider)
-        rescue => err
-          $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{err}")
-          fail_pending_vm(vm, pool, timeout)
+          _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
+          @redis.with_metrics do |redis|
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
+          end
           raise
         end
       end
     end
 
-    def _check_pending_vm(vm, pool, timeout, provider)
+    def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       mutex = vm_mutex(vm)
       return if mutex.locked?
+
       mutex.synchronize do
-        if provider.vm_ready?(pool, vm)
-          move_pending_vm_to_ready(vm, pool)
-        else
-          fail_pending_vm(vm, pool, timeout)
+        @redis.with_metrics do |redis|
+          request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+          if provider.vm_ready?(pool, vm, redis)
+            move_pending_vm_to_ready(vm, pool, redis, request_id)
+          else
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
+          end
         end
       end
     end
 
-    def remove_nonexistent_vm(vm, pool)
-      $redis.srem("vmpooler__pending__#{pool}", vm)
+    def remove_nonexistent_vm(vm, pool, redis)
+      redis.srem("vmpooler__pending__#{pool}", vm)
+      dns_plugin = get_dns_plugin_class_for_pool(pool)
+      dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool)
+      domain = get_dns_plugin_domain_for_pool(pool)
+      fqdn = "#{vm}.#{domain}"
+      dns_plugin.delete_record(fqdn) unless dns_plugin_class_name == 'dynamic-dns'
       $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
     end
 
-    def fail_pending_vm(vm, pool, timeout, exists = true)
-      clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone')
-      return true unless clone_stamp
-
+    def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
+      clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
-      if time_since_clone > timeout
-        if exists
-          $redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
-          $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-        else
-          remove_nonexistent_vm(vm, pool)
+
+      already_timed_out = time_since_clone > timeout
+      timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
+
+      return true if !already_timed_out && !timing_out_soon
+
+      if already_timed_out
+        unless exists
+          remove_nonexistent_vm(vm, pool, redis)
+          return true
         end
+        open_socket_error = handle_timed_out_vm(vm, pool, redis)
       end
+
+      redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
+
+      nonexist_warning = if already_timed_out
+                           "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
+                         elsif timing_out_soon
+                           time_remaining = timeout - timeout_notification
+                           open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+                           "[!] [#{pool}] '#{vm}' impending failure in #{time_remaining} minutes with error: #{open_socket_error}"
+                         else
+                           "[!] [#{pool}] '#{vm}' This error is wholly unexpected"
+                         end
+      $logger.log('d', nonexist_warning)
       true
-    rescue => err
-      $logger.log('d', "Fail pending VM failed with an error: #{err}")
+    rescue StandardError => e
+      $logger.log('d', "Fail pending VM failed with an error: #{e}")
       false
     end
 
-    def move_pending_vm_to_ready(vm, pool)
-      clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
-      finish = format('%.2f', Time.now - Time.parse(clone_time)) if clone_time
+    def handle_timed_out_vm(vm, pool, redis)
+      request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
+      open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+      retry_count = redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count').to_i if request_id
 
-      $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
-      $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
-      $redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
+      # Move to DLQ before moving to completed queue
+      move_to_dlq(vm, pool, 'pending', 'Timeout',
+                  open_socket_error || 'VM timed out during pending phase',
+                  redis, request_id: request_id, pool_alias: pool_alias, retry_count: retry_count)
 
-      # last boot time is displayed in API, and used by alarming script
-      $redis.hset('vmpooler__lastboot', pool, Time.now)
+      clone_error = redis.hget("vmpooler__vm__#{vm}", 'clone_error')
+      clone_error_class = redis.hget("vmpooler__vm__#{vm}", 'clone_error_class')
+      redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
 
-      $metrics.timing("time_to_ready_state.#{pool}", finish)
-      $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
+      if request_id
+        ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
+          # Check retry count and max retry limit before retrying
+          retry_count = (redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count') || '0').to_i
+          max_retries = $config[:config]['max_vm_retries'] || 3
+
+          $logger.log('s', "[!] [#{pool}] '#{vm}' checking retry logic: error='#{clone_error}', error_class='#{clone_error_class}', retry_count=#{retry_count}, max_retries=#{max_retries}")
+
+          # Determine if error is likely permanent (configuration issues)
+          permanent_error = permanent_error?(clone_error, clone_error_class)
+          $logger.log('s', "[!] [#{pool}] '#{vm}' permanent_error check result: #{permanent_error}")
+
+          if retry_count < max_retries && !permanent_error
+            # Increment retry count and retry VM creation
+            redis.hset("vmpooler__odrequest__#{request_id}", 'retry_count', retry_count + 1)
+            redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
+            $logger.log('s', "[!] [#{pool}] '#{vm}' failed, retrying (attempt #{retry_count + 1}/#{max_retries})")
+          else
+            # Max retries exceeded or permanent error, mark request as permanently failed
+            failure_reason = if permanent_error
+                               "Configuration error: #{clone_error}"
+                             else
+                               'Max retry attempts exceeded'
+                             end
+            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+            redis.hset("vmpooler__odrequest__#{request_id}", 'failure_reason', failure_reason)
+            $logger.log('s', "[!] [#{pool}] '#{vm}' permanently failed: #{failure_reason}")
+            $metrics.increment("vmpooler_errors.permanently_failed.#{pool}")
+          end
+        end
+      end
+      $metrics.increment("vmpooler_errors.markedasfailed.#{pool}")
+      open_socket_error || clone_error
     end
 
-    def vm_still_ready?(pool_name, vm_name, provider)
+    # Determine if an error is likely permanent (configuration issue) vs transient
+    def permanent_error?(error_message, error_class)
+      return false if error_message.nil? || error_class.nil?
+
+      permanent_error_patterns = [
+        /template.*not found/i,
+        /template.*does not exist/i,
+        /invalid.*path/i,
+        /folder.*not found/i,
+        /datastore.*not found/i,
+        /resource pool.*not found/i,
+        /permission.*denied/i,
+        /authentication.*failed/i,
+        /invalid.*credentials/i,
+        /configuration.*error/i
+      ]
+
+      permanent_error_classes = [
+        'ArgumentError',
+        'NoMethodError',
+        'NameError'
+      ]
+
+      # Check error message patterns
+      permanent_error_patterns.any? { |pattern| error_message.match?(pattern) } ||
+        # Check error class types
+        permanent_error_classes.include?(error_class)
+    end
+
+    def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
+      clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
+      finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
+
+      if request_id
+        ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        case ondemandrequest_hash['status']
+        when 'failed'
+          move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
+          return nil
+        when 'deleted'
+          move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
+          return nil
+        end
+        pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
+
+        redis.pipelined do |pipeline|
+          pipeline.hset("vmpooler__active__#{pool}", vm, Time.now.to_s)
+          pipeline.hset("vmpooler__vm__#{vm}", 'checkout', Time.now.to_s)
+          if ondemandrequest_hash['token:token']
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
+          end
+          pipeline.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
+        end
+        move_vm_queue(pool, vm, 'pending', 'running', redis)
+        check_ondemand_request_ready(request_id, redis)
+      else
+        redis.smove("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm)
+      end
+
+      redis.pipelined do |pipeline|
+        pipeline.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself?
+        pipeline.hset("vmpooler__vm__#{vm}", 'ready', Time.now.to_s)
+
+        # last boot time is displayed in API, and used by alarming script
+        pipeline.hset('vmpooler__lastboot', pool, Time.now.to_s)
+      end
+
+      $metrics.timing("time_to_ready_state.#{pool}", finish)
+      $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") unless request_id
+      $logger.log('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") if request_id
+    end
+
+    def vm_still_ready?(pool_name, vm_name, provider, redis)
       # Check if the VM is still ready/available
-      return true if provider.vm_ready?(pool_name, vm_name)
+      return true if provider.vm_ready?(pool_name, vm_name, redis)
+
       raise("VM #{vm_name} is not ready")
-    rescue
-      move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
+    rescue StandardError => e
+      open_socket_error = redis.hget("vmpooler__vm__#{vm_name}", 'open_socket_error')
+      request_id = redis.hget("vmpooler__vm__#{vm_name}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm_name}", 'pool_alias')
+
+      # Move to DLQ before moving to completed queue
+      move_to_dlq(vm_name, pool_name, 'ready', e.class.name,
+                  open_socket_error || 'VM became unreachable in ready queue',
+                  redis, request_id: request_id, pool_alias: pool_alias)
+
+      move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "removed from 'ready' queue. vm unreachable with error: #{open_socket_error}")
     end
 
     def check_ready_vm(vm, pool_name, ttl, provider)
       Thread.new do
         begin
           _check_ready_vm(vm, pool_name, ttl, provider)
-        rescue => err
-          $logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{err}")
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{e}")
           raise
         end
       end
@@ -149,29 +315,37 @@ module Vmpooler
       # Periodically check that the VM is available
       mutex = vm_mutex(vm)
       return if mutex.locked?
-      mutex.synchronize do
-        check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
-        return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
 
-        $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
-        # Check if the hosts TTL has expired
-        if ttl > 0
-          # host['boottime'] may be nil if host is not powered on
-          if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
-            $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
+      mutex.synchronize do
+        @redis.with_metrics do |redis|
+          check_stamp = redis.hget("vmpooler__vm__#{vm}", 'check')
+          last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
+          break if check_stamp && last_checked_too_soon
+
+          redis.hset("vmpooler__vm__#{vm}", 'check', Time.now.to_s)
+          # Check if the hosts TTL has expired
+          # if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
+          boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
+          if boottime
+            boottime = Time.parse(boottime)
+          else
+            boottime = Time.at(0)
+          end
+          if (Time.now - boottime).to_i > ttl * 60
+            redis.smove("vmpooler__ready__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
 
             $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
-            return
+            return nil
           end
+
+          break if mismatched_hostname?(vm, pool_name, provider, redis)
+
+          vm_still_ready?(pool_name, vm, provider, redis)
         end
-
-        return if has_mismatched_hostname?(vm, pool_name, provider)
-
-        vm_still_ready?(pool_name, vm, provider)
       end
     end
 
-    def has_mismatched_hostname?(vm, pool_name, provider)
+    def mismatched_hostname?(vm, pool_name, provider, redis)
       pool_config = $config[:pools][$config[:pool_index][pool_name]]
       check_hostname = pool_config['check_hostname_for_mismatch']
       check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
@@ -180,7 +354,7 @@ module Vmpooler
       # Wait one minute before checking a VM for hostname mismatch
       # When checking as soon as the VM passes the ready test the instance
       # often doesn't report its hostname yet causing the VM to be removed immediately
-      vm_ready_time = $redis.hget("vmpooler__vm__#{vm}", 'ready')
+      vm_ready_time = redis.hget("vmpooler__vm__#{vm}", 'ready')
       if vm_ready_time
         wait_before_checking = 60
         time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
@@ -190,22 +364,24 @@ module Vmpooler
       # Check if the hostname has magically changed from underneath Pooler
       vm_hash = provider.get_vm(pool_name, vm)
       return unless vm_hash.is_a? Hash
+
       hostname = vm_hash['hostname']
 
       return if hostname.nil?
       return if hostname.empty?
       return if hostname == vm
-      $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
+
+      redis.smove("vmpooler__ready__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
       $logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
-      return true
+      true
     end
 
     def check_running_vm(vm, pool, ttl, provider)
       Thread.new do
         begin
           _check_running_vm(vm, pool, ttl, provider)
-        rescue => err
-          $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{err}")
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{e}")
           raise
         end
       end
@@ -214,226 +390,909 @@ module Vmpooler
     def _check_running_vm(vm, pool, ttl, provider)
       mutex = vm_mutex(vm)
       return if mutex.locked?
+
       mutex.synchronize do
-        # Check that VM is within defined lifetime
-        checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
-        if checkouttime
-          running = (Time.now - Time.parse(checkouttime)) / 60 / 60
+        catch :stop_checking do
+          @redis.with_metrics do |redis|
+            # Check that VM is within defined lifetime
+            checkouttime = redis.hget("vmpooler__active__#{pool}", vm)
+            if checkouttime
+              time_since_checkout = Time.now - Time.parse(checkouttime)
+              running = time_since_checkout / 60 / 60
 
-          if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
-            move_vm_queue(pool, vm, 'running', 'completed', "reached end of TTL after #{ttl} hours")
-            return
-          end
-        end
+              if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
+                move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
+                throw :stop_checking
+              end
+            else
+              move_vm_queue(pool, vm, 'running', 'completed', redis, 'is listed as running, but has no checkouttime data. Removing from running')
+            end
 
-        if provider.vm_ready?(pool, vm)
-          return
-        else
-          host = provider.get_vm(pool, vm)
+            # tag VM if not tagged yet, this ensures the method is only called once
+            unless redis.hget("vmpooler__vm__#{vm}", 'user_tagged')
+              success = provider.tag_vm_user(pool, vm)
+              redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true') if success
+            end
 
-          if host
-            return
-          else
-            move_vm_queue(pool, vm, 'running', 'completed', "is no longer in inventory, removing from running")
+            throw :stop_checking if provider.vm_ready?(pool, vm, redis)
+
+            throw :stop_checking if provider.get_vm(pool, vm)
+
+            move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
           end
         end
       end
     end
 
-    def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
-      $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
+    def move_vm_queue(pool, vm, queue_from, queue_to, redis, msg = nil)
+      redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
       $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
     end
 
+    # Dead-Letter Queue (DLQ) helper methods
+    def dlq_enabled?
+      $config[:config] && $config[:config]['dlq_enabled'] == true
+    end
+
+    def dlq_ttl
+      ($config[:config] && $config[:config]['dlq_ttl']) || 168 # default 7 days in hours
+    end
+
+    def dlq_max_entries
+      ($config[:config] && $config[:config]['dlq_max_entries']) || 10_000
+    end
+
+    def move_to_dlq(vm, pool, queue_type, error_class, error_message, redis, request_id: nil, pool_alias: nil, retry_count: 0, skip_metrics: false)
+      return unless dlq_enabled?
+
+      dlq_key = "vmpooler__dlq__#{queue_type}"
+      timestamp = Time.now.to_i
+
+      # Build DLQ entry
+      dlq_entry = {
+        'vm' => vm,
+        'pool' => pool,
+        'queue_from' => queue_type,
+        'error_class' => error_class.to_s,
+        'error_message' => error_message.to_s,
+        'failed_at' => Time.now.iso8601,
+        'retry_count' => retry_count,
+        'request_id' => request_id,
+        'pool_alias' => pool_alias
+      }.compact
+
+      # Use sorted set with timestamp as score for easy age-based queries and TTL
+      dlq_entry_json = dlq_entry.to_json
+      redis.zadd(dlq_key, timestamp, "#{vm}:#{timestamp}:#{dlq_entry_json}")
+
+      # Enforce max entries limit by removing oldest entries
+      current_size = redis.zcard(dlq_key)
+      if current_size > dlq_max_entries
+        remove_count = current_size - dlq_max_entries
+        redis.zremrangebyrank(dlq_key, 0, remove_count - 1)
+        $logger.log('d', "[!] [dlq] Trimmed #{remove_count} oldest entries from #{dlq_key}")
+      end
+
+      # Set expiration on the entire DLQ (will be refreshed on next write)
+      ttl_seconds = dlq_ttl * 3600
+      redis.expire(dlq_key, ttl_seconds)
+
+      $metrics.increment("vmpooler_dlq.#{queue_type}.count") unless skip_metrics
+      $logger.log('d', "[!] [dlq] Moved '#{vm}' from '#{queue_type}' queue to DLQ: #{error_message}")
+    rescue StandardError => e
+      $logger.log('s', "[!] [dlq] Failed to move '#{vm}' to DLQ: #{e}")
+    end
+
     # Clone a VM
-    def clone_vm(pool_name, provider)
+    def clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil)
       Thread.new do
         begin
-          _clone_vm(pool_name, provider)
-        rescue => err
-          $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{err}")
+          _clone_vm(pool_name, provider, dns_plugin, request_id, pool_alias)
+        rescue StandardError => e
+          if request_id
+            $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
+            @redis.with_metrics do |redis|
+              # Only re-queue if the request wasn't already marked as failed (e.g., by permanent error detection)
+              request_status = redis.hget("vmpooler__odrequest__#{request_id}", 'status')
+              if request_status != 'failed'
+                redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
+              else
+                $logger.log('s', "[!] [#{pool_name}] Request #{request_id} already marked as failed, not re-queueing")
+              end
+            end
+          else
+            $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
+          end
           raise
         end
       end
     end
 
-    def _clone_vm(pool_name, provider)
-      # Generate a randomized hostname
-      o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-      new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
+    def generate_and_check_hostname
+      # Generate a randomized hostname. The total name must no longer than 15
+      # character including the hyphen. The shortest adjective in the corpus is
+      # three characters long. Therefore, we can technically select a noun up to 11
+      # characters long and still be guaranteed to have an available adjective.
+      # Because of the limited set of 11 letter nouns and corresponding 3
+      # letter adjectives, we actually limit the noun to 10 letters to avoid
+      # inviting more conflicts. We favor selecting a longer noun rather than a
+      # longer adjective because longer adjectives tend to be less fun.
+      @redis.with do |redis|
+        noun = @name_generator.noun(max: 10)
+        adjective = @name_generator.adjective(max: 14 - noun.length)
+        random_name = [adjective, noun].join('-')
+        hostname = $config[:config]['prefix'] + random_name
+        available = redis.hlen("vmpooler__vm__#{hostname}") == 0
 
-      # Add VM to Redis inventory ('pending' pool)
-      $redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
-      $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
-      $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
+        [hostname, available]
+      end
+    end
 
+    def find_unique_hostname(pool_name)
+      # generate hostname that is not already in use in vmpooler
+      # also check that no dns record already exists
+      hostname_retries = 0
+      max_hostname_retries = 3
+      while hostname_retries < max_hostname_retries
+        hostname, hostname_available = generate_and_check_hostname
+        domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name)
+        fqdn = "#{hostname}.#{domain}"
+
+        # skip dns check if the provider is set to skip_dns_check_before_creating_vm
+        provider = get_provider_for_pool(pool_name)
+        if provider && provider.provider_config['skip_dns_check_before_creating_vm']
+          dns_available = true
+        else
+          dns_ip, dns_available = check_dns_available(fqdn)
+        end
+
+        break if hostname_available && dns_available
+
+        hostname_retries += 1
+
+        if !hostname_available
+          $metrics.increment("vmpooler_errors.duplicatehostname.#{pool_name}")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
+        elsif !dns_available
+          $metrics.increment("vmpooler_errors.staledns.#{pool_name}")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS")
+        end
+      end
+
+      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available
+
+      hostname
+    end
+
+    # Query the DNS for the name we want to create and if it already exists, mark it unavailable
+    # This protects against stale DNS records
+    def check_dns_available(vm_name)
       begin
-        $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
-        start = Time.now
-        provider.create_vm(pool_name, new_vmname)
-        finish = format('%.2f', Time.now - start)
+        dns_ip = Resolv.getaddress(vm_name)
+      rescue Resolv::ResolvError
+        # this is the expected case, swallow the error
+        # eg "no address for blah-daisy.example.com"
+        return ['', true]
+      end
+      [dns_ip, false]
+    end
 
-        $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
-        $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
-        $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
+    def _clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil)
+      new_vmname = find_unique_hostname(pool_name)
+      pool_domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name)
+      mutex = vm_mutex(new_vmname)
+      mutex.synchronize do
+        @redis.with_metrics do |redis|
+          redis.multi do |transaction|
+            transaction.sadd("vmpooler__pending__#{pool_name}", new_vmname)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now.to_s)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool.
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias
+          end
+        end
 
-        $metrics.timing("clone.#{pool_name}", finish)
-      rescue => _err
-        $redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
-        expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
-        $redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
-        raise _err
-      ensure
-        $redis.decr('vmpooler__tasks__clone')
+        begin
+          $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
+          start = Time.now
+          provider.create_vm(pool_name, new_vmname)
+          finish = format('%<time>.2f', time: Time.now - start)
+          $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
+          $metrics.gauge("vmpooler_clone.#{pool_name}", finish)
+
+          $logger.log('d', "[ ] [#{pool_name}] Obtaining IP for '#{new_vmname}'")
+          ip_start = Time.now
+          ip = provider.get_vm_ip_address(new_vmname, pool_name)
+          ip_finish = format('%<time>.2f', time: Time.now - ip_start)
+
+          raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
+
+          $logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
+
+          @redis.with_metrics do |redis|
+            redis.pipelined do |pipeline|
+              pipeline.hset("vmpooler__clone__#{Date.today}", "#{pool_name}:#{new_vmname}", finish)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_time', finish)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'ip', ip)
+            end
+          end
+
+          dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool_name)
+          dns_plugin.create_or_replace_record(new_vmname) unless dns_plugin_class_name == 'dynamic-dns'
+        rescue StandardError => e
+          # Store error details for retry decision making
+          @redis.with_metrics do |redis|
+            # Get retry count before moving to DLQ
+            retry_count = 0
+            if request_id
+              ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+              retry_count = ondemandrequest_hash['retry_count'].to_i if ondemandrequest_hash
+            end
+
+            # Move to DLQ before removing from pending queue
+            move_to_dlq(new_vmname, pool_name, 'clone', e.class.name, e.message,
+                        redis, request_id: request_id, pool_alias: pool_alias, retry_count: retry_count)
+
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__pending__#{pool_name}", new_vmname)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_error', e.message)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_error_class', e.class.name)
+              expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
+              pipeline.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
+            end
+
+            # Handle retry logic for on-demand requests
+            if request_id
+              retry_count = (redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count') || '0').to_i
+              max_retries = $config[:config]['max_vm_retries'] || 3
+              is_permanent = permanent_error?(e.message, e.class.name)
+
+              $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' checking immediate failure retry: error='#{e.message}', error_class='#{e.class.name}', retry_count=#{retry_count}, max_retries=#{max_retries}, permanent_error=#{is_permanent}")
+
+              if is_permanent || retry_count >= max_retries
+                reason = is_permanent ? 'permanent error detected' : 'max retries exceeded'
+                $logger.log('s', "[!] [#{pool_name}] Cancelling request #{request_id} due to #{reason}")
+                redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+                redis.zadd('vmpooler__odcreate__task', 0, "#{pool_alias}:#{pool_name}:0:#{request_id}")
+              else
+                # Increment retry count and re-queue for retry
+                redis.hincrby("vmpooler__odrequest__#{request_id}", 'retry_count', 1)
+                $logger.log('s', "[+] [#{pool_name}] Request #{request_id} will be retried (attempt #{retry_count + 1}/#{max_retries})")
+                redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
+              end
+            end
+          end
+          $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed: #{e.class}: #{e.message}")
+          raise
+        ensure
+          @redis.with_metrics do |redis|
+            redis.decr('vmpooler__tasks__ondemandclone') if request_id
+            redis.decr('vmpooler__tasks__clone') unless request_id
+          end
+        end
       end
     end
 
     # Destroy a VM
-    def destroy_vm(vm, pool, provider)
+    def destroy_vm(vm, pool, provider, dns_plugin)
       Thread.new do
         begin
-          _destroy_vm(vm, pool, provider)
-        rescue => err
-          $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}")
+          _destroy_vm(vm, pool, provider, dns_plugin)
+        rescue StandardError => e
+          $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{e}")
           raise
         end
       end
     end
 
-    def _destroy_vm(vm, pool, provider)
+    def _destroy_vm(vm, pool, provider, dns_plugin)
       mutex = vm_mutex(vm)
       return if mutex.locked?
+
       mutex.synchronize do
-        $redis.hdel('vmpooler__active__' + pool, vm)
-        $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
+        @redis.with_metrics do |redis|
+          redis.pipelined do |pipeline|
+            pipeline.hdel("vmpooler__active__#{pool}", vm)
+            pipeline.hset("vmpooler__vm__#{vm}", 'destroy', Time.now.to_s)
 
-        # Auto-expire metadata key
-        $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+            # Auto-expire metadata key
+            pipeline.expire("vmpooler__vm__#{vm}", ($config[:redis]['data_ttl'].to_i * 60 * 60))
+          end
 
-        start = Time.now
+          start = Time.now
 
-        provider.destroy_vm(pool, vm)
+          provider.destroy_vm(pool, vm)
+          domain = get_dns_plugin_domain_for_pool(pool)
+          fqdn = "#{vm}.#{domain}"
 
-        $redis.srem('vmpooler__completed__' + pool, vm)
+          dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool)
+          dns_plugin.delete_record(fqdn) unless dns_plugin_class_name == 'dynamic-dns'
 
-        finish = format('%.2f', Time.now - start)
-        $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
-        $metrics.timing("destroy.#{pool}", finish)
-        get_vm_usage_labels(vm)
+          redis.srem("vmpooler__completed__#{pool}", vm)
+
+          finish = format('%<time>.2f', time: Time.now - start)
+          $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
+          $metrics.gauge("vmpooler_destroy.#{pool}", finish)
+        end
       end
       dereference_mutex(vm)
     end
 
-    def get_vm_usage_labels(vm)
-      return unless $config[:config]['usage_stats']
-      checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
-      return if checkout.nil?
-      jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
-      user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
-      poolname = $redis.hget("vmpooler__vm__#{vm}", "template")
-
-      unless jenkins_build_url
-        user = user.gsub('.', '_')
-        $metrics.increment("usage.#{user}.#{poolname}")
-        return
-      end
-
-      url_parts = jenkins_build_url.split('/')[2..-1]
-      instance = url_parts[0]
-      value_stream_parts = url_parts[2].split('_')
-      value_stream = value_stream_parts.shift
-      branch = value_stream_parts.pop
-      project = value_stream_parts.shift
-      job_name = value_stream_parts.join('_')
-      build_metadata_parts = url_parts[3]
-      component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
-
-      metric_parts = [
-        'usage',
-        user,
-        instance,
-        value_stream,
-        branch,
-        project,
-        job_name,
-        component_to_test,
-        poolname
-      ]
-
-      metric_parts = metric_parts.reject { |s| s.nil? }
-      metric_parts = metric_parts.map { |s| s.gsub('.', '_') }
-
-      $metrics.increment(metric_parts.join('.'))
-    rescue => err
-      logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{err}")
-    end
-
-    def component_to_test(match, labels_string)
-      return if labels_string.nil?
-      labels_string_parts = labels_string.split(',')
-      labels_string_parts.each do |part|
-        key, value = part.split('=')
-        next if value.nil?
-        if key == match
-          return value
-        end
-      end
-      return
-    end
-
-    def purge_unused_vms_and_folders
-      global_purge = $config[:config]['purge_unconfigured_folders']
+    def purge_unused_vms_and_resources
+      global_purge = $config[:config]['purge_unconfigured_resources']
       providers = $config[:providers].keys
-      providers.each do |provider|
-        provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
-        provider_purge = global_purge if provider_purge.nil?
+      providers.each do |provider_key|
+        provider_purge = $config[:providers][provider_key]['purge_unconfigured_resources'] || global_purge
         if provider_purge
           Thread.new do
             begin
-              purge_vms_and_folders($providers[provider.to_s])
-            rescue => err
-              $logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}")
+              purge_vms_and_resources(provider_key)
+            rescue StandardError => e
+              $logger.log('s', "[!] failed while purging provider #{provider_key} VMs and folders with an error: #{e}")
             end
           end
         end
       end
-      return
+      nil
     end
 
-    # Return a list of pool folders
-    def pool_folders(provider)
-      provider_name = provider.name
-      folders = {}
+    def purge_vms_and_resources(provider_name)
+      provider = $providers[provider_name.to_s]
+      # Deprecated, will be removed in version 3
+      if provider.provider_config['folder_whitelist']
+        $logger.log('d', "[!] [deprecation] rename configuration 'folder_whitelist' to 'resources_allowlist' for provider #{provider_name}")
+        allowlist = provider.provider_config['folder_whitelist']
+      else
+        allowlist = provider.provider_config['resources_allowlist']
+      end
+      provider.purge_unconfigured_resources(allowlist)
+    end
+
+    # Auto-purge stale queue entries
+    def purge_enabled?
+      $config[:config] && $config[:config]['purge_enabled'] == true
+    end
+
+    def purge_dry_run?
+      $config[:config] && $config[:config]['purge_dry_run'] == true
+    end
+
+    def max_pending_age
+      ($config[:config] && $config[:config]['max_pending_age']) || 7200 # default 2 hours in seconds
+    end
+
+    def max_ready_age
+      ($config[:config] && $config[:config]['max_ready_age']) || 86_400 # default 24 hours in seconds
+    end
+
+    def max_completed_age
+      ($config[:config] && $config[:config]['max_completed_age']) || 3600 # default 1 hour in seconds
+    end
+
+    def max_orphaned_age
+      ($config[:config] && $config[:config]['max_orphaned_age']) || 86_400 # default 24 hours in seconds
+    end
+
+    def purge_stale_queue_entries
+      return unless purge_enabled?
+
+      Thread.new do
+        begin
+          $logger.log('d', '[*] [purge] Starting stale queue entry purge cycle')
+          purge_start = Time.now
+
+          @redis.with_metrics do |redis|
+            total_purged = 0
+
+            # Purge stale entries from each pool
+            $config[:pools].each do |pool|
+              pool_name = pool['name']
+
+              # Purge pending queue
+              purged_pending = purge_pending_queue(pool_name, redis)
+              total_purged += purged_pending
+
+              # Purge ready queue
+              purged_ready = purge_ready_queue(pool_name, redis)
+              total_purged += purged_ready
+
+              # Purge completed queue
+              purged_completed = purge_completed_queue(pool_name, redis)
+              total_purged += purged_completed
+            end
+
+            # Purge orphaned VM metadata
+            purged_orphaned = purge_orphaned_metadata(redis)
+            total_purged += purged_orphaned
+
+            purge_duration = Time.now - purge_start
+            $logger.log('s', "[*] [purge] Completed purge cycle in #{purge_duration.round(2)}s: #{total_purged} entries purged")
+            $metrics.gauge('vmpooler_purge.cycle.duration', purge_duration)
+            $metrics.gauge('vmpooler_purge.total.count', total_purged)
+          end
+        rescue StandardError => e
+          $logger.log('s', "[!] [purge] Failed during purge cycle: #{e}")
+        end
+      end
+    end
+
+    def purge_pending_queue(pool_name, redis)
+      queue_key = "vmpooler__pending__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          clone_time_str = redis.hget("vmpooler__vm__#{vm}", 'clone')
+          next unless clone_time_str
+
+          clone_time = Time.parse(clone_time_str)
+          age = Time.now - clone_time
+
+          if age > max_pending_age
+            request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+            pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
+
+            purged_count += 1
+
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale pending VM '#{vm}' (age: #{age.round(0)}s, max: #{max_pending_age}s)")
+            else
+              # Move to DLQ before removing (skip DLQ metric since we're tracking purge metric)
+              move_to_dlq(vm, pool_name, 'pending', 'Purge',
+                          "Stale pending VM (age: #{age.round(0)}s > max: #{max_pending_age}s)",
+                          redis, request_id: request_id, pool_alias: pool_alias, skip_metrics: true)
+
+              redis.srem(queue_key, vm)
+
+              # Set expiration on VM metadata if data_ttl is configured
+              if $config[:redis] && $config[:redis]['data_ttl']
+                expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
+                redis.expire("vmpooler__vm__#{vm}", expiration_ttl)
+              end
+
+              $logger.log('d', "[!] [purge] Purged stale pending VM '#{vm}' from '#{pool_name}' (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.pending.#{pool_name}.count")
+            end
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking pending VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_ready_queue(pool_name, redis)
+      queue_key = "vmpooler__ready__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          ready_time_str = redis.hget("vmpooler__vm__#{vm}", 'ready')
+          next unless ready_time_str
+
+          ready_time = Time.parse(ready_time_str)
+          age = Time.now - ready_time
+
+          if age > max_ready_age
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale ready VM '#{vm}' (age: #{age.round(0)}s, max: #{max_ready_age}s)")
+            else
+              redis.smove(queue_key, "vmpooler__completed__#{pool_name}", vm)
+              $logger.log('d', "[!] [purge] Moved stale ready VM '#{vm}' from '#{pool_name}' to completed (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.ready.#{pool_name}.count")
+            end
+            purged_count += 1
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking ready VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_completed_queue(pool_name, redis)
+      queue_key = "vmpooler__completed__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          # Check destroy time or last activity time
+          destroy_time_str = redis.hget("vmpooler__vm__#{vm}", 'destroy')
+          checkout_time_str = redis.hget("vmpooler__vm__#{vm}", 'checkout')
+
+          # Use the most recent timestamp
+          timestamp_str = destroy_time_str || checkout_time_str
+          next unless timestamp_str
+
+          timestamp = Time.parse(timestamp_str)
+          age = Time.now - timestamp
+
+          if age > max_completed_age
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale completed VM '#{vm}' (age: #{age.round(0)}s, max: #{max_completed_age}s)")
+            else
+              redis.srem(queue_key, vm)
+              $logger.log('d', "[!] [purge] Removed stale completed VM '#{vm}' from '#{pool_name}' (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.completed.#{pool_name}.count")
+            end
+            purged_count += 1
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking completed VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_orphaned_metadata(redis)
+      # Find VM metadata that doesn't belong to any queue
+      all_vm_keys = redis.keys('vmpooler__vm__*')
+      purged_count = 0
+
+      all_vm_keys.each do |vm_key|
+        begin
+          vm = vm_key.sub('vmpooler__vm__', '')
+
+          # Check if VM exists in any queue
+          pool_name = redis.hget(vm_key, 'pool')
+          next unless pool_name
+
+          in_pending = redis.sismember("vmpooler__pending__#{pool_name}", vm)
+          in_ready = redis.sismember("vmpooler__ready__#{pool_name}", vm)
+          in_running = redis.sismember("vmpooler__running__#{pool_name}", vm)
+          in_completed = redis.sismember("vmpooler__completed__#{pool_name}", vm)
+          in_discovered = redis.sismember("vmpooler__discovered__#{pool_name}", vm)
+          in_migrating = redis.sismember("vmpooler__migrating__#{pool_name}", vm)
+
+          # VM is orphaned if not in any queue
+          unless in_pending || in_ready || in_running || in_completed || in_discovered || in_migrating
+            # Check age
+            clone_time_str = redis.hget(vm_key, 'clone')
+            next unless clone_time_str
+
+            clone_time = Time.parse(clone_time_str)
+            age = Time.now - clone_time
+
+            if age > max_orphaned_age
+              if purge_dry_run?
+                $logger.log('d', "[*] [purge][dry-run] Would purge orphaned metadata for '#{vm}' (age: #{age.round(0)}s, max: #{max_orphaned_age}s)")
+              else
+                expiration_ttl = 3600 # 1 hour
+                redis.expire(vm_key, expiration_ttl)
+                $logger.log('d', "[!] [purge] Set expiration on orphaned metadata for '#{vm}' (age: #{age.round(0)}s)")
+                $metrics.increment('vmpooler_purge.orphaned.count')
+              end
+              purged_count += 1
+            end
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking orphaned metadata '#{vm_key}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    # Health checks for Redis queues
+    def health_check_enabled?
+      $config[:config] && $config[:config]['health_check_enabled'] == true
+    end
+
+    def health_thresholds
+      defaults = {
+        'pending_queue_max' => 100,
+        'ready_queue_max' => 500,
+        'dlq_max_warning' => 100,
+        'dlq_max_critical' => 1000,
+        'stuck_vm_age_threshold' => 7200, # 2 hours
+        'stuck_vm_max_warning' => 10,
+        'stuck_vm_max_critical' => 50
+      }
+
+      if $config[:config] && $config[:config]['health_thresholds']
+        defaults.merge($config[:config]['health_thresholds'])
+      else
+        defaults
+      end
+    end
+
+    def check_queue_health
+      return unless health_check_enabled?
+
+      Thread.new do
+        begin
+          $logger.log('d', '[*] [health] Running queue health check')
+          health_start = Time.now
+
+          @redis.with_metrics do |redis|
+            health_metrics = calculate_health_metrics(redis)
+            health_status = determine_health_status(health_metrics)
+
+            # Store health metrics in Redis for API consumption
+            # Convert nested hash to JSON for storage
+            require 'json'
+            redis.hset('vmpooler__health', 'metrics', health_metrics.to_json)
+            redis.hset('vmpooler__health', 'status', health_status)
+            redis.hset('vmpooler__health', 'last_check', Time.now.iso8601)
+            redis.expire('vmpooler__health', 3600) # Expire after 1 hour
+
+            # Log health summary
+            log_health_summary(health_metrics, health_status)
+
+            # Push metrics
+            push_health_metrics(health_metrics, health_status)
+
+            health_duration = Time.now - health_start
+            $metrics.gauge('vmpooler_health.check.duration', health_duration)
+          end
+        rescue StandardError => e
+          $logger.log('s', "[!] [health] Failed during health check: #{e}")
+        end
+      end
+    end
+
+    def calculate_health_metrics(redis)
+      metrics = {
+        'queues' => {},
+        'tasks' => {},
+        'errors' => {}
+      }
+
+      total_stuck_vms = 0
+      total_dlq_size = 0
+      thresholds = health_thresholds
+
+      # Check each pool's queues
       $config[:pools].each do |pool|
-        next unless pool['provider'] == provider_name
-        folder_parts = pool['folder'].split('/')
-        datacenter = provider.get_target_datacenter_from_config(pool['name'])
-        folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
+        pool_name = pool['name']
+        metrics['queues'][pool_name] = {}
+
+        # Pending queue metrics
+        pending_key = "vmpooler__pending__#{pool_name}"
+        pending_vms = redis.smembers(pending_key)
+        pending_ages = calculate_queue_ages(pending_vms, 'clone', redis)
+        stuck_pending = pending_ages.count { |age| age > thresholds['stuck_vm_age_threshold'] }
+        total_stuck_vms += stuck_pending
+
+        metrics['queues'][pool_name]['pending'] = {
+          'size' => pending_vms.size,
+          'oldest_age' => pending_ages.max || 0,
+          'avg_age' => pending_ages.empty? ? 0 : (pending_ages.sum / pending_ages.size).round(0),
+          'stuck_count' => stuck_pending
+        }
+
+        # Ready queue metrics
+        ready_key = "vmpooler__ready__#{pool_name}"
+        ready_vms = redis.smembers(ready_key)
+        ready_ages = calculate_queue_ages(ready_vms, 'ready', redis)
+
+        metrics['queues'][pool_name]['ready'] = {
+          'size' => ready_vms.size,
+          'oldest_age' => ready_ages.max || 0,
+          'avg_age' => ready_ages.empty? ? 0 : (ready_ages.sum / ready_ages.size).round(0)
+        }
+
+        # Completed queue metrics
+        completed_key = "vmpooler__completed__#{pool_name}"
+        completed_size = redis.scard(completed_key)
+        metrics['queues'][pool_name]['completed'] = { 'size' => completed_size }
       end
-      folders
+
+      # Task queue metrics
+      clone_active = redis.get('vmpooler__tasks__clone').to_i
+      ondemand_active = redis.get('vmpooler__tasks__ondemandclone').to_i
+      odcreate_pending = redis.zcard('vmpooler__odcreate__task')
+
+      metrics['tasks']['clone'] = { 'active' => clone_active }
+      metrics['tasks']['ondemand'] = { 'active' => ondemand_active, 'pending' => odcreate_pending }
+
+      # DLQ metrics
+      if dlq_enabled?
+        dlq_keys = redis.keys('vmpooler__dlq__*')
+        dlq_keys.each do |dlq_key|
+          queue_type = dlq_key.sub('vmpooler__dlq__', '')
+          dlq_size = redis.zcard(dlq_key)
+          total_dlq_size += dlq_size
+          metrics['queues']['dlq'] ||= {}
+          metrics['queues']['dlq'][queue_type] = { 'size' => dlq_size }
+        end
+      end
+
+      # Error metrics
+      metrics['errors']['dlq_total_size'] = total_dlq_size
+      metrics['errors']['stuck_vm_count'] = total_stuck_vms
+
+      # Orphaned metadata count
+      orphaned_count = count_orphaned_metadata(redis)
+      metrics['errors']['orphaned_metadata_count'] = orphaned_count
+
+      metrics
     end
 
-    def get_base_folders(folders)
-      base = []
-      folders.each do |key, value|
-        base << value
+    def calculate_queue_ages(vms, timestamp_field, redis)
+      ages = []
+      vms.each do |vm|
+        begin
+          timestamp_str = redis.hget("vmpooler__vm__#{vm}", timestamp_field)
+          next unless timestamp_str
+
+          timestamp = Time.parse(timestamp_str)
+          age = (Time.now - timestamp).to_i
+          ages << age
+        rescue StandardError
+          # Skip VMs with invalid timestamps
+        end
       end
-      base.uniq
+      ages
     end
 
-    def purge_vms_and_folders(provider)
-      configured_folders = pool_folders(provider)
-      base_folders = get_base_folders(configured_folders)
-      whitelist = provider.provider_config['folder_whitelist']
-      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+    def count_orphaned_metadata(redis)
+      all_vm_keys = redis.keys('vmpooler__vm__*')
+      orphaned_count = 0
+
+      all_vm_keys.each do |vm_key|
+        begin
+          vm = vm_key.sub('vmpooler__vm__', '')
+          pool_name = redis.hget(vm_key, 'pool')
+          next unless pool_name
+
+          in_any_queue = redis.sismember("vmpooler__pending__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__ready__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__running__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__completed__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__discovered__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__migrating__#{pool_name}", vm)
+
+          orphaned_count += 1 unless in_any_queue
+        rescue StandardError
+          # Skip on error
+        end
+      end
+
+      orphaned_count
+    end
+
+    def determine_health_status(metrics)
+      thresholds = health_thresholds
+
+      # Check DLQ size
+      dlq_size = metrics['errors']['dlq_total_size']
+      return 'unhealthy' if dlq_size > thresholds['dlq_max_critical']
+
+      # Check stuck VM count
+      stuck_count = metrics['errors']['stuck_vm_count']
+      return 'unhealthy' if stuck_count > thresholds['stuck_vm_max_critical']
+
+      # Check queue sizes
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        pending_size = begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        ready_size = begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+
+        return 'unhealthy' if pending_size > thresholds['pending_queue_max'] * 2
+        return 'unhealthy' if ready_size > thresholds['ready_queue_max'] * 2
+      end
+
+      # Check for degraded conditions
+      return 'degraded' if dlq_size > thresholds['dlq_max_warning']
+      return 'degraded' if stuck_count > thresholds['stuck_vm_max_warning']
+
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        pending_size = begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        ready_size = begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+
+        return 'degraded' if pending_size > thresholds['pending_queue_max']
+        return 'degraded' if ready_size > thresholds['ready_queue_max']
+      end
+
+      'healthy'
+    end
+
+    def log_health_summary(metrics, status)
+      summary = "[*] [health] Status: #{status.upcase}"
+
+      # Queue summary
+      total_pending = 0
+      total_ready = 0
+      total_completed = 0
+
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        total_pending += begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        total_ready += begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+        total_completed += begin
+          queues['completed']['size']
+        rescue StandardError
+          0
+        end
+      end
+
+      summary += " | Queues: P=#{total_pending} R=#{total_ready} C=#{total_completed}"
+      summary += " | DLQ=#{metrics['errors']['dlq_total_size']}"
+      summary += " | Stuck=#{metrics['errors']['stuck_vm_count']}"
+      summary += " | Orphaned=#{metrics['errors']['orphaned_metadata_count']}"
+
+      log_level = status == 'healthy' ? 's' : 'd'
+      $logger.log(log_level, summary)
+    end
+
+    def push_health_metrics(metrics, status)
+      # Push error metrics first
+      $metrics.gauge('vmpooler_health.dlq.total_size', metrics['errors']['dlq_total_size'])
+      $metrics.gauge('vmpooler_health.stuck_vms.count', metrics['errors']['stuck_vm_count'])
+      $metrics.gauge('vmpooler_health.orphaned_metadata.count', metrics['errors']['orphaned_metadata_count'])
+
+      # Push per-pool queue metrics
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.size", queues['pending']['size'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.oldest_age", queues['pending']['oldest_age'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.stuck_count", queues['pending']['stuck_count'])
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.ready.size", queues['ready']['size'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.ready.oldest_age", queues['ready']['oldest_age'])
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.completed.size", queues['completed']['size'])
+      end
+
+      # Push DLQ metrics
+      metrics['queues']['dlq']&.each do |queue_type, dlq_metrics|
+        $metrics.gauge("vmpooler_health.dlq.#{queue_type}.size", dlq_metrics['size'])
+      end
+
+      # Push task metrics
+      $metrics.gauge('vmpooler_health.tasks.clone.active', metrics['tasks']['clone']['active'])
+      $metrics.gauge('vmpooler_health.tasks.ondemand.active', metrics['tasks']['ondemand']['active'])
+      $metrics.gauge('vmpooler_health.tasks.ondemand.pending', metrics['tasks']['ondemand']['pending'])
+
+      # Push status last (0=healthy, 1=degraded, 2=unhealthy)
+      status_value = { 'healthy' => 0, 'degraded' => 1, 'unhealthy' => 2 }[status] || 2
+      $metrics.gauge('vmpooler_health.status', status_value)
     end
 
     def create_vm_disk(pool_name, vm, disk_size, provider)
       Thread.new do
         begin
           _create_vm_disk(pool_name, vm, disk_size, provider)
-        rescue => err
-          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{err}")
+        rescue StandardError => e
+          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{e}")
           raise
         end
       end
@@ -448,13 +1307,15 @@ module Vmpooler
 
       result = provider.create_disk(pool_name, vm_name, disk_size.to_i)
 
-      finish = format('%.2f', Time.now - start)
+      finish = format('%<time>.2f', time: Time.now - start)
 
       if result
-        rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk')
-        disks = rdisks ? rdisks.split(':') : []
-        disks.push("+#{disk_size}gb")
-        $redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
+        @redis.with_metrics do |redis|
+          rdisks = redis.hget("vmpooler__vm__#{vm_name}", 'disk')
+          disks = rdisks ? rdisks.split(':') : []
+          disks.push("+#{disk_size}gb")
+          redis.hset("vmpooler__vm__#{vm_name}", 'disk', disks.join(':'))
+        end
 
         $logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
       else
@@ -468,8 +1329,8 @@ module Vmpooler
       Thread.new do
         begin
           _create_vm_snapshot(pool_name, vm, snapshot_name, provider)
-        rescue => err
-          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{err}")
+        rescue StandardError => e
+          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{e}")
           raise
         end
       end
@@ -481,10 +1342,12 @@ module Vmpooler
 
       result = provider.create_snapshot(pool_name, vm_name, snapshot_name)
 
-      finish = format('%.2f', Time.now - start)
+      finish = format('%<time>.2f', time: Time.now - start)
 
       if result
-        $redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
+        @redis.with_metrics do |redis|
+          redis.hset("vmpooler__vm__#{vm_name}", "snapshot:#{snapshot_name}", Time.now.to_s)
+        end
         $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
       else
         $logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'")
@@ -497,8 +1360,8 @@ module Vmpooler
       Thread.new do
         begin
           _revert_vm_snapshot(pool_name, vm, snapshot_name, provider)
-        rescue => err
-          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{err}")
+        rescue StandardError => e
+          $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{e}")
           raise
         end
       end
@@ -510,7 +1373,7 @@ module Vmpooler
 
       result = provider.revert_snapshot(pool_name, vm_name, snapshot_name)
 
-      finish = format('%.2f', Time.now - start)
+      finish = format('%<time>.2f', time: Time.now - start)
 
       if result
         $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' reverted to snapshot '#{snapshot_name}' in #{finish} seconds")
@@ -521,39 +1384,84 @@ module Vmpooler
       result
     end
 
+    # load only dns plugins used in config file
+    def load_used_dns_plugins
+      dns_plugins = Vmpooler::Dns.get_dns_plugin_config_classes(config)
+      Vmpooler::Dns.load_by_name(dns_plugins)
+    end
+
     # load only providers used in config file
     def load_used_providers
       Vmpooler::Providers.load_by_name(used_providers)
     end
 
     # @return [Array] - a list of used providers from the config file, defaults to the default providers
-    # ie. ["vsphere", "dummy"]
+    # ie. ["dummy"]
     def used_providers
-      pools = config[:pools] || []
-      @used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers ).uniq
+      # create an array of provider classes based on the config
+      if config[:providers]
+        config_provider_names = config[:providers].keys
+        config_providers = config_provider_names.map do |config_provider_name|
+          if config[:providers][config_provider_name] && config[:providers][config_provider_name]['provider_class']
+            config[:providers][config_provider_name]['provider_class'].to_s
+          else
+            config_provider_name.to_s
+          end
+        end.compact.uniq
+      else
+        config_providers = []
+      end
+      # return the unique array of providers from the config and VMPooler defaults
+      @used_providers ||= (config_providers + default_providers).uniq
     end
 
     # @return [Array] - returns a list of providers that should always be loaded
     # note: vsphere is the default if user does not specify although this should not be
     # if vsphere is to no longer be loaded by default please remove
     def default_providers
-      @default_providers ||= %w( vsphere dummy )
+      @default_providers ||= %w[dummy]
     end
 
-    def get_pool_name_for_vm(vm_name)
+    def get_pool_name_for_vm(vm_name, redis)
       # the 'template' is a bad name.  Should really be 'poolname'
-      $redis.hget('vmpooler__vm__' + vm_name, 'template')
+      redis.hget("vmpooler__vm__#{vm_name}", 'template')
     end
 
     # @param pool_name [String] - the name of the pool
     # @return [Provider] - returns the provider class Object
     def get_provider_for_pool(pool_name)
-      pool = $config[:pools].find { |pool| pool['name'] == pool_name }
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
       return nil unless pool
+
       provider_name = pool.fetch('provider', nil)
       $providers[provider_name]
     end
 
+    def get_dns_plugin_class_name_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      Vmpooler::Dns.get_dns_plugin_class_by_name(config, plugin_name)
+    end
+
+    def get_dns_plugin_class_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      plugin_class = Vmpooler::Dns.get_dns_plugin_class_by_name(config, plugin_name)
+      $dns_plugins[plugin_class]
+    end
+
+    def get_dns_plugin_domain_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      Vmpooler::Dns.get_dns_plugin_domain_by_name(config, plugin_name)
+    end
+
     def check_disk_queue(maxloop = 0, loop_delay = 5)
       $logger.log('d', '[*] [disk_manager] starting worker thread')
 
@@ -563,8 +1471,9 @@ module Vmpooler
           _check_disk_queue
           sleep(loop_delay)
 
-          unless maxloop.zero?
+          unless maxloop == 0
             break if loop_count >= maxloop
+
             loop_count += 1
           end
         end
@@ -572,19 +1481,21 @@ module Vmpooler
     end
 
     def _check_disk_queue
-      task_detail = $redis.spop('vmpooler__tasks__disk')
-      unless task_detail.nil?
-        begin
-          vm_name, disk_size = task_detail.split(':')
-          pool_name = get_pool_name_for_vm(vm_name)
-          raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
+      @redis.with_metrics do |redis|
+        task_detail = redis.spop('vmpooler__tasks__disk')
+        unless task_detail.nil?
+          begin
+            vm_name, disk_size = task_detail.split(':')
+            pool_name = get_pool_name_for_vm(vm_name, redis)
+            raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
 
-          provider = get_provider_for_pool(pool_name)
-          raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
+            provider = get_provider_for_pool(pool_name)
+            raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
 
-          create_vm_disk(pool_name, vm_name, disk_size, provider)
-        rescue => err
-          $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{err}")
+            create_vm_disk(pool_name, vm_name, disk_size, provider)
+          rescue StandardError => e
+            $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
+          end
         end
       end
     end
@@ -598,8 +1509,9 @@ module Vmpooler
           _check_snapshot_queue
           sleep(loop_delay)
 
-          unless maxloop.zero?
+          unless maxloop == 0
             break if loop_count >= maxloop
+
             loop_count += 1
           end
         end
@@ -607,37 +1519,39 @@ module Vmpooler
     end
 
     def _check_snapshot_queue
-      task_detail = $redis.spop('vmpooler__tasks__snapshot')
+      @redis.with_metrics do |redis|
+        task_detail = redis.spop('vmpooler__tasks__snapshot')
 
-      unless task_detail.nil?
-        begin
-          vm_name, snapshot_name = task_detail.split(':')
-          pool_name = get_pool_name_for_vm(vm_name)
-          raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
+        unless task_detail.nil?
+          begin
+            vm_name, snapshot_name = task_detail.split(':')
+            pool_name = get_pool_name_for_vm(vm_name, redis)
+            raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
 
-          provider = get_provider_for_pool(pool_name)
-          raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
+            provider = get_provider_for_pool(pool_name)
+            raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
 
-          create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
-        rescue => err
-          $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{err}")
+            create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
+          rescue StandardError => e
+            $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
+          end
         end
-      end
 
-      task_detail = $redis.spop('vmpooler__tasks__snapshot-revert')
+        task_detail = redis.spop('vmpooler__tasks__snapshot-revert')
 
-      unless task_detail.nil?
-        begin
-          vm_name, snapshot_name = task_detail.split(':')
-          pool_name = get_pool_name_for_vm(vm_name)
-          raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
+        unless task_detail.nil?
+          begin
+            vm_name, snapshot_name = task_detail.split(':')
+            pool_name = get_pool_name_for_vm(vm_name, redis)
+            raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
 
-          provider = get_provider_for_pool(pool_name)
-          raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
+            provider = get_provider_for_pool(pool_name)
+            raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
 
-          revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
-        rescue => err
-          $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{err}")
+            revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
+          rescue StandardError => e
+            $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
+          end
         end
       end
     end
@@ -647,11 +1561,13 @@ module Vmpooler
         begin
           mutex = vm_mutex(vm_name)
           mutex.synchronize do
-            $redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
+            @redis.with_metrics do |redis|
+              redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
+            end
             provider.migrate_vm(pool_name, vm_name)
           end
-        rescue => err
-          $logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{err}")
+        rescue StandardError => e
+          $logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{e}")
         end
       end
     end
@@ -670,55 +1586,80 @@ module Vmpooler
     #   - Fires when a template configuration update is requested
     #   - Additional options
     #       :poolname
+    # :pool_reset
+    #   - Fires when a pool reset is requested
+    #   - Additional options
+    #       :poolname
+    # :undo_override
+    #   - Fires when a pool override removal is requested
+    #   - Additional options
+    #       :poolname
     #
     def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {})
       exit_by = Time.now + loop_delay
       wakeup_by = Time.now + wakeup_period
+
       return if time_passed?(:exit_by, exit_by)
 
-      if options[:pool_size_change]
-        initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
-      end
+      @redis.with_metrics do |redis|
+        initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
 
-      if options[:clone_target_change]
-        initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target])
-      end
+        initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target].to_s) if options[:clone_target_change]
 
-      if options[:pool_template_change]
-        initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
-      end
+        initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
 
-      loop do
-        sleep(1)
-        break if time_passed?(:exit_by, exit_by)
+        loop do
+          sleep(1)
+          break if time_passed?(:exit_by, exit_by)
 
-        # Check for wakeup events
-        if time_passed?(:wakeup_by, wakeup_by)
-          wakeup_by = Time.now + wakeup_period
+          # Check for wakeup events
+          if time_passed?(:wakeup_by, wakeup_by)
+            wakeup_by = Time.now + wakeup_period
 
-          # Wakeup if the number of ready VMs has changed
-          if options[:pool_size_change]
-            ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
-            break unless ready_size == initial_ready_size
-          end
+            # Wakeup if the number of ready VMs has changed
+            if options[:pool_size_change]
+              ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}")
+              break unless ready_size == initial_ready_size
+            end
 
-          if options[:clone_target_change]
-            clone_target = $redis.hget("vmpooler__config__clone_target}", options[:poolname])
-            if clone_target
-              break unless clone_target == initial_clone_target
-            end
-          end
-
-          if options[:pool_template_change]
-            configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
-            if configured_template
-              break unless initial_template == configured_template
+            if options[:clone_target_change]
+              clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
+              break if clone_target && clone_target != initial_clone_target
+            end
+
+            if options[:pool_template_change]
+              configured_template = redis.hget('vmpooler__config__template', options[:poolname])
+              break if configured_template && initial_template != configured_template
+            end
+
+            if options[:pool_reset]
+              pending = redis.sismember('vmpooler__poolreset', options[:poolname])
+              break if pending
+            end
+
+            if options[:undo_override]
+              break if redis.sismember('vmpooler__pool__undo_template_override', options[:poolname])
+              break if redis.sismember('vmpooler__pool__undo_size_override', options[:poolname])
+            end
+
+            if options[:pending_vm]
+              pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
+              break unless pending_vm_count == 0
+            end
+
+            if options[:ondemand_request]
+              od_request = redis.zcard('vmpooler__provisioning__request')
+              od_processing = redis.zcard('vmpooler__provisioning__processing')
+              od_createtask = redis.zcard('vmpooler__odcreate__task')
+
+              break unless od_request == 0
+              break unless od_processing == 0
+              break unless od_createtask == 0
             end
           end
 
+          break if time_passed?(:exit_by, exit_by)
         end
-
-        break if time_passed?(:exit_by, exit_by)
       end
     end
 
@@ -742,10 +1683,16 @@ module Vmpooler
           loop_count = 1
           loop_delay = loop_delay_min
           provider = get_provider_for_pool(pool['name'])
-          raise("Could not find provider '#{pool['provider']}") if provider.nil?
+          raise("Could not find provider '#{pool['provider']}'") if provider.nil?
+
           sync_pool_template(pool)
           loop do
+            start_time = Time.now
             result = _check_pool(pool, provider)
+            duration = Time.now - start_time
+
+            $metrics.gauge("vmpooler_performance.check_pool.#{pool['name']}", duration)
+            $logger.log('d', "[!] check_pool for #{pool['name']} took #{duration.round(2)}s") if duration > 5
 
             if result[:cloned_vms] > 0 || result[:checked_pending_vms] > 0 || result[:discovered_vms] > 0
               loop_delay = loop_delay_min
@@ -753,15 +1700,18 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true)
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true, undo_override: true)
 
-            unless maxloop.zero?
+            unless maxloop == 0
               break if loop_count >= maxloop
+
               loop_count += 1
             end
           end
-        rescue => err
-          $logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{err}")
+        rescue Redis::CannotConnectError
+          raise
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{e}")
           raise
         end
       end
@@ -776,120 +1726,128 @@ module Vmpooler
     end
 
     def dereference_mutex(vmname)
-      if @vm_mutex.delete(vmname)
-        return true
-      else
-        return
-      end
+      true if @vm_mutex.delete(vmname)
     end
 
     def sync_pool_template(pool)
-      pool_template = $redis.hget('vmpooler__config__template', pool['name'])
-      if pool_template
-        unless pool['template'] == pool_template
-          pool['template'] = pool_template
-        end
+      @redis.with_metrics do |redis|
+        pool_template = redis.hget('vmpooler__config__template', pool['name'])
+        pool['template'] = pool_template if pool_template && pool['template'] != pool_template
       end
     end
 
-    def prepare_template(pool, provider)
-      if $config[:config]['create_template_delta_disks']
-        unless $redis.sismember('vmpooler__template__deltas', pool['template'])
-          begin
-            provider.create_template_delta_disks(pool)
-            $redis.sadd('vmpooler__template__deltas', pool['template'])
-          rescue => err
-            $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{err}")
-          end
+    def prepare_template(pool, provider, redis)
+      if $config[:config]['create_template_delta_disks'] && !redis.sismember('vmpooler__template__deltas', pool['template'])
+        begin
+          provider.create_template_delta_disks(pool)
+          redis.sadd('vmpooler__template__deltas', pool['template'])
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
         end
       end
-      $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
+      redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
     end
 
     def evaluate_template(pool, provider)
       mutex = pool_mutex(pool['name'])
-      prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
-      configured_template = $redis.hget('vmpooler__config__template', pool['name'])
       return if mutex.locked?
-      if prepared_template.nil?
-        mutex.synchronize do
-          prepare_template(pool, provider)
-          prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
-        end
-      elsif prepared_template != pool['template']
-        if configured_template.nil?
+
+      catch :update_not_needed do
+        @redis.with_metrics do |redis|
+          prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
+          configured_template = redis.hget('vmpooler__config__template', pool['name'])
+
+          if prepared_template.nil?
+            mutex.synchronize do
+              prepare_template(pool, provider, redis)
+              prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
+            end
+          elsif prepared_template != pool['template']
+            if configured_template.nil?
+              mutex.synchronize do
+                prepare_template(pool, provider, redis)
+                prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
+              end
+            end
+          end
+          throw :update_not_needed if configured_template.nil?
+          throw :update_not_needed if configured_template == prepared_template
+
           mutex.synchronize do
-            prepare_template(pool, provider)
-            prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
+            update_pool_template(pool, provider, configured_template, prepared_template, redis)
           end
         end
       end
-      return if configured_template.nil?
-      return if configured_template == prepared_template
-      mutex.synchronize do
-        update_pool_template(pool, provider, configured_template, prepared_template)
-      end
     end
 
-    def drain_pool(poolname)
+    def drain_pool(poolname, redis)
       # Clear a pool of ready and pending instances
-      if $redis.scard("vmpooler__ready__#{poolname}") > 0
+      if redis.scard("vmpooler__ready__#{poolname}") > 0
         $logger.log('s', "[*] [#{poolname}] removing ready instances")
-        $redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
-          move_vm_queue(poolname, vm, 'ready', 'completed')
+        redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
+          move_vm_queue(poolname, vm, 'ready', 'completed', redis)
         end
       end
-      if $redis.scard("vmpooler__pending__#{poolname}") > 0
+      if redis.scard("vmpooler__pending__#{poolname}") > 0
         $logger.log('s', "[*] [#{poolname}] removing pending instances")
-        $redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
-          move_vm_queue(poolname, vm, 'pending', 'completed')
+        redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
+          move_vm_queue(poolname, vm, 'pending', 'completed', redis)
         end
       end
     end
 
-    def update_pool_template(pool, provider, configured_template, prepared_template)
+    def update_pool_template(pool, provider, configured_template, prepared_template, redis)
       pool['template'] = configured_template
       $logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
       # Remove all ready and pending VMs so new instances are created from the new template
-      drain_pool(pool['name'])
+      drain_pool(pool['name'], redis)
       # Prepare template for deployment
       $logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
-      prepare_template(pool, provider)
+      prepare_template(pool, provider, redis)
       $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
     def update_clone_target(pool)
       mutex = pool_mutex(pool['name'])
       return if mutex.locked?
-      clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
-      return if clone_target.nil?
-      return if clone_target == pool['clone_target']
-      $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
-      mutex.synchronize do
-        pool['clone_target'] = clone_target
-        # Remove all ready and pending VMs so new instances are created for the new clone_target
-        drain_pool(pool['name'])
+
+      @redis.with_metrics do |redis|
+        clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
+        break if clone_target.nil?
+        break if clone_target == pool['clone_target']
+
+        $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
+        mutex.synchronize do
+          pool['clone_target'] = clone_target
+          # Remove all ready and pending VMs so new instances are created for the new clone_target
+          drain_pool(pool['name'], redis)
+        end
+        $logger.log('s', "[*] [#{pool['name']}] is ready for use")
       end
-      $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
     def remove_excess_vms(pool)
-      ready = $redis.scard("vmpooler__ready__#{pool['name']}")
-      total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
-      return if total.nil?
-      return if total == 0
-      mutex = pool_mutex(pool['name'])
-      return if mutex.locked?
-      return unless ready > pool['size']
-      mutex.synchronize do
-        difference = ready - pool['size']
-        difference.times do
-          next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
-          move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
-        end
-        if total > ready
-          $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
-            move_vm_queue(pool['name'], vm, 'pending', 'completed')
+      @redis.with_metrics do |redis|
+        ready = redis.scard("vmpooler__ready__#{pool['name']}")
+        pending = redis.scard("vmpooler__pending__#{pool['name']}")
+        total = pending.to_i + ready.to_i
+        break if total.nil?
+        break if total == 0
+
+        mutex = pool_mutex(pool['name'])
+        break if mutex.locked?
+        break unless ready.to_i > pool['size']
+
+        mutex.synchronize do
+          difference = ready.to_i - pool['size']
+          difference.times do
+            next_vm = redis.spop("vmpooler__ready__#{pool['name']}")
+            move_vm_queue(pool['name'], next_vm, 'ready', 'completed', redis)
+          end
+          if total > ready
+            redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+              move_vm_queue(pool['name'], vm, 'pending', 'completed', redis)
+            end
           end
         end
       end
@@ -898,12 +1856,66 @@ module Vmpooler
     def update_pool_size(pool)
       mutex = pool_mutex(pool['name'])
       return if mutex.locked?
-      poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
-      return if poolsize.nil?
-      poolsize = Integer(poolsize)
-      return if poolsize == pool['size']
-      mutex.synchronize do
-        pool['size'] = poolsize
+
+      @redis.with_metrics do |redis|
+        pool_size_requested = redis.hget('vmpooler__config__poolsize', pool['name'])
+        break if pool_size_requested.nil?
+
+        pool_size_requested = Integer(pool_size_requested)
+        pool_size_currently = pool['size']
+        break if pool_size_requested == pool_size_currently
+
+        mutex.synchronize do
+          pool['size'] = pool_size_requested
+        end
+
+        $logger.log('s', "[*] [#{pool['name']}] size updated from #{pool_size_currently} to #{pool_size_requested}")
+      end
+    end
+
+    def reset_pool(pool)
+      poolname = pool['name']
+      @redis.with_metrics do |redis|
+        break unless redis.sismember('vmpooler__poolreset', poolname)
+
+        redis.srem('vmpooler__poolreset', poolname)
+        mutex = pool_mutex(poolname)
+        mutex.synchronize do
+          drain_pool(poolname, redis)
+          $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
+        end
+      end
+    end
+
+    def undo_override(pool, provider)
+      poolname = pool['name']
+      mutex = pool_mutex(poolname)
+      return if mutex.locked?
+
+      @redis.with_metrics do |redis|
+        break unless redis.sismember('vmpooler__pool__undo_template_override', poolname)
+
+        redis.srem('vmpooler__pool__undo_template_override', poolname)
+        template_now = pool['template']
+        template_original = $config[:pools_at_startup][$config[:pool_index][poolname]]['template']
+
+        mutex.synchronize do
+          update_pool_template(pool, provider, template_original, template_now, redis)
+        end
+      end
+
+      @redis.with_metrics do |redis|
+        break unless redis.sismember('vmpooler__pool__undo_size_override', poolname)
+
+        redis.srem('vmpooler__pool__undo_size_override', poolname)
+        pool_size_now = pool['size']
+        pool_size_original = $config[:pools_at_startup][$config[:pool_index][poolname]]['size']
+
+        mutex.synchronize do
+          pool['size'] = pool_size_original
+        end
+
+        $logger.log('s', "[*] [#{poolname}] size updated from #{pool_size_now} to #{pool_size_original}")
       end
     end
 
@@ -912,125 +1924,141 @@ module Vmpooler
       begin
         mutex = pool_mutex(pool['name'])
         mutex.synchronize do
-          provider.vms_in_pool(pool['name']).each do |vm|
-            if !$redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
-               !$redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
-               !$redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
-               !$redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
-               !$redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
-               !$redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
+          @redis.with_metrics do |redis|
+            provider.vms_in_pool(pool['name']).each do |vm|
+              if !redis.sismember("vmpooler__running__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__ready__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__pending__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__completed__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__discovered__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__migrating__#{pool['name']}", vm['name'])
 
-              pool_check_response[:discovered_vms] += 1
-              $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
+                pool_check_response[:discovered_vms] += 1
+                redis.sadd("vmpooler__discovered__#{pool['name']}", vm['name'])
 
-              $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
+                $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
+              end
+
+              inventory[vm['name']] = 1
             end
-
-            inventory[vm['name']] = 1
           end
         end
-      rescue => err
-        $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{err}")
-        raise(err)
+      rescue StandardError => e
+        $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{e}")
+        raise(e)
       end
       inventory
     end
 
     def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
-      $redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
-        if inventory[vm]
-          begin
-            vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
-            pool_check_response[:checked_running_vms] += 1
-            check_running_vm(vm, pool_name, vm_lifetime, provider)
-          rescue => err
-            $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{err}")
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
+          if inventory[vm]
+            begin
+              vm_lifetime = redis.hget("vmpooler__vm__#{vm}", 'lifetime') || $config[:config]['vm_lifetime'] || 12
+              pool_check_response[:checked_running_vms] += 1
+              check_running_vm(vm, pool_name, vm_lifetime, provider)
+            rescue StandardError => e
+              $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
+            end
+          else
+            move_vm_queue(pool_name, vm, 'running', 'completed', redis, 'is a running VM but is missing from inventory.  Marking as completed.')
           end
-        else
-          move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory.  Marking as completed.')
         end
       end
     end
 
-    def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
-      $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
-        if inventory[vm]
-          begin
-            pool_check_response[:checked_ready_vms] += 1
-            check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
-          rescue => err
-            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}")
+    def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
+          if inventory[vm]
+            begin
+              pool_check_response[:checked_ready_vms] += 1
+              check_ready_vm(vm, pool_name, pool_ttl, provider)
+            rescue StandardError => e
+              $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
+            end
+          else
+            move_vm_queue(pool_name, vm, 'ready', 'completed', redis, 'is a ready VM but is missing from inventory.  Marking as completed.')
           end
-        else
-          move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory.  Marking as completed.')
         end
       end
     end
 
-    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout = nil)
+    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
       pool_timeout ||= $config[:config]['timeout'] || 15
-      $redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
-        if inventory[vm]
-          begin
-            pool_check_response[:checked_pending_vms] += 1
-            check_pending_vm(vm, pool_name, pool_timeout, provider)
-          rescue => err
-            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{err}")
+      pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
+          if inventory[vm]
+            begin
+              pool_check_response[:checked_pending_vms] += 1
+              check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
+            rescue StandardError => e
+              $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
+            end
+          else
+            fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
           end
-        else
-          fail_pending_vm(vm, pool_name, pool_timeout, false)
         end
       end
     end
 
     def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
-      $redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
-        if inventory[vm]
-          begin
-            pool_check_response[:destroyed_vms] += 1
-            destroy_vm(vm, pool_name, provider)
-          rescue => err
-            $redis.srem("vmpooler__completed__#{pool_name}", vm)
-            $redis.hdel("vmpooler__active__#{pool_name}", vm)
-            $redis.del("vmpooler__vm__#{vm}")
-            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{err}")
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
+          if inventory[vm]
+            begin
+              pool_check_response[:destroyed_vms] += 1
+              dns_plugin = get_dns_plugin_class_for_pool(pool_name)
+              destroy_vm(vm, pool_name, provider, dns_plugin)
+            rescue StandardError => e
+              redis.pipelined do |pipeline|
+                pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+                pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+                pipeline.del("vmpooler__vm__#{vm}")
+              end
+              $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
+            end
+          else
+            $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+              pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+              pipeline.del("vmpooler__vm__#{vm}")
+            end
           end
-        else
-          $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
-          $redis.srem("vmpooler__completed__#{pool_name}", vm)
-          $redis.hdel("vmpooler__active__#{pool_name}", vm)
-          $redis.del("vmpooler__vm__#{vm}")
         end
       end
     end
 
     def check_discovered_pool_vms(pool_name)
-      begin
-        $redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
           %w[pending ready running completed].each do |queue|
-            if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
+            if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
               $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
-              $redis.srem("vmpooler__discovered__#{pool_name}", vm)
+              redis.srem("vmpooler__discovered__#{pool_name}", vm)
             end
           end
 
-          if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
-            $redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
-          end
+          redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if redis.sismember("vmpooler__discovered__#{pool_name}", vm)
         end
-      rescue => err
-        $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{err}")
       end
+    rescue StandardError => e
+      $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
     end
 
     def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
-      $redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
-        if inventory[vm]
-          begin
-            pool_check_response[:migrated_vms] += 1
-            migrate_vm(vm, pool_name, provider)
-          rescue => err
-            $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{err}")
+      @redis.with_metrics do |redis|
+        redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
+          if inventory[vm]
+            begin
+              pool_check_response[:migrated_vms] += 1
+              migrate_vm(vm, pool_name, provider)
+            rescue StandardError => e
+              $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
+            end
           end
         end
       end
@@ -1038,29 +2066,39 @@ module Vmpooler
 
     def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
       return if pool_mutex(pool_name).locked?
-      ready = $redis.scard("vmpooler__ready__#{pool_name}")
-      total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
 
-      $metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
-      $metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
+      @redis.with_metrics do |redis|
+        ready = redis.scard("vmpooler__ready__#{pool_name}")
+        pending = redis.scard("vmpooler__pending__#{pool_name}")
+        running = redis.scard("vmpooler__running__#{pool_name}")
 
-      if $redis.get("vmpooler__empty__#{pool_name}")
-        $redis.del("vmpooler__empty__#{pool_name}") unless ready.zero?
-      elsif ready.zero?
-        $redis.set("vmpooler__empty__#{pool_name}", 'true')
-        $logger.log('s', "[!] [#{pool_name}] is empty")
-      end
+        total = pending.to_i + ready.to_i
 
-      (pool_size - total).times do
-        if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
-          begin
-            $redis.incr('vmpooler__tasks__clone')
-            pool_check_response[:cloned_vms] += 1
-            clone_vm(pool_name, provider)
-          rescue => err
-            $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{err}")
-            $redis.decr('vmpooler__tasks__clone')
-            raise
+        $metrics.gauge("ready.#{pool_name}", ready)
+        $metrics.gauge("running.#{pool_name}", running)
+
+        dns_plugin = get_dns_plugin_class_for_pool(pool_name)
+
+        unless pool_size == 0
+          if redis.get("vmpooler__empty__#{pool_name}")
+            redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
+          elsif ready == 0
+            redis.set("vmpooler__empty__#{pool_name}", 'true')
+            $logger.log('s', "[!] [#{pool_name}] is empty")
+          end
+        end
+
+        (pool_size - total.to_i).times do
+          if redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
+            begin
+              redis.incr('vmpooler__tasks__clone')
+              pool_check_response[:cloned_vms] += 1
+              clone_vm(pool_name, provider, dns_plugin)
+            rescue StandardError => e
+              $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
+              redis.decr('vmpooler__tasks__clone')
+              raise
+            end
           end
         end
       end
@@ -1079,15 +2117,15 @@ module Vmpooler
 
       begin
         inventory = create_inventory(pool, provider, pool_check_response)
-      rescue => err
+      rescue StandardError
         return(pool_check_response)
       end
 
       check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
 
-      check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
+      check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
 
-      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
+      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
 
       check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
 
@@ -1115,6 +2153,12 @@ module Vmpooler
       # Remove VMs in excess of the configured pool size
       remove_excess_vms(pool)
 
+      # Reset a pool when poolreset is requested from the API
+      reset_pool(pool)
+
+      # Undo overrides submitted via the api
+      undo_override(pool, provider)
+
       pool_check_response
     end
 
@@ -1125,22 +2169,219 @@ module Vmpooler
     #
     # returns an object Vmpooler::PoolManager::Provider::*
     # or raises an error if the class does not exist
-    def create_provider_object(config, logger, metrics, provider_class, provider_name, options)
+    def create_provider_object(config, logger, metrics, redis_connection_pool, provider_class, provider_name, options)
       provider_klass = Vmpooler::PoolManager::Provider
       provider_klass.constants.each do |classname|
         next unless classname.to_s.casecmp(provider_class) == 0
-        return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
+
+        return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
+      end
+      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil?
+    end
+
+    def create_dns_object(config, logger, metrics, redis_connection_pool, dns_class, dns_name, options)
+      if defined?(Vmpooler::PoolManager::Dns)
+        dns_klass = Vmpooler::PoolManager::Dns
+        dns_klass.constants.each do |classname|
+          next unless classname.to_s.casecmp(dns_class) == 0
+
+          return dns_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, dns_name, options)
+        end
+        raise("DNS '#{dns_class}' is unknown for pool with dns name '#{dns_name}'") if dns_klass.nil?
+      end
+    end
+
+    def check_ondemand_requests(maxloop = 0,
+                                loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
+                                loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
+                                loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
+
+      $logger.log('d', '[*] [ondemand_provisioner] starting worker thread')
+
+      $threads['ondemand_provisioner'] = Thread.new do
+        _check_ondemand_requests(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
+      end
+    end
+
+    def _check_ondemand_requests(maxloop = 0,
+                                 loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
+                                 loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
+                                 loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
+
+      loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
+      loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
+      loop_delay_decay = $config[:config]['check_loop_delay_decay'] unless $config[:config]['check_loop_delay_decay'].nil?
+
+      loop_delay_decay = 2.0 if loop_delay_decay <= 1.0
+      loop_delay_max = loop_delay_min if loop_delay_max.nil? || loop_delay_max < loop_delay_min
+
+      loop_count = 1
+      loop_delay = loop_delay_min
+
+      loop do
+        result = process_ondemand_requests
+
+        loop_delay = (loop_delay * loop_delay_decay).to_i
+        loop_delay = loop_delay_min if result > 0
+        loop_delay = loop_delay_max if loop_delay > loop_delay_max
+        sleep_with_wakeup_events(loop_delay, loop_delay_min, ondemand_request: true)
+
+        unless maxloop == 0
+          break if loop_count >= maxloop
+
+          loop_count += 1
+        end
+      end
+    end
+
+    def process_ondemand_requests
+      @redis.with_metrics do |redis|
+        requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
+        requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
+        provisioning_tasks = process_ondemand_vms(redis)
+        requests_ready = check_ondemand_requests_ready(redis)
+        requests.length + provisioning_tasks + requests_ready
+      end
+    end
+
+    def create_ondemand_vms(request_id, redis)
+      requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
+      unless requested
+        $logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
+        redis.zrem('vmpooler__provisioning__request', request_id)
+        return
+      end
+
+      # Check if request was already marked as failed (e.g., by delete endpoint)
+      request_status = redis.hget("vmpooler__odrequest__#{request_id}", 'status')
+      if request_status == 'failed'
+        $logger.log('s', "Request '#{request_id}' already marked as failed, skipping VM creation")
+        redis.zrem('vmpooler__provisioning__request', request_id)
+        return
+      end
+
+      score = redis.zscore('vmpooler__provisioning__request', request_id)
+      requested = requested.split(',')
+
+      redis.pipelined do |pipeline|
+        requested.each do |request|
+          pipeline.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
+        end
+        pipeline.zrem('vmpooler__provisioning__request', request_id)
+        pipeline.zadd('vmpooler__provisioning__processing', score, request_id)
+      end
+    end
+
+    def process_ondemand_vms(redis)
+      queue_key = 'vmpooler__odcreate__task'
+      queue = redis.zrange(queue_key, 0, -1, with_scores: true)
+      ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
+      queue.each do |request, score|
+        clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
+        break unless clone_count < ondemand_clone_limit
+
+        pool_alias, pool, count, request_id = request.split(':')
+        count = count.to_i
+        provider = get_provider_for_pool(pool)
+        dns_plugin = get_dns_plugin_class_for_pool(pool)
+        slots = ondemand_clone_limit - clone_count
+        break if slots == 0
+
+        if slots >= count
+          count.times do
+            redis.incr('vmpooler__tasks__ondemandclone')
+            clone_vm(pool, provider, dns_plugin, request_id, pool_alias)
+          end
+          redis.zrem(queue_key, request)
+        else
+          remaining_count = count - slots
+          slots.times do
+            redis.incr('vmpooler__tasks__ondemandclone')
+            clone_vm(pool, provider, dns_plugin, request_id, pool_alias)
+          end
+          redis.pipelined do |pipeline|
+            pipeline.zrem(queue_key, request)
+            pipeline.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
+          end
+        end
+      end
+      queue.length
+    end
+
+    def vms_ready?(request_id, redis)
+      catch :request_not_ready do
+        request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+          pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+          throw :request_not_ready unless pools_filled.to_i == count.to_i
+        end
+        return true
+      end
+      false
+    end
+
+    def check_ondemand_requests_ready(redis)
+      in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1, with_scores: true)
+      in_progress_requests&.each do |request_id, score|
+        check_ondemand_request_ready(request_id, redis, score)
+      end
+      in_progress_requests.length
+    end
+
+    def check_ondemand_request_ready(request_id, redis, score = nil)
+      # default expiration is one month to ensure the data does not stay in redis forever
+      default_expiration = 259_200_0
+      processing_key = 'vmpooler__provisioning__processing'
+      ondemand_hash_key = "vmpooler__odrequest__#{request_id}"
+      score ||= redis.zscore(processing_key, request_id)
+      return if request_expired?(request_id, score, redis)
+
+      return unless vms_ready?(request_id, redis)
+
+      redis.hset(ondemand_hash_key, 'status', 'ready')
+      redis.expire(ondemand_hash_key, default_expiration)
+      redis.zrem(processing_key, request_id)
+    end
+
+    def request_expired?(request_id, score, redis)
+      delta = Time.now.to_i - score.to_i
+      ondemand_request_ttl = $config[:config]['ondemand_request_ttl']
+      return false unless delta > ondemand_request_ttl * 60
+
+      $logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
+      expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
+      redis.pipelined do |pipeline|
+        pipeline.zrem('vmpooler__provisioning__processing', request_id)
+        pipeline.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+        pipeline.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
+      end
+      remove_vms_for_failed_request(request_id, expiration_ttl, redis)
+      true
+    end
+
+    def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
+      request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+      Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+        pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+        redis.pipelined do |pipeline|
+          pools_filled&.each do |vm|
+            move_vm_queue(pool, vm, 'running', 'completed', pipeline, "moved to completed queue. '#{request_id}' could not be filled in time")
+          end
+          pipeline.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+        end
       end
-      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
     end
 
     def execute!(maxloop = 0, loop_delay = 1)
       $logger.log('d', 'starting vmpooler')
 
-      # Clear out the tasks manager, as we don't know about any tasks at this point
-      $redis.set('vmpooler__tasks__clone', 0)
-      # Clear out vmpooler__migrations since stale entries may be left after a restart
-      $redis.del('vmpooler__migration')
+      @redis.with_metrics do |redis|
+        # Clear out the tasks manager, as we don't know about any tasks at this point
+        redis.set('vmpooler__tasks__clone', 0)
+        redis.set('vmpooler__tasks__ondemandclone', 0)
+        # Clear out vmpooler__migrations since stale entries may be left after a restart
+        redis.del('vmpooler__migration')
+      end
 
       # Copy vSphere settings to correct location.  This happens with older configuration files
       if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
@@ -1152,8 +2393,8 @@ module Vmpooler
       # Set default provider for all pools that do not have one defined
       $config[:pools].each do |pool|
         if pool['provider'].nil?
-          $logger.log('d', "[!] Setting provider for pool '#{pool['name']}' to 'vsphere' as default")
-          pool['provider'] = 'vsphere'
+          $logger.log('d', "[!] Setting provider for pool '#{pool['name']}' to 'dummy' as default")
+          pool['provider'] = 'dummy'
         end
       end
 
@@ -1169,35 +2410,46 @@ module Vmpooler
       # Create the providers
       $config[:pools].each do |pool|
         provider_name = pool['provider']
+        dns_plugin_name = pool['dns_plugin']
         # The provider_class parameter can be defined in the provider's data eg
-        #:providers:
-        # :vsphere:
-        #  provider_class: 'vsphere'
-        # :another-vsphere:
-        #  provider_class: 'vsphere'
+        # :providers:
+        #   :vsphere:
+        #     provider_class: 'vsphere'
+        #   :another-vsphere:
+        #     provider_class: 'vsphere'
         # the above would create two providers/vsphere.rb class objects named 'vsphere' and 'another-vsphere'
         # each pools would then define which provider definition to use: vsphere or another-vsphere
         #
         # if provider_class is not defined it will try to use the provider_name as the class, this is to be
         # backwards compatible for example when there is only one provider listed
         # :providers:
-        #  :dummy:
-        #   filename: 'db.txs'
+        #   :dummy:
+        #     filename: 'db.txs'
         # the above example would create an object based on the class providers/dummy.rb
         if $config[:providers].nil? || $config[:providers][provider_name.to_sym].nil? || $config[:providers][provider_name.to_sym]['provider_class'].nil?
           provider_class = provider_name
         else
           provider_class = $config[:providers][provider_name.to_sym]['provider_class']
         end
+
         begin
-          $providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
-        rescue => err
-          $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{err}")
+          $providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
+        rescue StandardError => e
+          $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
+          raise
+        end
+
+        dns_plugin_class = $config[:dns_configs][dns_plugin_name.to_sym]['dns_class']
+
+        begin
+          $dns_plugins[dns_plugin_class] = create_dns_object($config, $logger, $metrics, @redis, dns_plugin_class, dns_plugin_name, {}) if $dns_plugins[dns_plugin_class].nil?
+        rescue StandardError => e
+          $logger.log('s', "Error while creating dns plugin for pool #{pool['name']}: #{e}")
           raise
         end
       end
 
-      purge_unused_vms_and_folders
+      purge_unused_vms_and_resources
 
       loop_count = 1
       loop do
@@ -1224,13 +2476,66 @@ module Vmpooler
           end
         end
 
+        if !$threads['ondemand_provisioner']
+          check_ondemand_requests
+        elsif !$threads['ondemand_provisioner'].alive?
+          $logger.log('d', '[!] [ondemand_provisioner] worker thread died, restarting')
+          check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
+        end
+
+        # Queue purge thread
+        if purge_enabled?
+          purge_interval = ($config[:config] && $config[:config]['purge_interval']) || 3600 # default 1 hour
+          if !$threads['queue_purge']
+            $threads['queue_purge'] = Thread.new do
+              loop do
+                purge_stale_queue_entries
+                sleep(purge_interval)
+              end
+            end
+          elsif !$threads['queue_purge'].alive?
+            $logger.log('d', '[!] [queue_purge] worker thread died, restarting')
+            $threads['queue_purge'] = Thread.new do
+              loop do
+                purge_stale_queue_entries
+                sleep(purge_interval)
+              end
+            end
+          end
+        end
+
+        # Health check thread
+        if health_check_enabled?
+          health_interval = ($config[:config] && $config[:config]['health_check_interval']) || 300 # default 5 minutes
+          if !$threads['health_check']
+            $threads['health_check'] = Thread.new do
+              loop do
+                check_queue_health
+                sleep(health_interval)
+              end
+            end
+          elsif !$threads['health_check'].alive?
+            $logger.log('d', '[!] [health_check] worker thread died, restarting')
+            $threads['health_check'] = Thread.new do
+              loop do
+                check_queue_health
+                sleep(health_interval)
+              end
+            end
+          end
+        end
+
         sleep(loop_delay)
 
-        unless maxloop.zero?
+        unless maxloop == 0
           break if loop_count >= maxloop
+
           loop_count += 1
         end
       end
+    rescue Redis::CannotConnectError => e
+      $logger.log('s', "Cannot connect to the redis server: #{e}")
+      raise
     end
   end
 end
diff --git a/lib/vmpooler/providers.rb b/lib/vmpooler/providers.rb
index 9a5d955..94673f4 100644
--- a/lib/vmpooler/providers.rb
+++ b/lib/vmpooler/providers.rb
@@ -1,27 +1,28 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Vmpooler
   class Providers
-
     # @param names [Array] - an array of names or string name of a provider
     # @return [Array] - list of provider files loaded
     # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
     def self.load_by_name(names)
       names = Array(names)
-      instance = self.new
-      names.map {|name| instance.load_from_gems(name)}.flatten
+      instance = new
+      names.map { |name| instance.load_from_gems(name) }.flatten
     end
 
     # @return [Array] - array of provider files
     # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
     # although these files can come from any gem
     def self.load_all_providers
-      self.new.load_from_gems
+      new.load_from_gems
     end
 
     # @return [Array] - returns an array of gem names that contain a provider
     def self.installed_providers
-      self.new.vmpooler_provider_gem_list.map(&:name)
+      new.vmpooler_provider_gem_list.map(&:name)
     end
 
     # @return [Array] returns a list of vmpooler providers gem plugin specs
@@ -38,7 +39,7 @@ module Vmpooler
         # we don't exactly know if the provider name matches the main file name that should be loaded
         # so we use globs to get everything like the name
         # this could mean that vsphere5 and vsphere6 are loaded when only vsphere5 is used
-        Dir.glob(File.join(gem_path, "*#{name}*.rb")).each do |file|
+        Dir.glob(File.join(gem_path, "*#{name}*.rb")).sort.each do |file|
           require file
         end
       end
@@ -50,7 +51,7 @@ module Vmpooler
     # @return [String] - the relative path to the vmpooler provider dir
     # this is used when searching gems for this path
     def provider_path
-      File.join('lib','vmpooler','providers')
+      File.join('lib', 'vmpooler', 'providers')
     end
 
     # Add constants to array to skip over classes, ie. Vmpooler::PoolManager::Provider::Dummy
@@ -81,26 +82,24 @@ module Vmpooler
       @plugin_map ||= Hash[plugin_classes.map { |gem| [gem.send(:name), gem] }]
     end
 
-
-
     # Internal: Retrieve a list of available gem paths from RubyGems.
     #
     # Returns an Array of Pathname objects.
     def gem_directories
       dirs = []
-      if has_rubygems?
+      if rubygems?
         dirs = gemspecs.map do |spec|
-          lib_path = File.expand_path(File.join(spec.full_gem_path,provider_path))
-          lib_path if File.exists? lib_path
+          lib_path = File.expand_path(File.join(spec.full_gem_path, provider_path))
+          lib_path if File.exist? lib_path
         end + included_lib_dirs
       end
-      dirs.reject { |dir| dir.nil? }.uniq
+      dirs.reject(&:nil?).uniq
     end
 
     # Internal: Check if RubyGems is loaded and available.
     #
     # Returns true if RubyGems is available, false if not.
-    def has_rubygems?
+    def rubygems?
       defined? ::Gem
     end
 
@@ -114,6 +113,5 @@ module Vmpooler
                       Gem.searcher.init_gemspecs
                     end
     end
-
   end
 end
diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb
index d30564f..e5d458e 100644
--- a/lib/vmpooler/providers/base.rb
+++ b/lib/vmpooler/providers/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class PoolManager
     class Provider
@@ -12,10 +14,11 @@ module Vmpooler
         # Provider options passed in during initialization
         attr_reader :provider_options
 
-        def initialize(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
           @config = config
           @logger = logger
           @metrics = metrics
+          @redis = redis_connection_pool
           @provider_name = name
 
           # Ensure that there is not a nil provider configuration
@@ -55,6 +58,10 @@ module Vmpooler
           nil
         end
 
+        def dns_config(dns_config_name)
+          Vmpooler::Dns.get_dns_plugin_domain_by_name(@config, dns_config_name)
+        end
+
         # returns
         #   [Hashtable] : The entire VMPooler configuration
         def global_config
@@ -209,6 +216,22 @@ module Vmpooler
           raise("#{self.class.name} does not implement vm_ready?")
         end
 
+        # tag_vm_user This method is called once we know who is using the VM (it is running). This method enables seeing
+        # who is using what in the provider pools.
+        # This method should be implemented in the providers, if it is not implemented, this base method will be called
+        # and should be a noop. The implementation should check if the vm has a user (as per redis) and add a new tag
+        # with the information.
+        # inputs
+        #   [String] pool_name : Name of the pool
+        #   [String] vm_name   : Name of the VM to check if ready
+        # returns
+        #   [Boolean] : true if successful, false if an error occurred and it should retry
+        def tag_vm_user(_pool_name, _vm_name)
+          # noop by design. If the provider does not implement this method, this base method is called (because inherited)
+          # and should basically do nothing.
+          true
+        end
+
         # inputs
         #   [String] pool_name : Name of the pool
         #   [String] vm_name   : Name of the VM to check if it exists
@@ -222,20 +245,31 @@ module Vmpooler
         #   [Hash] pool : Configuration for the pool
         # returns
         #   nil when successful. Raises error when encountered
-        def create_template_delta_disks(pool)
-          raise("#{self.class.name} does not implement create_template_delta_disks")
+        def create_template_delta_disks(_pool)
+          puts("#{self.class.name} does not implement create_template_delta_disks")
         end
 
         # inputs
         #   [String] provider_name : Name of the provider
         # returns
         #   Hash of folders
-        def get_target_datacenter_from_config(provider_name)
+        def get_target_datacenter_from_config(_provider_name)
           raise("#{self.class.name} does not implement get_target_datacenter_from_config")
         end
 
-        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        def purge_unconfigured_resources(_allowlist)
+          raise("#{self.class.name} does not implement purge_unconfigured_resources")
+        end
+
+        def get_vm_ip_address(vm_name, pool_name)
+          raise("#{self.class.name} does not implement get_vm_ip_address for vm #{vm_name} in pool #{pool_name}")
+        end
+
+        # DEPRECATED if a provider does not implement the new method, it will hit this base class method
+        # and return a deprecation message
+        def purge_unconfigured_folders(_deprecated, _deprecated2, allowlist)
+          logger.log('s', '[!] purge_unconfigured_folders was renamed to purge_unconfigured_resources, please update your provider implementation')
+          purge_unconfigured_resources(allowlist)
         end
       end
     end
diff --git a/lib/vmpooler/providers/dummy.rb b/lib/vmpooler/providers/dummy.rb
index bc924a0..0a1a60d 100644
--- a/lib/vmpooler/providers/dummy.rb
+++ b/lib/vmpooler/providers/dummy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'vmpooler/providers/base'
 
@@ -7,8 +9,8 @@ module Vmpooler
       class Dummy < Vmpooler::PoolManager::Provider::Base
         # Fake VM Provider for testing
 
-        def initialize(config, logger, metrics, name, options)
-          super(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          super(config, logger, metrics, redis_connection_pool, name, options)
           dummyfilename = provider_config['filename']
 
           # This initial_state option is only intended to be used by spec tests
@@ -27,7 +29,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do
@@ -73,10 +76,10 @@ module Vmpooler
           return current_vm['vm_host'] if provider_config['migratevm_couldmove_percent'].nil?
 
           # Only migrate if migratevm_couldmove_percent is met
-          return current_vm['vm_host'] if 1 + rand(100) > provider_config['migratevm_couldmove_percent']
+          return current_vm['vm_host'] if rand(1..100) > provider_config['migratevm_couldmove_percent']
 
           # Simulate a 10 node cluster and randomly pick a different one
-          new_host = 'HOST' + (1 + rand(10)).to_s while new_host == current_vm['vm_host']
+          new_host = "HOST#{rand(1..10)}" while new_host == current_vm['vm_host']
 
           new_host
         end
@@ -92,9 +95,7 @@ module Vmpooler
             end
 
             # Inject clone failure
-            unless provider_config['migratevm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['migratevm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['migratevm_fail_percent'].nil? && rand(1..100) <= provider_config['migratevm_fail_percent']
 
             @write_lock.synchronize do
               current_vm = get_dummy_vm(pool_name, vm_name)
@@ -113,27 +114,23 @@ module Vmpooler
             return nil if dummy.nil?
 
             # Randomly power off the VM
-            unless dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?
-              if 1 + rand(100) <= provider_config['getvm_poweroff_percent']
-                @write_lock.synchronize do
-                  dummy = get_dummy_vm(pool_name, vm_name)
-                  dummy['powerstate'] = 'PoweredOff'
-                  write_backing_file
-                end
-                logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
+            if !(dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?) && rand(1..100) <= provider_config['getvm_poweroff_percent']
+              @write_lock.synchronize do
+                dummy = get_dummy_vm(pool_name, vm_name)
+                dummy['powerstate'] = 'PoweredOff'
+                write_backing_file
               end
+              logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
             end
 
             # Randomly rename the host
-            unless dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?
-              if 1 + rand(100) <= provider_config['getvm_rename_percent']
-                @write_lock.synchronize do
-                  dummy = get_dummy_vm(pool_name, vm_name)
-                  dummy['hostname'] = 'DUMMY' + dummy['name']
-                  write_backing_file
-                end
-                logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
+            if !(dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?) && rand(1..100) <= provider_config['getvm_rename_percent']
+              @write_lock.synchronize do
+                dummy = get_dummy_vm(pool_name, vm_name)
+                dummy['hostname'] = "DUMMY#{dummy['name']}"
+                write_backing_file
               end
+              logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
             end
 
             obj['name'] = dummy['name']
@@ -193,16 +190,14 @@ module Vmpooler
 
             begin
               # Inject clone failure
-              unless provider_config['createvm_fail_percent'].nil?
-                raise('Dummy Failure for createvm_fail_percent') if 1 + rand(100) <= provider_config['createvm_fail_percent']
-              end
+              raise('Dummy Failure for createvm_fail_percent') if !provider_config['createvm_fail_percent'].nil? && rand(1..100) <= provider_config['createvm_fail_percent']
 
               # Assert the VM is ready for use
               @write_lock.synchronize do
                 vm['dummy_state'] = 'RUNNING'
                 write_backing_file
               end
-            rescue => _err
+            rescue StandardError => _e
               @write_lock.synchronize do
                 remove_dummy_vm(pool_name, dummy_hostname)
                 write_backing_file
@@ -226,9 +221,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createdisk_fail_percent'].nil?
-              raise('Dummy Failure for createdisk_fail_percent') if 1 + rand(100) <= provider_config['createdisk_fail_percent']
-            end
+            raise('Dummy Failure for createdisk_fail_percent') if !provider_config['createdisk_fail_percent'].nil? && rand(1..100) <= provider_config['createdisk_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -252,9 +245,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for createsnapshot_fail_percent') if 1 + rand(100) <= provider_config['createsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for createsnapshot_fail_percent') if !provider_config['createsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['createsnapshot_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -279,9 +270,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['revertsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for revertsnapshot_fail_percent') if 1 + rand(100) <= provider_config['revertsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for revertsnapshot_fail_percent') if !provider_config['revertsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['revertsnapshot_fail_percent']
           end
 
           vm_object['snapshots'].include?(snapshot_name)
@@ -317,9 +306,7 @@ module Vmpooler
             end
 
             # Inject destroy VM failure
-            unless provider_config['destroyvm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['destroyvm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['destroyvm_fail_percent'].nil? && rand(1..100) <= provider_config['destroyvm_fail_percent']
 
             # 'Destroy' the VM
             @write_lock.synchronize do
@@ -351,9 +338,7 @@ module Vmpooler
             # it's ready to receive a connection
             sleep(2)
 
-            unless provider_config['vmready_fail_percent'].nil?
-              raise('Dummy Failure for vmready_fail_percent') if 1 + rand(100) <= provider_config['vmready_fail_percent']
-            end
+            raise('Dummy Failure for vmready_fail_percent') if !provider_config['vmready_fail_percent'].nil? && rand(1..100) <= provider_config['vmready_fail_percent']
 
             @write_lock.synchronize do
               vm_object['ready'] = true
@@ -370,6 +355,7 @@ module Vmpooler
 
         def remove_dummy_vm(pool_name, vm_name)
           return if @dummylist['pool'][pool_name].nil?
+
           new_poollist = @dummylist['pool'][pool_name].delete_if { |vm| vm['name'] == vm_name }
           @dummylist['pool'][pool_name] = new_poollist
         end
@@ -395,6 +381,7 @@ module Vmpooler
         def write_backing_file
           dummyfilename = provider_config['filename']
           return if dummyfilename.nil?
+
           File.open(dummyfilename, 'w') { |file| file.write(YAML.dump(@dummylist)) }
         end
       end
diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb
deleted file mode 100644
index d0ca8c6..0000000
--- a/lib/vmpooler/providers/vsphere.rb
+++ /dev/null
@@ -1,1047 +0,0 @@
-require 'vmpooler/providers/base'
-
-module Vmpooler
-  class PoolManager
-    class Provider
-      class VSphere < Vmpooler::PoolManager::Provider::Base
-        # The connection_pool method is normally used only for testing
-        attr_reader :connection_pool
-
-        def initialize(config, logger, metrics, name, options)
-          super(config, logger, metrics, name, options)
-
-          task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i
-          # The default connection pool size is:
-          # Whatever is biggest from:
-          #   - How many pools this provider services
-          #   - Maximum number of cloning tasks allowed
-          #   - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc.
-          default_connpool_size = [provided_pools.count, task_limit, 2].max
-          connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i
-          # The default connection pool timeout should be quite large - 60 seconds
-          connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i
-          logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
-          @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
-            metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
-            size: connpool_size,
-            timeout: connpool_timeout
-          ) do
-            logger.log('d', "[#{name}] Connection Pool - Creating a connection object")
-            # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve
-            # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection
-            # object.  Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the
-            # Hash can change, and is preserved across invocations.
-            new_conn = connect_to_vsphere
-            { connection: new_conn }
-          end
-          @provider_hosts = {}
-          @provider_hosts_lock = Mutex.new
-        end
-
-        # name of the provider class
-        def name
-          'vsphere'
-        end
-
-        def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
-          if whitelist
-            return true if whitelist.include?(folder_title)
-          end
-          return false unless configured_folders.keys.include?(folder_title)
-          return false unless configured_folders[folder_title] == base_folder
-          return true
-        end
-
-        def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
-          try = 0 if try.nil?
-          max_tries = 3
-          $redis.srem("vmpooler__completed__#{pool}", vm_name)
-          $redis.hdel("vmpooler__active__#{pool}", vm_name)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
-
-          # Auto-expire metadata key
-          $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
-
-          start = Time.now
-
-          if vm_object.is_a? RbVmomi::VIM::Folder
-            logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
-            raise('Expected VM, but received a folder object')
-          end
-          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
-          vm_object.Destroy_Task.wait_for_completion
-
-          finish = format('%.2f', Time.now - start)
-          logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
-          metrics.timing("destroy.#{pool}", finish)
-        rescue RuntimeError
-          raise
-        rescue => err
-          try += 1
-          logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}")
-          try >= max_tries ? raise : retry
-        end
-
-        def destroy_folder_and_children(folder_object)
-          vms = {}
-          data_ttl = $config[:redis]['data_ttl'].to_i
-          folder_name = folder_object.name
-          unless folder_object.childEntity.count == 0
-            folder_object.childEntity.each do |vm|
-              vms[vm.name] = vm
-            end
-
-            vms.each do |vm_name, vm_object|
-              destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
-            end
-          end
-          destroy_folder(folder_object)
-        end
-
-        def destroy_folder(folder_object)
-          try = 0 if try.nil?
-          max_tries = 3
-          logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
-          folder_object.Destroy_Task.wait_for_completion
-        rescue
-          try += 1
-          try >= max_tries ? raise : retry
-        end
-
-        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-
-            base_folders.each do |base_folder|
-              folder_children = get_folder_children(base_folder, connection)
-              unless folder_children.empty?
-                folder_children.each do |folder_hash|
-                  folder_hash.each do |folder_title, folder_object|
-                    unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
-                      destroy_folder_and_children(folder_object)
-                    end
-                  end
-                end
-              end
-            end
-          end
-        end
-
-        def get_folder_children(folder_name, connection)
-          folders = []
-
-          propSpecs = {
-            :entity => self,
-            :inventoryPath => folder_name
-          }
-          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
-
-          return folders if folder_object.nil?
-
-          folder_object.childEntity.each do |folder|
-            next unless folder.is_a? RbVmomi::VIM::Folder
-            folders << { folder.name => folder }
-          end
-
-          folders
-        end
-
-        def vms_in_pool(pool_name)
-          vms = []
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            folder_object = find_vm_folder(pool_name, connection)
-
-            return vms if folder_object.nil?
-
-            folder_object.childEntity.each do |vm|
-              vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
-            end
-          end
-          vms
-        end
-
-        def select_target_hosts(target, cluster, datacenter)
-          percentage = 100
-          dc = "#{datacenter}_#{cluster}"
-          @provider_hosts_lock.synchronize do
-            begin
-              target[dc] = {} unless target.key?(dc)
-              target[dc]['checking'] = true
-              hosts_hash = find_least_used_hosts(cluster, datacenter, percentage)
-              target[dc] = hosts_hash
-            rescue => _err
-              target[dc] = {}
-              raise(_err)
-            ensure
-              target[dc]['check_time_finished'] = Time.now
-            end
-          end
-        end
-
-        def run_select_hosts(pool_name, target)
-          now = Time.now
-          max_age = @config[:config]['host_selection_max_age'] || 60
-          loop_delay = 5
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-          dc = "#{datacenter}_#{cluster}"
-          unless target.key?(dc)
-            select_target_hosts(target, cluster, datacenter)
-            return
-          end
-          if target[dc].key?('checking')
-            wait_for_host_selection(dc, target, loop_delay, max_age)
-          end
-          if target[dc].key?('check_time_finished')
-            if now - target[dc]['check_time_finished'] > max_age
-              select_target_hosts(target, cluster, datacenter)
-            end
-          end
-        end
-
-        def wait_for_host_selection(dc, target, maxloop = 0, loop_delay = 1, max_age = 60)
-          loop_count = 1
-          until target.key?(dc) and target[dc].key?('check_time_finished')
-            sleep(loop_delay)
-            unless maxloop.zero?
-              break if loop_count >= maxloop
-              loop_count += 1
-            end
-          end
-          return unless target[dc].key?('check_time_finished')
-          loop_count = 1
-          while Time.now - target[dc]['check_time_finished'] > max_age
-            sleep(loop_delay)
-            unless maxloop.zero?
-              break if loop_count >= maxloop
-              loop_count += 1
-            end
-          end
-        end
-
-        def select_next_host(pool_name, target, architecture = nil)
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-          dc = "#{datacenter}_#{cluster}"
-          @provider_hosts_lock.synchronize do
-            if architecture
-              raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('architectures')
-              host = target[dc]['architectures'][architecture].shift
-              target[dc]['architectures'][architecture] << host
-              if target[dc]['hosts'].include?(host)
-                target[dc]['hosts'].delete(host)
-                target[dc]['hosts'] << host
-              end
-              return host
-            else
-              raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('hosts')
-              host = target[dc]['hosts'].shift
-              target[dc]['hosts'] << host
-              target[dc]['architectures'].each do |arch|
-                if arch.include?(host)
-                  target[dc]['architectures'][arch] = arch.partition { |v| v != host }.flatten
-                end
-              end
-              return host
-            end
-          end
-        end
-
-        def vm_in_target?(pool_name, parent_host, architecture, target)
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-          dc = "#{datacenter}_#{cluster}"
-          raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('hosts')
-          return true if target[dc]['hosts'].include?(parent_host)
-          return true if target[dc]['architectures'][architecture].include?(parent_host)
-          return false
-        end
-
-        def get_vm(pool_name, vm_name)
-          vm_hash = nil
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            return vm_hash if vm_object.nil?
-
-            vm_hash = generate_vm_hash(vm_object, pool_name)
-          end
-          vm_hash
-        end
-
-        def create_vm(pool_name, new_vmname)
-          pool = pool_config(pool_name)
-          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
-          vm_hash = nil
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            # Assume all pool config is valid i.e. not missing
-            template_path = pool['template']
-            target_folder_path = pool['folder']
-            target_datastore = pool['datastore']
-            target_cluster_name = get_target_cluster_from_config(pool_name)
-            target_datacenter_name = get_target_datacenter_from_config(pool_name)
-
-            # Get the template VM object
-            raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless valid_template_path? template_path
-
-            template_vm_object = find_template_vm(pool, connection)
-
-            # Annotate with creation time, origin template, etc.
-            # Add extraconfig options that can be queried by vmtools
-            config_spec = RbVmomi::VIM.VirtualMachineConfigSpec(
-              annotation: JSON.pretty_generate(
-                name: new_vmname,
-                created_by: provider_config['username'],
-                base_template: template_path,
-                creation_timestamp: Time.now.utc
-              ),
-              extraConfig: [
-                { key: 'guestinfo.hostname', value: new_vmname }
-              ]
-            )
-
-            # Put the VM in the specified folder and resource pool
-            relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-              datastore: find_datastore(target_datastore, connection, target_datacenter_name),
-              diskMoveType: get_disk_backing(pool)
-            )
-
-            manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
-            if manage_host_selection
-              run_select_hosts(pool_name, @provider_hosts)
-              target_host = select_next_host(pool_name, @provider_hosts)
-              host_object = find_host_by_dnsname(connection, target_host)
-              relocate_spec.host = host_object
-            else
-            # Choose a cluster/host to place the new VM on
-              target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
-              relocate_spec.pool = target_cluster_object.resourcePool
-            end
-
-            # Create a clone spec
-            clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-              location: relocate_spec,
-              config: config_spec,
-              powerOn: true,
-              template: false
-            )
-
-            begin
-              vm_target_folder = find_vm_folder(pool_name, connection)
-              if vm_target_folder.nil? and @config[:config].key?('create_folders') and @config[:config]['create_folders'] == true
-                vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
-              end
-            rescue => _err
-              if @config[:config].key?('create_folders') and @config[:config]['create_folders'] == true
-                vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
-              else
-                raise(_err)
-              end
-            end
-
-            # Create the new VM
-            new_vm_object = template_vm_object.CloneVM_Task(
-              folder: vm_target_folder,
-              name: new_vmname,
-              spec: clone_spec
-            ).wait_for_completion
-
-            vm_hash = generate_vm_hash(new_vm_object, pool_name)
-          end
-          vm_hash
-        end
-
-        def create_disk(pool_name, vm_name, disk_size)
-          pool = pool_config(pool_name)
-          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
-
-          datastore_name = pool['datastore']
-          raise("Pool #{pool_name} does not have a datastore defined for the provider #{name}") if datastore_name.nil?
-
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            add_disk(vm_object, disk_size, datastore_name, connection, get_target_datacenter_from_config(pool_name))
-          end
-          true
-        end
-
-        def create_snapshot(pool_name, vm_name, new_snapshot_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            old_snap = find_snapshot(vm_object, new_snapshot_name)
-            raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil?
-
-            vm_object.CreateSnapshot_Task(
-              name: new_snapshot_name,
-              description: 'vmpooler',
-              memory: true,
-              quiesce: true
-            ).wait_for_completion
-          end
-          true
-        end
-
-        def revert_snapshot(pool_name, vm_name, snapshot_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            snapshot_object = find_snapshot(vm_object, snapshot_name)
-            raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil?
-
-            snapshot_object.RevertToSnapshot_Task.wait_for_completion
-          end
-          true
-        end
-
-        def destroy_vm(pool_name, vm_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            # If a VM doesn't exist then it is effectively deleted
-            return true if vm_object.nil?
-
-            # Poweroff the VM if it's running
-            vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
-
-            # Kill it with fire
-            vm_object.Destroy_Task.wait_for_completion
-          end
-          true
-        end
-
-        def vm_ready?(_pool_name, vm_name)
-          begin
-            open_socket(vm_name, global_config[:config]['domain'])
-          rescue => _err
-            return false
-          end
-
-          true
-        end
-
-        # VSphere Helper methods
-
-        def get_target_cluster_from_config(pool_name)
-          pool = pool_config(pool_name)
-          return nil if pool.nil?
-
-          return pool['clone_target'] unless pool['clone_target'].nil?
-          return global_config[:config]['clone_target'] unless global_config[:config]['clone_target'].nil?
-
-          nil
-        end
-
-        def get_target_datacenter_from_config(pool_name)
-          pool = pool_config(pool_name)
-          return nil if pool.nil?
-
-          return pool['datacenter']            unless pool['datacenter'].nil?
-          return provider_config['datacenter'] unless provider_config['datacenter'].nil?
-
-          nil
-        end
-
-        # Return a hash of VM data
-        # Provides vmname, hostname, template, poolname, boottime and powerstate information
-        def generate_vm_hash(vm_object, pool_name)
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-
-          hostname = vm_object.summary.guest.hostName if vm_object.summary && vm_object.summary.guest && vm_object.summary.guest.hostName
-          boottime = vm_object.runtime.bootTime if vm_object.runtime && vm_object.runtime.bootTime
-          powerstate = vm_object.runtime.powerState if vm_object.runtime && vm_object.runtime.powerState
-
-          hash = {
-            'name' => vm_object.name,
-            'hostname' => hostname,
-            'template' => pool_configuration['template'],
-            'poolname' => pool_name,
-            'boottime' => boottime,
-            'powerstate' => powerstate,
-          }
-
-          hash
-        end
-
-        # vSphere helper methods
-        ADAPTER_TYPE = 'lsiLogic'.freeze
-        DISK_TYPE = 'thin'.freeze
-        DISK_MODE = 'persistent'.freeze
-
-        def ensured_vsphere_connection(connection_pool_object)
-          connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection])
-          connection_pool_object[:connection]
-        end
-
-        def vsphere_connection_ok?(connection)
-          _result = connection.serviceInstance.CurrentTime
-          return true
-        rescue
-          return false
-        end
-
-        def connect_to_vsphere
-          max_tries = global_config[:config]['max_tries'] || 3
-          retry_factor = global_config[:config]['retry_factor'] || 10
-          try = 1
-          begin
-            connection = RbVmomi::VIM.connect host: provider_config['server'],
-                                              user: provider_config['username'],
-                                              password: provider_config['password'],
-                                              insecure: provider_config['insecure'] || false
-            metrics.increment('connect.open')
-            return connection
-          rescue => err
-            metrics.increment('connect.fail')
-            raise err if try >= max_tries
-            sleep(try * retry_factor)
-            try += 1
-            retry
-          end
-        end
-
-        # This should supercede the open_socket method in the Pool Manager
-        def open_socket(host, domain = nil, timeout = 5, port = 22, &_block)
-          Timeout.timeout(timeout) do
-            target_host = host
-            target_host = "#{host}.#{domain}" if domain
-            sock = TCPSocket.new target_host, port
-            begin
-              yield sock if block_given?
-            ensure
-              sock.close
-            end
-          end
-        end
-
-        def get_vm_folder_path(vm_object)
-          # This gives an array starting from the root Datacenters folder all the way to the VM
-          # [ [Object, String], [Object, String ] ... ]
-          # It's then reversed so that it now goes from the VM to the Datacenter
-          full_path = vm_object.path.reverse
-
-          # Find the Datacenter object
-          dc_index = full_path.index { |p| p[0].is_a?(RbVmomi::VIM::Datacenter) }
-          return nil if dc_index.nil?
-          # The Datacenter should be at least 2 otherwise there's something
-          # wrong with the array passed in
-          # This is the minimum:
-          # [ VM (0), VM ROOT FOLDER (1), DC (2)]
-          return nil if dc_index <= 1
-
-          # Remove the VM name (Starting position of 1 in the slice)
-          # Up until the Root VM Folder of DataCenter Node (dc_index - 2)
-          full_path = full_path.slice(1..dc_index - 2)
-
-          # Reverse the array back to normal and
-          # then convert the array of paths into a '/' seperated string
-          (full_path.reverse.map { |p| p[1] }).join('/')
-        end
-
-        def add_disk(vm, size, datastore, connection, datacentername)
-          return false unless size.to_i > 0
-
-          vmdk_datastore = find_datastore(datastore, connection, datacentername)
-          raise("Datastore '#{datastore}' does not exist in datacenter '#{datacentername}'") if vmdk_datastore.nil?
-
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          controller = find_disk_controller(vm)
-          disk_unit_number = find_disk_unit_number(vm, controller)
-          disk_count = vm.config.hardware.device.grep(RbVmomi::VIM::VirtualDisk).count
-          vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{disk_count}.vmdk"
-
-          vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec(
-            capacityKb: size.to_i * 1024 * 1024,
-            adapterType: ADAPTER_TYPE,
-            diskType: DISK_TYPE
-          )
-
-          vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo(
-            datastore: vmdk_datastore,
-            diskMode: DISK_MODE,
-            fileName: "[#{datastore}] #{vmdk_file_name}"
-          )
-
-          device = RbVmomi::VIM::VirtualDisk(
-            backing: vmdk_backing,
-            capacityInKB: size.to_i * 1024 * 1024,
-            controllerKey: controller.key,
-            key: -1,
-            unitNumber: disk_unit_number
-          )
-
-          device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec(
-            device: device,
-            operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add')
-          )
-
-          vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec(
-            deviceChange: [device_config_spec]
-          )
-
-          connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
-            datacenter: datacenter,
-            name: "[#{datastore}] #{vmdk_file_name}",
-            spec: vmdk_spec
-          ).wait_for_completion
-
-          vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion
-
-          true
-        end
-
-        def find_datastore(datastorename, connection, datacentername)
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
-          datacenter.find_datastore(datastorename)
-        end
-
-        def find_device(vm, device_name)
-          vm.config.hardware.device.each do |device|
-            return device if device.deviceInfo.label == device_name
-          end
-
-          nil
-        end
-
-        def find_disk_controller(vm)
-          devices = find_disk_devices(vm)
-
-          devices.keys.sort.each do |device|
-            if devices[device]['children'].length < 15
-              return find_device(vm, devices[device]['device'].deviceInfo.label)
-            end
-          end
-
-          nil
-        end
-
-        def find_disk_devices(vm)
-          devices = {}
-
-          vm.config.hardware.device.each do |device|
-            if device.is_a? RbVmomi::VIM::VirtualSCSIController
-              if devices[device.controllerKey].nil?
-                devices[device.key] = {}
-                devices[device.key]['children'] = []
-              end
-
-              devices[device.key]['device'] = device
-            end
-
-            if device.is_a? RbVmomi::VIM::VirtualDisk
-              if devices[device.controllerKey].nil?
-                devices[device.controllerKey] = {}
-                devices[device.controllerKey]['children'] = []
-              end
-
-              devices[device.controllerKey]['children'].push(device)
-            end
-          end
-
-          devices
-        end
-
-        def find_disk_unit_number(vm, controller)
-          used_unit_numbers = []
-          available_unit_numbers = []
-
-          devices = find_disk_devices(vm)
-
-          devices.keys.sort.each do |c|
-            next unless controller.key == devices[c]['device'].key
-            used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber)
-            devices[c]['children'].each do |disk|
-              used_unit_numbers.push(disk.unitNumber)
-            end
-          end
-
-          (0..15).each do |scsi_id|
-            if used_unit_numbers.grep(scsi_id).length <= 0
-              available_unit_numbers.push(scsi_id)
-            end
-          end
-
-          available_unit_numbers.sort[0]
-        end
-
-        # Finds a folder object by inventory path
-        # Params:
-        # +pool_name+:: the pool to find the folder for
-        # +connection+:: the vsphere connection object
-        # returns a ManagedObjectReference for the folder found or nil if not found
-        def find_vm_folder(pool_name, connection)
-          # Find a folder by its inventory path and return the object
-          # Returns nil when the object found is not a folder
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-          folder = pool_configuration['folder']
-          datacenter = get_target_datacenter_from_config(pool_name)
-          return nil if datacenter.nil?
-
-          propSpecs = {
-            :entity => self,
-            :inventoryPath => "#{datacenter}/vm/#{folder}"
-          }
-
-          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
-          return nil unless folder_object.class == RbVmomi::VIM::Folder
-          folder_object
-        end
-
-        # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference
-        # Params:
-        # +model+:: CPU arch version to match on
-        # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
-        # returns nil if one on these conditions is true:
-        #    the model param is defined and cannot be found
-        #    the host is in maintenance mode
-        #    the host status is not 'green'
-        #    the cpu or memory utilization is bigger than the limit param
-        def get_host_utilization(host, model = nil, limit = 90)
-          limit = @config[:config]['utilization_limit'] if @config[:config].key?('utilization_limit')
-          if model
-            return nil unless host_has_cpu_model?(host, model)
-          end
-          return nil if host.runtime.inMaintenanceMode
-          return nil unless host.overallStatus == 'green'
-          return nil unless host.configIssue.empty?
-
-          cpu_utilization = cpu_utilization_for host
-          memory_utilization = memory_utilization_for host
-
-          return nil if cpu_utilization.nil?
-          return nil if cpu_utilization == 0.0
-          return nil if memory_utilization.nil?
-          return nil if memory_utilization == 0.0
-
-          return nil if cpu_utilization > limit
-          return nil if memory_utilization > limit
-
-          [cpu_utilization, host]
-        end
-
-        def host_has_cpu_model?(host, model)
-          get_host_cpu_arch_version(host) == model
-        end
-
-        def get_host_cpu_arch_version(host)
-          cpu_model = host.hardware.cpuPkg[0].description
-          cpu_model_parts = cpu_model.split
-          arch_version = cpu_model_parts[4]
-          arch_version
-        end
-
-        def cpu_utilization_for(host)
-          cpu_usage = host.summary.quickStats.overallCpuUsage
-          return nil if cpu_usage.nil?
-          cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores
-          (cpu_usage.to_f / cpu_size.to_f) * 100
-        end
-
-        def memory_utilization_for(host)
-          memory_usage = host.summary.quickStats.overallMemoryUsage
-          return nil if memory_usage.nil?
-          memory_size = host.summary.hardware.memorySize / 1024 / 1024
-          (memory_usage.to_f / memory_size.to_f) * 100
-        end
-
-        def get_average_cluster_utilization(hosts)
-          utilization_counts = hosts.map { |host| host[0] }
-          utilization_counts.inject(:+) / hosts.count
-        end
-
-        def build_compatible_hosts_lists(hosts, percentage)
-          hosts_with_arch_versions = hosts.map { |h|
-            {
-            'utilization' => h[0],
-            'host_object' => h[1],
-            'architecture' => get_host_cpu_arch_version(h[1])
-            }
-          }
-          versions = hosts_with_arch_versions.map { |host| host['architecture'] }.uniq
-          architectures = {}
-          versions.each do |version|
-            architectures[version] = []
-          end
-
-          hosts_with_arch_versions.each do |h|
-            architectures[h['architecture']] << [h['utilization'], h['host_object'], h['architecture']]
-          end
-
-          versions.each do |version|
-            targets = []
-            targets = select_least_used_hosts(architectures[version], percentage)
-            architectures[version] = targets
-          end
-          architectures
-        end
-
-        def select_least_used_hosts(hosts, percentage)
-          raise('Provided hosts list to select_least_used_hosts is empty') if hosts.empty?
-          average_utilization = get_average_cluster_utilization(hosts)
-          least_used_hosts = []
-          hosts.each do |host|
-            least_used_hosts << host if host[0] <= average_utilization
-          end
-          hosts_to_select = (hosts.count * (percentage  / 100.0)).to_int
-          hosts_to_select = hosts.count - 1 if percentage == 100
-          least_used_hosts.sort[0..hosts_to_select].map { |host| host[1].name }
-        end
-
-        def find_least_used_hosts(cluster, datacentername, percentage)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            cluster_object = find_cluster(cluster, connection, datacentername)
-            raise("Cluster #{cluster} cannot be found") if cluster_object.nil?
-            target_hosts = get_cluster_host_utilization(cluster_object)
-            raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-            architectures = build_compatible_hosts_lists(target_hosts, percentage)
-            least_used_hosts = select_least_used_hosts(target_hosts, percentage)
-            {
-              'hosts' => least_used_hosts,
-              'architectures' => architectures
-            }
-          end
-        end
-
-        def find_host_by_dnsname(connection, dnsname)
-          host_object = connection.searchIndex.FindByDnsName(dnsName: dnsname, vmSearch: false)
-          return nil if host_object.nil?
-          host_object
-        end
-
-        def find_least_used_host(cluster, connection, datacentername)
-          cluster_object = find_cluster(cluster, connection, datacentername)
-          target_hosts = get_cluster_host_utilization(cluster_object)
-          raise("There is no host candidate in vcenter that meets all the required conditions, check that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-          least_used_host = target_hosts.sort[0][1]
-          least_used_host
-        end
-
-        def find_cluster(cluster, connection, datacentername)
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
-          datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster }
-        end
-
-        def get_cluster_host_utilization(cluster, model = nil)
-          cluster_hosts = []
-          cluster.host.each do |host|
-            host_usage = get_host_utilization(host, model)
-            cluster_hosts << host_usage if host_usage
-          end
-          cluster_hosts
-        end
-
-        def find_least_used_vpshere_compatible_host(vm)
-          source_host = vm.summary.runtime.host
-          model = get_host_cpu_arch_version(source_host)
-          cluster = source_host.parent
-          target_hosts = get_cluster_host_utilization(cluster, model)
-          raise("There is no host candidate in vcenter that meets all the required conditions, check that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-          target_host = target_hosts.sort[0][1]
-          [target_host, target_host.name]
-        end
-
-        def find_snapshot(vm, snapshotname)
-          get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) if vm.snapshot
-        end
-
-        def build_propSpecs(datacenter, folder, vmname)
-          propSpecs = {
-            entity => self,
-            :inventoryPath => "#{datacenter}/vm/#{folder}/#{vmname}"
-          }
-          propSpecs
-        end
-
-        def find_vm(pool_name, vmname, connection)
-          # Find a VM by its inventory path and return the VM object
-          # Returns nil when a VM, or pool configuration, cannot be found
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-          folder = pool_configuration['folder']
-          datacenter = get_target_datacenter_from_config(pool_name)
-          return nil if datacenter.nil?
-
-          propSpecs = {
-            :entity => self,
-            :inventoryPath => "#{datacenter}/vm/#{folder}/#{vmname}"
-          }
-
-          connection.searchIndex.FindByInventoryPath(propSpecs)
-        end
-
-        def get_base_vm_container_from(connection)
-          view_manager = connection.serviceContent.viewManager
-          view_manager.CreateContainerView(
-            container: connection.serviceContent.rootFolder,
-            recursive: true,
-            type: ['VirtualMachine']
-          )
-        end
-
-        def get_snapshot_list(tree, snapshotname)
-          snapshot = nil
-
-          tree.each do |child|
-            if child.name == snapshotname
-              snapshot ||= child.snapshot
-            else
-              snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname)
-            end
-          end
-
-          snapshot
-        end
-
-        def get_vm_details(pool_name, vm_name, connection)
-          vm_object = find_vm(pool_name, vm_name, connection)
-          return nil if vm_object.nil?
-          parent_host_object = vm_object.summary.runtime.host if vm_object.summary && vm_object.summary.runtime && vm_object.summary.runtime.host
-          raise('Unable to determine which host the VM is running on') if parent_host_object.nil?
-          parent_host = parent_host_object.name
-          architecture = get_host_cpu_arch_version(parent_host_object)
-          {
-            'host_name' => parent_host,
-            'object' => vm_object,
-            'architecture' => architecture
-          }
-        end
-
-        def migration_enabled?(config)
-          migration_limit = config[:config]['migration_limit']
-          return false unless migration_limit.is_a? Integer
-          return true if migration_limit > 0
-          false
-        end
-
-        def migrate_vm(pool_name, vm_name)
-          @connection_pool.with_metrics do |pool_object|
-            begin
-              connection = ensured_vsphere_connection(pool_object)
-              vm_hash = get_vm_details(pool_name, vm_name, connection)
-              $redis.hset("vmpooler__vm__#{vm_name}", 'host', vm_hash['host_name'])
-              migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
-              migration_count = $redis.scard('vmpooler__migration')
-              if migration_enabled? @config
-                if migration_count >= migration_limit
-                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
-                  return
-                end
-                run_select_hosts(pool_name, @provider_hosts)
-                if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
-                  logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
-                else
-                  migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-                end
-              else
-                logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
-              end
-            rescue => _err
-              logger.log('s', "[!] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
-              raise _err
-            end
-          end
-        end
-
-        def migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-          $redis.sadd('vmpooler__migration', vm_name)
-          target_host_name = select_next_host(pool_name, @provider_hosts, vm_hash['architecture'])
-          $redis.hset("vmpooler__vm__#{vm_name}", 'host', target_host_name)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'migrated', true)
-          target_host_object = find_host_by_dnsname(connection, target_host_name)
-          finish = migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, target_host_name)
-          logger.log('s', "[>] [#{pool_name}] '#{vm_name}' migrated from #{vm_hash['host_name']} to #{target_host_name} in #{finish} seconds")
-        ensure
-          $redis.srem('vmpooler__migration', vm_name)
-        end
-
-        def migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, dest_host_name)
-          start = Time.now
-          migrate_vm_host(vm_hash['object'], target_host_object)
-          finish = format('%.2f', Time.now - start)
-          metrics.timing("migrate.#{pool_name}", finish)
-          metrics.increment("migrate_from.#{vm_hash['host_name']}")
-          metrics.increment("migrate_to.#{dest_host_name}")
-          checkout_to_migration = format('%.2f', Time.now - Time.parse($redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
-          $redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration)
-          finish
-        end
-
-        def migrate_vm_host(vm_object, host)
-          relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host)
-          vm_object.RelocateVM_Task(spec: relospec).wait_for_completion
-        end
-
-        def create_folder(connection, new_folder, datacenter)
-          dc = connection.serviceInstance.find_datacenter(datacenter)
-          folder_object = dc.vmFolder.traverse(new_folder, type=RbVmomi::VIM::Folder, create=true)
-          raise("Cannot create folder #{new_folder}") if folder_object.nil?
-          folder_object
-        end
-
-        def find_template_vm(pool, connection)
-          datacenter = get_target_datacenter_from_config(pool['name'])
-          raise('cannot find datacenter') if datacenter.nil?
-
-          propSpecs = {
-            :entity => self,
-            :inventoryPath => "#{datacenter}/vm/#{pool['template']}"
-          }
-
-          template_vm_object = connection.searchIndex.FindByInventoryPath(propSpecs)
-          raise("Pool #{pool['name']} specifies a template VM of #{pool['template']} which does not exist for the provider #{name}") if template_vm_object.nil?
-
-          template_vm_object
-        end
-
-        def create_template_delta_disks(pool)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            template_vm_object = find_template_vm(pool, connection)
-
-            template_vm_object.add_delta_disk_layer_on_all_disks
-          end
-        end
-
-        def valid_template_path?(template)
-          return false unless template.include?('/')
-          return false if template[0] == '/'
-          return false if template[-1] == '/'
-          return true
-        end
-
-        def get_disk_backing(pool)
-          return :moveChildMostDiskBacking if linked_clone?(pool)
-          return :moveAllDiskBackingsAndConsolidate
-        end
-
-        def linked_clone?(pool)
-          return if pool['create_linked_clone'] == false
-          return true if pool['create_linked_clone']
-          return true if @config[:config]['create_linked_clones']
-        end
-      end
-    end
-  end
-end
diff --git a/lib/vmpooler/public/img/logo.png b/lib/vmpooler/public/img/logo.png
new file mode 100644
index 0000000..0cefaba
Binary files /dev/null and b/lib/vmpooler/public/img/logo.png differ
diff --git a/lib/vmpooler/public/lib/dashboard.js b/lib/vmpooler/public/lib/dashboard.js
index e726b01..bd12074 100644
--- a/lib/vmpooler/public/lib/dashboard.js
+++ b/lib/vmpooler/public/lib/dashboard.js
@@ -29,10 +29,10 @@ Date.prototype.yyyymmdd = function() {
 
 var data_url = {
   'capacity': '/dashboard/stats/vmpooler/pool',
-  'pools'   : '/api/v1/vm',
+  'pools'   : '/api/v3/vm',
   'running' : '/dashboard/stats/vmpooler/running',
-  'status'  : '/api/v1/status',
-  'summary' : '/api/v1/summary'
+  'status'  : '/api/v3/status',
+  'summary' : '/api/v3/summary'
 };
 
 
diff --git a/lib/vmpooler/statsd.rb b/lib/vmpooler/statsd.rb
deleted file mode 100644
index 141649f..0000000
--- a/lib/vmpooler/statsd.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-require 'statsd'
-
-module Vmpooler
-  class Statsd
-    attr_reader :server, :port, :prefix
-
-    def initialize(params = {})
-      if params['server'].nil? || params['server'].empty?
-        raise ArgumentError, "Statsd server is required. Config: #{params.inspect}"
-      end
-
-      host    = params['server']
-      @port   = params['port'] || 8125
-      @prefix = params['prefix'] || 'vmpooler'
-      @server = ::Statsd.new(host, @port)
-    end
-
-    def increment(label)
-      server.increment(prefix + '.' + label)
-    rescue => err
-      $stderr.puts "Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-
-    def gauge(label, value)
-      server.gauge(prefix + '.' + label, value)
-    rescue => err
-      $stderr.puts "Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-
-    def timing(label, duration)
-      server.timing(prefix + '.' + label, duration)
-    rescue => err
-      $stderr.puts "Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
-    end
-  end
-end
diff --git a/lib/vmpooler/util/parsing.rb b/lib/vmpooler/util/parsing.rb
new file mode 100644
index 0000000..1ca6b13
--- /dev/null
+++ b/lib/vmpooler/util/parsing.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# utility class shared between apps api and pool_manager
+module Vmpooler
+  class Parsing
+    def self.get_platform_pool_count(requested, &_block)
+      requested_platforms = requested.split(',')
+      requested_platforms.each do |platform|
+        platform_alias, pool, count = platform.split(':')
+        raise ArgumentError if platform_alias.nil? || pool.nil? || count.nil?
+
+        yield platform_alias, pool, count
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/version.rb b/lib/vmpooler/version.rb
index fc884b9..4469c2a 100644
--- a/lib/vmpooler/version.rb
+++ b/lib/vmpooler/version.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
-  VERSION = '0.7.1'.freeze
+  VERSION = '3.8.1'
 end
diff --git a/release-notes.md b/release-notes.md
new file mode 100644
index 0000000..7e9892e
--- /dev/null
+++ b/release-notes.md
@@ -0,0 +1,15 @@
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
+
+**Implemented enhancements:**
+
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
+
+**Fixed bugs:**
+
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
diff --git a/release-prep b/release-prep
new file mode 100755
index 0000000..a0082de
--- /dev/null
+++ b/release-prep
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+#
+# Update Gemfile.lock
+docker run -t --rm \
+  -v $(pwd):/app \
+  jruby:9.4.12.1-jdk11 \
+  /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase && cd /app && gem install bundler && bundle install --jobs 3; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"'
+
+# Update Changelog
+docker run -t --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \
+  githubchangeloggenerator/github-changelog-generator:1.16.4 \
+  github_changelog_generator --future-release $(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev) \
+  --token $CHANGELOG_GITHUB_TOKEN --release-branch main
+
diff --git a/scripts/create_template_deltas.rb b/scripts/create_template_deltas.rb
deleted file mode 100755
index f03ce39..0000000
--- a/scripts/create_template_deltas.rb
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/ruby
-
-require 'rubygems'
-require 'rbvmomi'
-require 'yaml'
-
-def load_configuration( file_array )
-  file_array.each do |file|
-    file = File.expand_path( file )
-
-    if File.exists?( file )
-      return YAML.load_file( file )
-    end
-  end
-
-  return false
-end
-
-def create_template_deltas( folder )
-  config = load_configuration( [ 'vmpooler.yaml', '~/.vmpooler' ] ) || nil
-
-  abort 'No config file (./vmpooler.yaml or ~/.vmpooler) found!' unless config
-
-  vim = RbVmomi::VIM.connect(
-    :host     => config[ :providers ][ :vsphere ][ "server" ],
-    :user     => config[ :providers ][ :vsphere ][ "username" ],
-    :password => config[ :providers ][ :vsphere ][ "password" ],
-    :ssl      => true,
-    :insecure => true,
-  ) or abort "Unable to connect to #{config[ :vsphere ][ "server" ]}!"
-
-  containerView = vim.serviceContent.viewManager.CreateContainerView( {
-    :container => vim.serviceContent.rootFolder,
-    :recursive => true,
-    :type      => [ 'VirtualMachine' ]
-  } )
-
-  datacenter = vim.serviceInstance.find_datacenter
-  base = datacenter.vmFolder
-
-  case base
-    when RbVmomi::VIM::Folder
-      base = base.childEntity.find { |f| f.name == folder }
-    else
-      abort "Unexpected object type encountered (#{base.class}) while finding folder!"
-  end
-
-  unless base
-    abort "Folder #{ARGV[0]} not found!"
-  end
-
-  base.childEntity.each do |vm|
-    print vm.name
-
-    begin
-      disks = vm.config.hardware.device.grep( RbVmomi::VIM::VirtualDisk )
-    rescue
-      puts ' !'
-      next
-    end
-
-    begin
-      disks.select { |d| d.backing.parent == nil }.each do |disk|
-        linkSpec = {
-          :deviceChange => [
-            {
-              :operation => :remove,
-              :device => disk
-            },
-            {
-              :operation => :add,
-              :fileOperation => :create,
-              :device => disk.dup.tap { |x|
-                x.backing = x.backing.dup
-                x.backing.fileName = "[#{disk.backing.datastore.name}]"
-                x.backing.parent = disk.backing
-              }
-            }
-          ]
-        }
-
-        vm.ReconfigVM_Task( :spec => linkSpec ).wait_for_completion
-      end
-
-      puts " \u2713"
-    rescue
-      puts ' !'
-    end
-  end
-
-  vim.close
-end
-
-if ARGV[0]
-  create_template_deltas( ARGV[0] )
-else
-  puts "Usage: #{$0} <folder>"
-end
diff --git a/spec/fixtures/extra_config1.yaml b/spec/fixtures/extra_config1.yaml
new file mode 100644
index 0000000..d19523c
--- /dev/null
+++ b/spec/fixtures/extra_config1.yaml
@@ -0,0 +1,4 @@
+---
+:providers:
+  :alice:
+    foo: "foo"
\ No newline at end of file
diff --git a/spec/fixtures/extra_config2.yaml b/spec/fixtures/extra_config2.yaml
new file mode 100644
index 0000000..af122c2
--- /dev/null
+++ b/spec/fixtures/extra_config2.yaml
@@ -0,0 +1,12 @@
+---
+:providers:
+  :bob:
+    foo: "foo_bob"
+    bar: "bar"
+
+:pools:
+  - name: 'pool05'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
\ No newline at end of file
diff --git a/spec/fixtures/vmpooler.yaml b/spec/fixtures/vmpooler.yaml
index 1e3e78f..c7dad1b 100644
--- a/spec/fixtures/vmpooler.yaml
+++ b/spec/fixtures/vmpooler.yaml
@@ -23,9 +23,13 @@
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,6 +40,10 @@
   - name: 'pool01'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
   - name: 'pool02'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
diff --git a/spec/fixtures/vmpooler2.yaml b/spec/fixtures/vmpooler2.yaml
index 39a617a..92731a5 100644
--- a/spec/fixtures/vmpooler2.yaml
+++ b/spec/fixtures/vmpooler2.yaml
@@ -23,9 +23,13 @@
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,6 +40,10 @@
   - name: 'pool03'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
   - name: 'pool04'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
diff --git a/spec/fixtures/vmpooler_domain.yaml b/spec/fixtures/vmpooler_domain.yaml
new file mode 100644
index 0000000..9a79726
--- /dev/null
+++ b/spec/fixtures/vmpooler_domain.yaml
@@ -0,0 +1,45 @@
+---
+:providers:
+  :dummy:
+
+:redis:
+  server: 'localhost'
+
+:auth:
+   provider: dummy
+
+:tagfilter:
+  url: '(.*)\/'
+
+:config:
+  site_name: 'vmpooler'
+  # Need to change this on Windows
+  logfile: '/var/log/vmpooler.log'
+  task_limit: 10
+  timeout: 15
+  vm_checktime: 1
+  vm_lifetime: 12
+  vm_lifetime_auth: 24
+  allowed_tags:
+    - 'created_by'
+    - 'project'
+  domain: 'example.com'
+  prefix: 'poolvm-'
+
+# Uncomment the lines below to suppress metrics to STDOUT
+# :statsd:
+#   server: 'localhost'
+#   prefix: 'vmpooler'
+#   port: 8125
+
+:pools:
+  - name: 'pool01'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
+  - name: 'pool02'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
diff --git a/spec/helpers.rb b/spec/helpers.rb
index bd3c748..53af74c 100644
--- a/spec/helpers.rb
+++ b/spec/helpers.rb
@@ -14,6 +14,16 @@ class MockLogger
   end
 end
 
+class MockPoolManagerDnsBase
+  def delete_record(hostname)
+
+  end
+
+  def create_or_replace_record(hostname)
+
+  end
+end
+
 def expect_json(ok = true, http = 200)
   expect(last_response.header['Content-Type']).to eq('application/json')
 
@@ -40,93 +50,115 @@ def token_exists?(token)
   result && !result.empty?
 end
 
-def create_ready_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_ready_vm(template, name, redis, token = nil)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__ready__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_running_vm(template, name, token = nil, user = nil)
-  create_vm(name, token, nil, user)
+def create_running_vm(template, name, redis, token = nil, user = nil)
+  create_vm(name, redis, token, user)
   redis.sadd("vmpooler__running__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", 'template', template)
   redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
   redis.hset("vmpooler__vm__#{name}", 'host', 'host1')
 end
 
-def create_pending_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_pending_vm(template, name, redis, token = nil)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__pending__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_vm(name, token = nil, redis_handle = nil, user = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'token:token', token) if token
-  redis_db.hset("vmpooler__vm__#{name}", 'token:user', user) if user
+def create_vm(name, redis, token = nil, user = nil)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'clone', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'token:token', token) if token
+  redis.hset("vmpooler__vm__#{name}", 'token:user', user) if user
 end
 
-def create_completed_vm(name, pool, active = false, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd("vmpooler__completed__#{pool}", name)
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__active__#{pool}", name, Time.now) if active
+def create_completed_vm(name, pool, redis, active = false)
+  redis.sadd("vmpooler__completed__#{pool}", name)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__active__#{pool}", name, Time.now) if active
 end
 
-def create_discovered_vm(name, pool, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd("vmpooler__discovered__#{pool}", name)
+def create_discovered_vm(name, pool, redis)
+  redis.sadd("vmpooler__discovered__#{pool}", name)
 end
 
-def create_migrating_vm(name, pool, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.sadd("vmpooler__migrating__#{pool}", name)
+def create_migrating_vm(name, pool, redis)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.sadd("vmpooler__migrating__#{pool}", name)
 end
 
-def create_tag(vm, tag_name, tag_value, redis_handle = nil)
-  redis_db = redis_handle ? redis-handle : redis
-  redis_db.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
+def create_tag(vm, tag_name, tag_value, redis)
+  redis.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
 end
 
-def add_vm_to_migration_set(name, redis_handle = nil)
-  redis_db = redis_handle ? redis_handle : redis
-  redis_db.sadd('vmpooler__migration', name)
+def add_vm_to_migration_set(name, redis)
+  redis.sadd('vmpooler__migration', name)
 end
 
 def fetch_vm(vm)
   redis.hgetall("vmpooler__vm__#{vm}")
 end
 
-def snapshot_revert_vm(vm, snapshot = '12345678901234567890123456789012')
+def set_vm_data(vm, key, value, redis)
+  redis.hset("vmpooler__vm__#{vm}", key, value)
+end
+
+def snapshot_revert_vm(vm, snapshot = '12345678901234567890123456789012', redis)
   redis.sadd('vmpooler__tasks__snapshot-revert', "#{vm}:#{snapshot}")
   redis.hset("vmpooler__vm__#{vm}", "snapshot:#{snapshot}", "1")
 end
 
-def snapshot_vm(vm, snapshot = '12345678901234567890123456789012')
+def snapshot_vm(vm, snapshot = '12345678901234567890123456789012', redis)
   redis.sadd('vmpooler__tasks__snapshot', "#{vm}:#{snapshot}")
   redis.hset("vmpooler__vm__#{vm}", "snapshot:#{snapshot}", "1")
 end
 
-def disk_task_vm(vm, disk_size = '10')
+def disk_task_vm(vm, disk_size = '10', redis)
   redis.sadd('vmpooler__tasks__disk', "#{vm}:#{disk_size}")
 end
 
-def has_vm_snapshot?(vm)
+def has_vm_snapshot?(vm, redis)
   redis.smembers('vmpooler__tasks__snapshot').any? do |snapshot|
-    instance, sha = snapshot.split(':')
+    instance, _sha = snapshot.split(':')
     vm == instance
   end
 end
 
-def vm_reverted_to_snapshot?(vm, snapshot = nil)
+def vm_reverted_to_snapshot?(vm, redis, snapshot = nil)
   redis.smembers('vmpooler__tasks__snapshot-revert').any? do |action|
     instance, sha = action.split(':')
     instance == vm and (snapshot ? (sha == snapshot) : true)
   end
 end
 
-def pool_has_ready_vm?(pool, vm)
+def pool_has_ready_vm?(pool, vm, redis)
   !!redis.sismember('vmpooler__ready__' + pool, vm)
 end
+
+def create_ondemand_request_for_test(request_id, score, platforms_string, redis, user = nil, token = nil)
+  redis.zadd('vmpooler__provisioning__request', score, request_id)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'token:token', token) if token
+  redis.hset("vmpooler__odrequest__#{request_id}", 'token:user', user) if user
+end
+
+def set_ondemand_request_status(request_id, status, redis)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'status', status)
+end
+
+def create_ondemand_vm(vmname, request_id, pool, pool_alias, redis)
+  redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vmname)
+end
+
+def create_ondemand_creationtask(request_string, score, redis)
+  redis.zadd('vmpooler__odcreate__task', score, request_string)
+end
+
+def create_ondemand_processing(request_id, score, redis)
+  redis.zadd('vmpooler__provisioning__processing', score, request_id)
+end
diff --git a/spec/integration/api/healthcheck_spec.rb b/spec/integration/api/healthcheck_spec.rb
new file mode 100644
index 0000000..ecce07e
--- /dev/null
+++ b/spec/integration/api/healthcheck_spec.rb
@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::Healthcheck do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method
+  before(:each) do
+    app.reset!
+  end
+
+  let(:config) {
+    {
+        config: {
+            'site_name' => 'test pooler',
+            'vm_lifetime_auth' => 2,
+        },
+        pools: [
+                    {'name' => 'pool1', 'size' => 5, 'alias' => ['poolone', 'poolun']},
+                    {'name' => 'pool2', 'size' => 10},
+                    {'name' => 'pool3', 'size' => 10, 'alias' => 'NotArray'}
+                ]
+    }
+  }
+
+  let(:current_time) { Time.now }
+
+  let(:metrics) {
+    double("metrics")
+  }
+
+  before(:each) do
+    expect(app).to receive(:run!).once
+    expect(metrics).to receive(:setup_prometheus_metrics)
+    expect(metrics).to receive(:prometheus_prefix)
+    expect(metrics).to receive(:prometheus_endpoint)
+    app.execute([:api], config, redis, metrics, nil)
+    app.settings.set :config, auth: false
+  end
+
+  describe '/healthcheck' do
+    it 'returns OK' do
+      get "/healthcheck"
+      expect(last_response.header['Content-Type']).to eq('application/json')
+      expect(last_response.status).to eq(200)
+      result = JSON.parse(last_response.body)
+      expect(result).to eq({'ok' => true})
+    end
+  end
+end
diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v3/config_spec.rb
similarity index 70%
rename from spec/integration/api/v1/config_spec.rb
rename to spec/integration/api/v3/config_spec.rb
index 0b73d54..301c4aa 100644
--- a/spec/integration/api/v1/config_spec.rb
+++ b/spec/integration/api/v3/config_spec.rb
@@ -1,13 +1,20 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   let(:config) {
     {
       config: {
@@ -19,6 +26,10 @@ describe Vmpooler::API::V1 do
         {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
         {'name' => 'pool2', 'size' => 10}
       ],
+      pools_at_startup: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
       statsd: { 'prefix' => 'stats_prefix'},
       alias: { 'poolone' => 'pool1' },
       pool_names: [ 'pool1', 'pool2', 'poolone' ]
@@ -26,19 +37,59 @@ describe Vmpooler::API::V1 do
   }
 
   describe '/config/pooltemplate' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:current_time) { Time.now }
 
     before(:each) do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
+    describe 'DELETE /config/pooltemplate/:pool' do
+      it 'resets a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          template_before_reset: 'templates/new_template',
+          template_before_overrides: 'templates/pool1'
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when the pool has not been overridden' do
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails on nonexistent pools' do
+        delete "#{prefix}/config/pooltemplate/poolpoolpool"
+        expect_json(ok = false, http = 404)
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/pooltemplate/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+    end
+
     describe 'POST /config/pooltemplate' do
       it 'updates a pool template' do
         post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
@@ -136,6 +187,56 @@ describe Vmpooler::API::V1 do
 
     end
 
+    describe 'DELETE /config/poolsize' do
+      it 'resets a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          pool_size_before_reset: 2,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        delete "#{prefix}/config/poolsize/pool10"
+        expect_json(ok = false, http = 404)
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when a pool has not been overridden' do
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 200)
+        expected = {
+          ok: true,
+          pool_size_before_reset: 5,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/poolsize/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
     describe 'POST /config/poolsize' do
       it 'changes a pool size' do
         post "#{prefix}/config/poolsize", '{"pool1":"2"}'
@@ -160,7 +261,7 @@ describe Vmpooler::API::V1 do
         expect_json(ok = false, http = 400)
         expected = {
           ok: false,
-          bad_templates: ['pool10']
+          not_configured: ['pool10']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -190,7 +291,7 @@ describe Vmpooler::API::V1 do
 
         expected = {
           ok: false,
-          bad_templates: ['pool1']
+          not_configured: ['pool1']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -202,7 +303,7 @@ describe Vmpooler::API::V1 do
 
         expected = {
           ok: false,
-          bad_templates: ['pool1']
+          not_configured: ['pool1']
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -287,7 +388,7 @@ describe Vmpooler::API::V1 do
     end
 
     describe 'GET /config' do
-      let(:prefix) { '/api/v1' }
+      let(:prefix) { '/api/v3' }
 
       it 'returns pool configuration when set' do
         get "#{prefix}/config"
diff --git a/spec/integration/api/v3/ondemandvm_spec.rb b/spec/integration/api/v3/ondemandvm_spec.rb
new file mode 100644
index 0000000..0a043bb
--- /dev/null
+++ b/spec/integration/api/v3/ondemandvm_spec.rb
@@ -0,0 +1,355 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V3 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/ondemandvm' do
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+          'max_ondemand_instances_per_request' => 50,
+          'backend_weight' => {
+            'compute1' => 5,
+            'compute2' => 0
+          }
+        },
+        dns_configs: {
+          :mock => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'},
+          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2', 'dns_plugin' => 'mock'},
+          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'}
+        ],
+        alias: {
+          'poolone' => ['pool1'],
+          'pool2' => ['pool1']
+        },
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ],
+        providers: {
+          :dummy => {},
+        }
+      }
+    }
+    let(:current_time) { Time.now }
+    let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
+    let(:uuid) { SecureRandom.uuid }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+      config[:pools].each do |pool|
+        redis.sadd('vmpooler__pools', pool['name'])
+      end
+    end
+
+    describe 'POST /ondemandvm' do
+
+      context 'with a configured pool' do
+
+        context 'with no request_id provided in payload' do
+          before(:each) do
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          end
+
+          it 'generates a request_id when none is provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'uses a configured platform to fulfill a ondemand request' do
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+            expect_json(true, 201)
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'creates a provisioning request in redis' do
+            expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1)
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          it 'sets a platform string in redis for the request to indicate selected platforms' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1')
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          context 'with a backend of 0 weight' do
+            before(:each) do
+              config[:config]['backend_weight']['compute1'] = 0
+            end
+
+            it 'sets the platform string in redis for the request to indicate the selected platforms' do
+              expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool1:pool1:1')
+              post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            end
+          end
+
+          it 'sets the platform string in redis for the request to indicate the selected platforms using weight' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool2:pool1:1')
+            post "#{prefix}/ondemandvm", '{"pool2":"1"}'
+          end
+        end
+
+        context 'with a resource request that exceeds the specified limit' do
+          let(:max_instances) { 50 }
+          before(:each) do
+            config[:config]['max_ondemand_instances_per_request'] = max_instances
+          end
+
+          it 'should reject the request with a message' do
+            post "#{prefix}/ondemandvm", '{"pool1":"51"}'
+            expect_json(false, 403)
+            expected = {
+              "ok": false,
+              "message": "requested amount of instances exceeds the maximum #{max_instances}"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with request_id provided in the payload' do
+          it 'uses the given request_id when provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": "1234"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'returns 409 conflict error when the request_id has been used' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(false, 409)
+
+            expected = {
+              "ok": false,
+              "request_id": "1234",
+              "message": "request_id '1234' has already been created"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with auth configured' do
+
+          it 'sets the token and user' do
+            app.settings.set :config, auth: true
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+            allow(redis).to receive(:hset)
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345')
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe')
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}', {
+              'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+            }
+          end
+        end
+      end
+
+      context 'with a pool that is not configured' do
+        let(:badpool) { 'pool4' }
+        it 'returns the bad template' do
+          post "#{prefix}/ondemandvm", '{"pool4":"1"}'
+          expect_json(false, 404)
+
+          expected = {
+            "ok": false,
+            "bad_templates": [ badpool ]
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'returns 400 and a message when JSON is invalid' do
+        post "#{prefix}/ondemandvm", '{"pool1":"1}'
+        expect_json(false, 400)
+        expected = {
+          "ok": false,
+          "message": "JSON payload could not be parsed"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+
+    describe 'GET /ondemandvm' do
+      it 'returns 404 with message when request is not found' do
+        get "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:score) { current_time }
+        let(:platforms_string) { 'pool1:pool1:1' }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 202 while the request is waiting' do
+          get "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 202)
+          expected = {
+            "ok": true,
+            "request_id": uuid,
+            "ready": false,
+            "pool1": {
+              "ready": "0",
+              "pending": "1"
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        context 'with ready instances' do
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis)
+            set_ondemand_request_status(uuid, 'ready', redis)
+          end
+
+          it 'returns 200 with hostnames when the request is ready' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": true,
+              "pool1": {
+                "hostname": [
+                  "#{vmname}.example.com"
+                ]
+              }
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with a deleted request' do
+          before(:each) do
+            set_ondemand_request_status(uuid, 'deleted', redis)
+          end
+
+          it 'returns a message that the request has been deleted' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request has been deleted"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with a failed request' do
+          let(:ondemand_request_ttl) { 5 }
+          before(:each) do
+            config[:config]['ondemand_request_ttl'] = ondemand_request_ttl
+            set_ondemand_request_status(uuid, 'failed', redis)
+          end
+
+          it 'returns a message that the request has failed' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request failed to provision instances within the configured ondemand_request_ttl '#{ondemand_request_ttl}'"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+      end
+    end
+
+    describe 'DELETE /ondemandvm' do
+      let(:expiration) { 129_600_0 }
+      it 'returns 404 with message when request is not found' do
+        delete "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:platforms_string) { 'pool1:pool1:1' }
+        let(:score) { current_time.to_i }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 200 for a deleted request' do
+          delete "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 200)
+          expected = { 'ok': true }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        it 'marks the request hash for expiration in two weeks' do
+          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{uuid}", expiration)
+          delete "#{prefix}/ondemandvm/#{uuid}"
+        end
+
+        context 'with running instances' do
+          let(:pool) { 'pool1' }
+          let(:pool_alias) { pool }
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, pool, pool_alias, redis)
+          end
+
+          it 'moves allocated instances to the completed queue' do
+            expect(redis).to receive(:smove).with("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vmname)
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+
+          it 'deletes the set tracking instances allocated for the request' do
+            expect(redis).to receive(:del).with("vmpooler__#{uuid}__#{pool_alias}__#{pool}")
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/integration/api/v3/poolreset.rb b/spec/integration/api/v3/poolreset.rb
new file mode 100644
index 0000000..74a11e3
--- /dev/null
+++ b/spec/integration/api/v3/poolreset.rb
@@ -0,0 +1,120 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V3 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  after(:each) do
+    Vmpooler::API.reset!
+  end
+
+  let(:config) {
+    {
+      config: {
+        'site_name' => 'test pooler',
+        'vm_lifetime_auth' => 2,
+        'experimental_features' => true
+      },
+      pools: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      statsd: { 'prefix' => 'stats_prefix'},
+      alias: { 'poolone' => 'pool1' },
+      pool_names: [ 'pool1', 'pool2', 'poolone' ]
+    }
+  }
+
+  describe '/poolreset' do
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /poolreset' do
+      it 'refreshes ready and pending instances from a pool' do
+        post "#{prefix}/poolreset", '{"pool1":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/poolreset", '{"poolpoolpool":"1"}'
+        expect_json(ok = false, http = 400)
+      end
+
+      it 'resets multiple pools' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_pools: ['pool3']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/poolreset", '{"pool1":"1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'should return 400 for invalid json' do
+        post "#{prefix}/poolreset", '{"pool1":"1}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 400 with a bad pool name' do
+        post "#{prefix}/poolreset", '{"pool11":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 404 when there is no payload' do
+        post "#{prefix}/poolreset"
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+  end
+end
diff --git a/spec/integration/api/v1/status_spec.rb b/spec/integration/api/v3/status_spec.rb
similarity index 90%
rename from spec/integration/api/v1/status_spec.rb
rename to spec/integration/api/v3/status_spec.rb
index ffd69aa..5a5449c 100644
--- a/spec/integration/api/v1/status_spec.rb
+++ b/spec/integration/api/v3/status_spec.rb
@@ -5,15 +5,24 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+    # Clear status cache to prevent test interference
+    Vmpooler::API::V3.clear_status_cache
+  end
+
   describe 'status and metrics endpoints' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
 
     let(:config) {
       {
@@ -32,8 +41,8 @@ describe Vmpooler::API::V1 do
     let(:current_time) { Time.now }
 
     before(:each) do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
       app.settings.set :config, auth: false
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
@@ -50,7 +59,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of ready vms for each pool' do
-        3.times {|i| create_ready_vm("pool1", "vm-#{i}") }
+        3.times {|i| create_ready_vm("pool1", "vm-#{i}", redis) }
         get "#{prefix}/status/"
 
         # of course /status doesn't conform to the weird standard everything else uses...
@@ -61,8 +70,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of running vms for each pool' do
-        3.times {|i| create_running_vm("pool1", "vm-#{i}") }
-        4.times {|i| create_running_vm("pool2", "vm-#{i}") }
+        3.times {|i| create_running_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_running_vm("pool2", "vm-#{i}", redis) }
 
         get "#{prefix}/status/"
 
@@ -74,8 +83,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the number of pending vms for each pool' do
-        3.times {|i| create_pending_vm("pool1", "vm-#{i}") }
-        4.times {|i| create_pending_vm("pool2", "vm-#{i}") }
+        3.times {|i| create_pending_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_pending_vm("pool2", "vm-#{i}", redis) }
 
         get "#{prefix}/status/"
 
@@ -230,8 +239,8 @@ describe Vmpooler::API::V1 do
       it 'returns the number of running VMs' do
         get "#{prefix}/totalrunning"
         expect(last_response.header['Content-Type']).to eq('application/json')
-        5.times {|i| create_running_vm("pool1", "vm-#{i}") }
-        5.times {|i| create_running_vm("pool3", "vm-#{i}") }
+        5.times {|i| create_running_vm("pool1", "vm-#{i}", redis, redis) }
+        5.times {|i| create_running_vm("pool3", "vm-#{i}", redis, redis) }
         result = JSON.parse(last_response.body)
         expect(result["running"] == 10)
       end
diff --git a/spec/integration/api/v1/token_spec.rb b/spec/integration/api/v3/token_spec.rb
similarity index 80%
rename from spec/integration/api/v1/token_spec.rb
rename to spec/integration/api/v3/token_spec.rb
index 983dac6..2fd0dac 100644
--- a/spec/integration/api/v1/token_spec.rb
+++ b/spec/integration/api/v3/token_spec.rb
@@ -1,26 +1,38 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
-  describe '/token' do
-    let(:prefix) { '/api/v1' }
-    let(:current_time) { Time.now }
-    let(:config) { { } }
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
 
-    before do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
+  describe '/token' do
+    let(:prefix) { '/api/v3' }
+    let(:current_time) { Time.now }
+    let(:config) { { 
+      config: {}
+    } }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
     end
 
     describe 'GET /token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
 
         it 'returns a 404' do
           get "#{prefix}/token"
@@ -31,6 +43,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }
@@ -58,7 +71,10 @@ describe Vmpooler::API::V1 do
 
     describe 'POST /token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {}, 
+          auth: false 
+        } }
 
         it 'returns a 404' do
           post "#{prefix}/token"
@@ -69,6 +85,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }
@@ -94,10 +111,12 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/token/:token' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:current_time) { Time.now }
 
-    before do
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
       app.settings.set :config, config
       app.settings.set :redis, redis
     end
@@ -109,7 +128,10 @@ describe Vmpooler::API::V1 do
 
     describe 'GET /token/:token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
 
         it 'returns a 404' do
           get "#{prefix}/token/this"
@@ -119,6 +141,7 @@ describe Vmpooler::API::V1 do
 
       context '(auth configured)' do
         let(:config) { {
+          config: {},
           auth: true,
           pools: [
             {'name' => 'pool1', 'size' => 5}
@@ -141,7 +164,10 @@ describe Vmpooler::API::V1 do
 
     describe 'DELETE /token/:token' do
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        let(:config) { { 
+          config: {},
+          auth: false
+        } }
 
         it 'returns a 404' do
           delete "#{prefix}/token/this"
@@ -152,6 +178,7 @@ describe Vmpooler::API::V1 do
       context '(auth configured)' do
         let(:config) {
           {
+            config: {},
             auth: {
               'provider' => 'dummy'
             }
diff --git a/spec/integration/api/v1/vm_hostname_spec.rb b/spec/integration/api/v3/vm_hostname_spec.rb
similarity index 67%
rename from spec/integration/api/v1/vm_hostname_spec.rb
rename to spec/integration/api/v3/vm_hostname_spec.rb
index 1e6ac19..b4c064d 100644
--- a/spec/integration/api/v1/vm_hostname_spec.rb
+++ b/spec/integration/api/v3/vm_hostname_spec.rb
@@ -5,42 +5,51 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm/:hostname' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
+
         },
         pools: [
           {'name' => 'pool1', 'size' => 5},
           {'name' => 'pool2', 'size' => 10}
         ],
         alias: { 'poolone' => 'pool1' },
+        auth: false
       }
     }
 
     let(:current_time) { Time.now }
 
     before(:each) do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :config, auth: false
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'PUT /vm/:hostname' do
       it 'allows tags to be set' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested_by":"rspec"}}'
         expect_json(ok = true, http = 200)
 
@@ -48,7 +57,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'skips empty tags' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested_by":""}}'
         expect_json(ok = true, http = 200)
 
@@ -56,7 +65,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'does not set tags if request body format is invalid' do
-        create_vm('testhost')
+        create_vm('testhost', redis)
         put "#{prefix}/vm/testhost", '{"tags":{"tested"}}'
         expect_json(ok = false, http = 400)
 
@@ -68,7 +77,7 @@ describe Vmpooler::API::V1 do
           app.settings.set :config,
              { :config => { 'allowed_tags' => ['created_by', 'project', 'url'] } }
 
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"tags":{"created_by":"rspec","tested_by":"rspec"}}'
           expect_json(ok = false, http = 400)
@@ -78,12 +87,12 @@ describe Vmpooler::API::V1 do
       end
 
       context '(tagfilter configured)' do
-        let(:config) { {
-          tagfilter: { 'url' => '(.*)\/' },
-        } }
+        before(:each) do
+          app.settings.set :config, tagfilter: { 'url' => '(.*)\/' }
+        end
 
         it 'correctly filters tags' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com/something.html"}}'
           expect_json(ok = true, http = 200)
@@ -92,7 +101,7 @@ describe Vmpooler::API::V1 do
         end
 
         it "doesn't eat tags not matching filter" do
-          create_vm('testhost')
+          create_vm('testhost', redis)
           put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com"}}'
           expect_json(ok = true, http = 200)
 
@@ -101,10 +110,12 @@ describe Vmpooler::API::V1 do
       end
 
       context '(auth not configured)' do
-        let(:config) { { auth: false } }
+        before(:each) do
+          app.settings.set :config, auth: false
+        end
 
         it 'allows VM lifetime to be modified without a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
           expect_json(ok = true, http = 200)
@@ -114,7 +125,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'does not allow a lifetime to be 0' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"0"}'
           expect_json(ok = false, http = 400)
@@ -122,6 +133,42 @@ describe Vmpooler::API::V1 do
           vm = fetch_vm('testhost')
           expect(vm['lifetime']).to be_nil
         end
+
+        it 'does not enforce a lifetime' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"20000"}'
+          expect_json(ok = true, http = 200)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to eq("20000")
+        end
+
+        it 'does not allow a lifetime to be initially past config max_lifetime_upper_limit' do
+          app.settings.set :config,
+                           { :config => { 'max_lifetime_upper_limit' => 168 } }
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"200"}'
+          expect_json(ok = false, http = 400)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to be_nil
+        end
+
+#       it 'does not allow a lifetime to be extended past config 168' do
+#         app.settings.set :config,
+#                          { :config => { 'max_lifetime_upper_limit' => 168 } }
+#         create_vm('testhost', redis)
+#
+#         set_vm_data('testhost', "checkout", (Time.now - (69*60*60)), redis)
+#         puts redis.hget("vmpooler__vm__testhost", 'checkout')
+#         put "#{prefix}/vm/testhost", '{"lifetime":"100"}'
+#         expect_json(ok = false, http = 400)
+#
+#         vm = fetch_vm('testhost')
+#         expect(vm['lifetime']).to be_nil
+#       end
       end
 
       context '(auth configured)' do
@@ -130,7 +177,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'allows VM lifetime to be modified with a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}', {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
@@ -142,7 +189,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'does not allows VM lifetime to be modified without a token' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
           expect_json(ok = false, http = 401)
@@ -158,7 +205,7 @@ describe Vmpooler::API::V1 do
         end
 
         it 'deletes an existing VM' do
-          create_running_vm('pool1', 'testhost')
+          create_running_vm('pool1', 'testhost', redis)
           expect fetch_vm('testhost')
 
           delete "#{prefix}/vm/testhost"
@@ -174,7 +221,7 @@ describe Vmpooler::API::V1 do
 
         context '(checked-out without token)' do
           it 'deletes a VM without supplying a token' do
-            create_running_vm('pool1', 'testhost')
+            create_running_vm('pool1', 'testhost', redis)
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost"
@@ -185,7 +232,7 @@ describe Vmpooler::API::V1 do
 
         context '(checked-out with token)' do
           it 'fails to delete a VM without supplying a token' do
-            create_running_vm('pool1', 'testhost', 'abcdefghijklmnopqrstuvwxyz012345')
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost"
@@ -194,7 +241,7 @@ describe Vmpooler::API::V1 do
           end
 
           it 'deletes a VM when token is supplied' do
-            create_running_vm('pool1', 'testhost', 'abcdefghijklmnopqrstuvwxyz012345')
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
             expect fetch_vm('testhost')
 
             delete "#{prefix}/vm/testhost", "", {
@@ -211,7 +258,7 @@ describe Vmpooler::API::V1 do
     describe 'POST /vm/:hostname/snapshot' do
       context '(auth not configured)' do
         it 'creates a snapshot' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
           post "#{prefix}/vm/testhost/snapshot"
           expect_json(ok = true, http = 202)
           expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
@@ -226,19 +273,19 @@ describe Vmpooler::API::V1 do
         it 'returns a 401 if not authed' do
           post "#{prefix}/vm/testhost/snapshot"
           expect_json(ok = false, http = 401)
-          expect !has_vm_snapshot?('testhost')
+          expect !has_vm_snapshot?('testhost', redis)
         end
 
         it 'creates a snapshot if authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = true, http = 202)
           expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
-          expect has_vm_snapshot?('testhost')
+          expect has_vm_snapshot?('testhost', redis)
         end
       end
     end
@@ -246,22 +293,22 @@ describe Vmpooler::API::V1 do
     describe 'POST /vm/:hostname/snapshot/:snapshot' do
       context '(auth not configured)' do
         it 'reverts to a snapshot' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot"
           expect_json(ok = true, http = 202)
-          expect vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'fails if the specified snapshot does not exist' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = false, http = 404)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
       end
 
@@ -271,33 +318,33 @@ describe Vmpooler::API::V1 do
         end
 
         it 'returns a 401 if not authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot"
           expect_json(ok = false, http = 401)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'fails if authed and the specified snapshot does not exist' do
-          create_vm('testhost')
+          create_vm('testhost', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = false, http = 404)
-          expect !vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
 
         it 'reverts to a snapshot if authed' do
-          create_vm('testhost')
-          snapshot_vm('testhost', 'testsnapshot')
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
 
           post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
             'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
           }
           expect_json(ok = true, http = 202)
-          expect vm_reverted_to_snapshot?('testhost', 'testsnapshot')
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
         end
       end
     end
diff --git a/spec/integration/api/v1/vm_spec.rb b/spec/integration/api/v3/vm_spec.rb
similarity index 70%
rename from spec/integration/api/v1/vm_spec.rb
rename to spec/integration/api/v3/vm_spec.rb
index 9bbe08a..0800d7c 100644
--- a/spec/integration/api/v1/vm_spec.rb
+++ b/spec/integration/api/v3/vm_spec.rb
@@ -1,26 +1,52 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2
         },
+        providers: {
+          vsphere: {},
+          gce: {},
+          foo: {}
+        },
+        dns_configs: {
+          :one => {
+            'dns_class' => 'mock',
+            'domain' => 'one.example.com'
+          },
+          :two => {
+            'dns_class' => 'mock',
+            'domain' => 'two.example.com'
+          },
+          :three => {
+            'dns_class' => 'mock',
+            'domain' => 'three.example.com'
+          }
+        },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10},
-          {'name' => 'pool3', 'size' => 10}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'one'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'gce', 'dns_plugin' => 'two'},
+          {'name' => 'pool3', 'size' => 10, 'provider' => 'foo', 'dns_plugin' => 'three'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
         alias: { 'poolone' => ['pool1'] },
@@ -29,18 +55,20 @@ describe Vmpooler::API::V1 do
     }
     let(:current_time) { Time.now }
     let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
 
     before(:each) do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'GET /vm/:hostname' do
       it 'returns correct information on a running vm' do
-        create_running_vm 'pool1', vmname
+        create_running_vm 'pool1', vmname, redis
+        expect(TCPSocket).to receive(:gethostbyname).and_raise(RuntimeError)
         get "#{prefix}/vm/#{vmname}"
         expect_json(ok = true, http = 200)
         response_body = (JSON.parse(last_response.body)[vmname])
@@ -61,7 +89,7 @@ describe Vmpooler::API::V1 do
 
       let(:socket) { double('socket') }
       it 'returns a single VM' do
-        create_ready_vm 'pool1', vmname
+        create_ready_vm 'pool1', vmname, redis
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
         post "#{prefix}/vm", '{"pool1":"1"}'
@@ -70,7 +98,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -78,7 +106,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns a single VM for an alias' do
-        create_ready_vm 'pool1', vmname
+        create_ready_vm 'pool1', vmname, redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -88,7 +116,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           poolone: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -101,10 +129,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool when aliases are not defined' do
-        Vmpooler::API.settings.config.delete(:alias)
-        Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2']
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
 
-        create_ready_vm 'pool1', vmname
+        create_ready_vm 'pool1', vmname, redis
         post "#{prefix}/vm/pool1"
         post "#{prefix}/vm/pool1"
 
@@ -115,8 +143,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', vmname
-        post "#{prefix}/vm/poolone"
+        create_ready_vm 'pool2', vmname, redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -126,8 +153,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs' do
-        create_ready_vm 'pool1', vmname
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', vmname, redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -137,10 +164,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.two.example.com'
           }
         }
 
@@ -148,9 +175,9 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs even when multiple instances from the same pool are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -168,18 +195,18 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
 
       it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
-        create_ready_vm 'pool2', '1qrstuvwxyz012345'
-        create_ready_vm 'pool2', '2qrstuvwxyz012345'
-        create_ready_vm 'pool2', '3qrstuvwxyz012345'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -197,8 +224,8 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.two.example.com', '2qrstuvwxyz012345.two.example.com', '3qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
@@ -206,9 +233,9 @@ describe Vmpooler::API::V1 do
       it 'returns VMs from multiple backend pools requested by an alias' do
         Vmpooler::API.settings.config[:alias]['genericpool'] = ['pool1', 'pool2', 'pool3']
 
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool2', '2abcdefghijklmnop'
-        create_ready_vm 'pool3', '1qrstuvwxyz012345'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool3', '1qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -223,15 +250,15 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop', '1qrstuvwxyz012345')
+        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.two.example.com', '1qrstuvwxyz012345.three.example.com')
 
         expect_json(ok = true, http = 200)
       end
 
       it 'returns the first VM that was moved to the ready state when checking out a VM' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
-        create_ready_vm 'pool1', '3abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '3abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -240,7 +267,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           "pool1": {
-            "hostname": "1abcdefghijklmnop"
+            "hostname": "1abcdefghijklmnop.one.example.com"
           }
         }
 
@@ -249,7 +276,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'fails when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
 
@@ -260,7 +287,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -271,11 +298,11 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
 
@@ -286,7 +313,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -297,11 +324,11 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
 
         post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
 
@@ -312,8 +339,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -324,13 +351,13 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'returns the second VM when the first fails to respond' do
-        create_ready_vm 'pool1', vmname
-        create_ready_vm 'pool1', "2#{vmname}"
+        create_running_vm 'pool1', vmname, redis
+        create_ready_vm 'pool1', "2#{vmname}", redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with(vmname, nil).and_raise('mockerror')
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with("2#{vmname}", nil).and_return(socket)
@@ -341,20 +368,20 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: "2#{vmname}"
+            hostname: "2#{vmname}.one.example.com"
           }
         }
 
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
 
-        expect(pool_has_ready_vm?('pool1', vmname)).to be false
+        expect(pool_has_ready_vm?('pool1', vmname, redis)).to be false
       end
 
       context '(auth not configured)' do
         it 'does not extend VM lifetime if auth token is provided' do
           app.settings.set :config, auth: false
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -366,7 +393,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -380,7 +407,7 @@ describe Vmpooler::API::V1 do
         it 'extends VM lifetime if auth token is provided' do
           app.settings.set :config, auth: true
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -392,7 +419,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -403,7 +430,7 @@ describe Vmpooler::API::V1 do
 
         it 'does not extend VM lifetime if auth token is not provided' do
           app.settings.set :config, auth: true
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -413,7 +440,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
diff --git a/spec/integration/api/v1/vm_template_spec.rb b/spec/integration/api/v3/vm_template_spec.rb
similarity index 67%
rename from spec/integration/api/v1/vm_template_spec.rb
rename to spec/integration/api/v3/vm_template_spec.rb
index d94b537..f809244 100644
--- a/spec/integration/api/v1/vm_template_spec.rb
+++ b/spec/integration/api/v3/vm_template_spec.rb
@@ -1,25 +1,40 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
     Vmpooler::API
   end
 
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
   describe '/vm/:template' do
-    let(:prefix) { '/api/v1' }
-    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:prefix) { '/api/v3' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
         },
+        dns_configs: {
+          :example => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
+        providers: { vsphere: {} },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'poolone', 'size' => 1, 'provider' => 'vsphere', 'dns_plugin' => 'example'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
         alias: { 'poolone' => 'pool1' },
@@ -29,18 +44,19 @@ describe Vmpooler::API::V1 do
 
     let(:current_time) { Time.now }
     let(:socket) { double('socket') }
+    let(:checkoutlock) { Mutex.new }
 
     before(:each) do
-      app.settings.set :config, config
-      app.settings.set :redis, redis
-      app.settings.set :metrics, metrics
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
       app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
     describe 'POST /vm/:template' do
       it 'returns a single VM' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -50,7 +66,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
 
@@ -58,7 +74,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns a single VM for an alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -67,7 +83,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           poolone: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
         expect_json(ok = true, http = 200)
@@ -81,10 +97,10 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool when aliases are not defined' do
-        Vmpooler::API.settings.config.delete(:alias)
-        Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2']
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
 
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
         post "#{prefix}/vm/pool1"
         post "#{prefix}/vm/pool1"
 
@@ -95,8 +111,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        post "#{prefix}/vm/poolone"
+        create_ready_vm 'pool2', 'abcdefghijklmnop', redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -106,8 +121,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', 'qrstuvwxyz012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -117,10 +132,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.example.com'
           }
         }
 
@@ -128,12 +143,12 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
-        create_ready_vm 'pool1', '1abcdefghijklmnop'
-        create_ready_vm 'pool1', '2abcdefghijklmnop'
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
 
-        create_ready_vm 'pool2', '1qrstuvwxyz012345'
-        create_ready_vm 'pool2', '2qrstuvwxyz012345'
-        create_ready_vm 'pool2', '3qrstuvwxyz012345'
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -142,22 +157,22 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+            hostname: [ '1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com' ]
           },
           pool2: {
-            hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ]
+            hostname: [ '1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com' ]
           }
         }
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com')
         expect_json(ok = true, http = 200)
       end
 
       it 'fails when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
         post "#{prefix}/vm/pool1+pool2", ''
 
@@ -168,7 +183,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -179,12 +194,12 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool1', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
 
         post "#{prefix}/vm/pool1+pool1+pool2", ''
 
@@ -195,8 +210,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool1', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -207,13 +222,13 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool1', '0123456789012345')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '0123456789012345', redis)).to eq(true)
       end
 
       it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
 
         post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
 
@@ -224,8 +239,8 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop'
-        create_ready_vm 'pool2', '0123456789012345'
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -236,15 +251,15 @@ describe Vmpooler::API::V1 do
         expect(last_response.body).to eq(JSON.pretty_generate(expected))
         expect_json(ok = false, http = 503)
 
-        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true)
-        expect(pool_has_ready_vm?('pool2', '0123456789012345')).to eq(true)
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool2', '0123456789012345', redis)).to eq(true)
       end
 
       context '(auth not configured)' do
         it 'does not extend VM lifetime if auth token is provided' do
           app.settings.set :config, auth: false
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -256,7 +271,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
 
@@ -270,7 +285,7 @@ describe Vmpooler::API::V1 do
         it 'extends VM lifetime if auth token is provided' do
           app.settings.set :config, auth: true
 
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -282,7 +297,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -293,7 +308,7 @@ describe Vmpooler::API::V1 do
 
         it 'does not extend VM lifetime if auth token is not provided' do
           app.settings.set :config, auth: true
-          create_ready_vm 'pool1', 'abcdefghijklmnop'
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
 
           allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
 
@@ -302,7 +317,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect_json(ok = true, http = 200)
diff --git a/spec/integration/dashboard_spec.rb b/spec/integration/dashboard_spec.rb
index 5bb9ac2..b0abbea 100644
--- a/spec/integration/dashboard_spec.rb
+++ b/spec/integration/dashboard_spec.rb
@@ -5,13 +5,36 @@ describe Vmpooler::API do
   include Rack::Test::Methods
 
   def app()
-    described_class
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
   end
 
   describe 'Dashboard' do
 
+    let(:config) { {
+      config: {},
+      pools: [
+          {'name' => 'pool1', 'size' => 5}
+      ],
+      graphite: {}
+    } }
+
+    before(:each) do
+      expect(app).to receive(:run!)
+      app.execute([:api], config, redis, nil, nil)
+      app.settings.set :config, auth: false
+    end
+
     context '/' do
-      before { get '/' }
+      before(:each) do
+        get '/'
+      end
 
       it { expect(last_response.status).to eq(302) }
       it { expect(last_response.location).to eq('http://example.org/dashboard/') }
@@ -21,7 +44,7 @@ describe Vmpooler::API do
       ENV['SITE_NAME'] = 'test pooler'
       ENV['VMPOOLER_CONFIG'] = 'thing'
 
-      before do
+      before(:each) do
         get '/dashboard/'
       end
 
@@ -31,15 +54,18 @@ describe Vmpooler::API do
     end
 
     context 'unknown route' do
-      before { get '/doesnotexist' }
+      before(:each) do
+        get '/doesnotexist'
+      end
 
-      it { expect(last_response.status).to eq(404) }
       it { expect(last_response.header['Content-Type']).to eq('application/json') }
+      it { expect(last_response.status).to eq(404) }
       it { expect(last_response.body).to eq(JSON.pretty_generate({ok: false})) }
     end
 
     describe '/dashboard/stats/vmpooler/pool' do
       let(:config) { {
+          config: {},
           pools: [
               {'name' => 'pool1', 'size' => 5},
               {'name' => 'pool2', 'size' => 1}
@@ -47,20 +73,18 @@ describe Vmpooler::API do
           graphite: {}
       } }
 
-      before do
-        $config = config
+      before(:each) do
         app.settings.set :config, config
-        app.settings.set :redis, redis
         app.settings.set :environment, :test
       end
 
       context 'without history param' do
         it 'returns basic JSON' do
-          create_ready_vm('pool1', 'vm1')
-          create_ready_vm('pool1', 'vm2')
-          create_ready_vm('pool1', 'vm3')
-          create_ready_vm('pool2', 'vm4')
-          create_ready_vm('pool2', 'vm5')
+          create_ready_vm('pool1', 'vm1', redis)
+          create_ready_vm('pool1', 'vm2', redis)
+          create_ready_vm('pool1', 'vm3', redis)
+          create_ready_vm('pool2', 'vm4', redis)
+          create_ready_vm('pool2', 'vm5', redis)
 
           get '/dashboard/stats/vmpooler/pool'
 
@@ -90,11 +114,11 @@ describe Vmpooler::API do
         end
 
         it 'returns JSON with history when redis has values' do
-          create_ready_vm('pool1', 'vm1')
-          create_ready_vm('pool1', 'vm2')
-          create_ready_vm('pool1', 'vm3')
-          create_ready_vm('pool2', 'vm4')
-          create_ready_vm('pool2', 'vm5')
+          create_ready_vm('pool1', 'vm1', redis)
+          create_ready_vm('pool1', 'vm2', redis)
+          create_ready_vm('pool1', 'vm3', redis)
+          create_ready_vm('pool2', 'vm4', redis)
+          create_ready_vm('pool2', 'vm5', redis)
 
           get '/dashboard/stats/vmpooler/pool', :history => true
 
@@ -112,6 +136,7 @@ describe Vmpooler::API do
 
     describe '/dashboard/stats/vmpooler/running' do
       let(:config) { {
+          config: {},
           pools: [
               {'name' => 'pool-1', 'size' => 5},
               {'name' => 'pool-2', 'size' => 1},
@@ -120,10 +145,8 @@ describe Vmpooler::API do
           graphite: {}
       } }
 
-      before do
-        $config = config
+      before(:each) do
         app.settings.set :config, config
-        app.settings.set :redis, redis
         app.settings.set :environment, :test
       end
 
@@ -140,18 +163,18 @@ describe Vmpooler::API do
         end
 
         it 'adds major correctly' do
-          create_running_vm('pool-1', 'vm1')
-          create_running_vm('pool-1', 'vm2')
-          create_running_vm('pool-1', 'vm3')
+          create_running_vm('pool-1', 'vm1', redis)
+          create_running_vm('pool-1', 'vm2', redis)
+          create_running_vm('pool-1', 'vm3', redis)
 
-          create_running_vm('pool-2', 'vm4')
-          create_running_vm('pool-2', 'vm5')
-          create_running_vm('pool-2', 'vm6')
-          create_running_vm('pool-2', 'vm7')
-          create_running_vm('pool-2', 'vm8')
+          create_running_vm('pool-2', 'vm4', redis)
+          create_running_vm('pool-2', 'vm5', redis)
+          create_running_vm('pool-2', 'vm6', redis)
+          create_running_vm('pool-2', 'vm7', redis)
+          create_running_vm('pool-2', 'vm8', redis)
 
-          create_running_vm('diffpool-1', 'vm9')
-          create_running_vm('diffpool-1', 'vm10')
+          create_running_vm('diffpool-1', 'vm9', redis)
+          create_running_vm('diffpool-1', 'vm10', redis)
 
           get '/dashboard/stats/vmpooler/running'
 
diff --git a/spec/rbvmomi_helper.rb b/spec/rbvmomi_helper.rb
deleted file mode 100644
index 88da343..0000000
--- a/spec/rbvmomi_helper.rb
+++ /dev/null
@@ -1,781 +0,0 @@
-# -----------------------------------------------------------------------------------------------------------------
-# Managed Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-mo_types.html)
-# -----------------------------------------------------------------------------------------------------------------
-
-MockClusterComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-  # From MockClusterComputeResource
-  :actionHistory, :configuration, :drsFault, :drsRecommendation, :migrationHistory, :recommendation,
-  # From ComputeResource
-  :resourcePool,
-  # From ManagedEntity
-  :name
-)
-
-MockComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-  # From ComputeResource
-  :configurationEx, :datastore, :host, :network, :resourcePool, :summary,
-  # From ManagedEntity
-  :name
-)
-
-MockContainerView = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-  # From ContainerView
-  :container, :recursive, :type
-)
-
-MockDatacenter = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datacenter.html
-  # From Datacenter
-  :datastore, :datastoreFolder, :hostFolder, :network, :networkFolder, :vmFolder,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Datacenter https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Datacenter.rb
-
-  # Find the Datastore with the given +name+.
-  def find_datastore name
-    datastore.find { |x| x.name == name }
-  end
-end
-
-MockDatastore = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-  # From Datastore
-  :browser, :capability, :host, :info, :iormConfiguration, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockFolder = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Folder.html
-  # From Folder
-  :childEntity, :childType,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Folder https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L107-L110
-  def children
-    childEntity
-  end
-
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L9-L12
-  def find(name, type=Object)
-    # Fake the searchIndex
-    childEntity.each do |child|
-      if child.name == name
-        if child.kind_of?(type)
-          return child
-        else
-          return nil
-        end
-      end
-    end
-
-    nil
-  end
-end
-
-MockHostSystem = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-  # From HostSystem
-  :capability, :config, :configManager, :datastore, :datastoreBrowser, :hardware, :network, :runtime, :summary, :systemResources, :vm,
-  # From ManagedEntity
-  :overallStatus, :name, :parent,
-  # From ManagedObject
-  :configIssue
-)
-
-MockPropertyCollector = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-  # PropertyCollector
-  :filter
-)
-
-MockResourcePool = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-  # From ResourcePool
-  :childConfiguration, :config, :owner, :resourcePool, :runtime, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockSearchIndex = Object
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.SearchIndex.html
-
-MockServiceInstance = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-  # From ServiceInstance
-  :capability, :content, :serverClock
-) do
-  # From ServiceInstance 
-  # Mock the CurrentTime method so that it appears the ServiceInstance is valid.
-  def CurrentTime
-    Time.now
-  end
-
-  # From RBVMOMI::VIM::ServiceInstance https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ServiceInstance.rb
-  def find_datacenter(path=nil)
-    # In our mocked instance, DataCenters are always in the root Folder.
-    # If path is nil the first DC is returned otherwise match by name
-    content.rootFolder.childEntity.each do |child|
-      if child.is_a?(RbVmomi::VIM::Datacenter)
-        return child if path.nil? || child.name == path
-      end
-    end
-    nil
-  end
-end
-
-MockTask = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-  # From Task
-  :info,
-) do
-  # From RBVMOMI https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Task.rb
-  # Mock the with 'Not Implemented'
-  def wait_for_completion
-    raise(RuntimeError,'Not Implemented')
-  end
-end
-
-MockViewManager = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ViewManager.html
-  # From ViewManager
-  :viewList,
-) do
-  # From ViewManager 
-  def CreateContainerView(options)
-    mock_RbVmomi_VIM_ContainerView({
-      :container => options[:container],
-      :recursive => options[:recursive],
-      :type => options[:type],
-    })
-  end
-end
-
-MockVirtualDiskManager = Object
-# https://pubs.vmware.com/vsphere-55/index.jsp#com.vmware.wssdk.apiref.doc/vim.VirtualDiskManager.html
-
-MockVirtualMachine = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-  # From VirtualMachine
-  :config, :runtime, :snapshot, :summary,
-  # From ManagedEntity
-  :name,
-  # From RbVmomi::VIM::ManagedEntity
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ManagedEntity.rb
-  :path
-)
-
-MockVirtualMachineSnapshot = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-  # From VirtualMachineSnapshot
-  :config
-)
-
-# -------------------------------------------------------------------------------------------------------------
-# Data Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-do_types.html)
-# -------------------------------------------------------------------------------------------------------------
-
-MockDescription = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Description.html
-  # From Description
-  :label, :summary
-)
-
-MockDynamicProperty = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.DynamicProperty.html
-  # From DynamicProperty
-  :name, :val
-)
-
-MockHostCpuPackage = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.CpuPackage.html
-  # From HostCpuPackage
-  :busHz, :cpuFeature, :description, :hz, :index, :threadId, :vendor
-)
-
-MockHostHardwareSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.HardwareSummary.html
-  # From HostHardwareSummary
-  :cpuMhz, :numCpuCores, :numCpuPkgs, :memorySize
-)
-
-MockHostListSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.html
-  # From HostListSummary
-  :quickStats, :hardware
-)
-
-MockHostListSummaryQuickStats = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.QuickStats.html
-  # From HostListSummaryQuickStats
-  :overallCpuUsage, :overallMemoryUsage
-)
-
-MockHostRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.RuntimeInfo.html
-  # From HostRuntimeInfo
-  :bootTime, :connectionState, :healthSystemRuntime, :inMaintenanceMode, :powerState, :tpmPcrValues
-)
-
-MockHostSystemHostHardwareInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.HardwareInfo.html
-  # From HostHardwareInfo
-  :biosInfo, :cpuFeature, :cpuInfo, :cpuPkg, :cpuPowerManagementInfo, :memorySize, :numaInfo, :pciDevice, :systemInfo
-)
-
-MockObjectContent = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.ObjectContent.html
-  # From ObjectContent
-  :missingSet, :obj, :propSet
-)
-
-MockRetrieveResult = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-  # From RetrieveResult
-  :objects, :token
-)
-
-MockServiceContent = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-  # From ServiceContent
-  :propertyCollector, :rootFolder, :searchIndex, :viewManager, :virtualDiskManager
-)
-
-MockVirtualDevice = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualDisk = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-  # From VirtualDisk
-  :capacityInKB, :shares,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualHardware = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.VirtualHardware.html
-  # From VirtualHardware
-  :device
-)
-
-MockVirtualMachineConfigInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.ConfigInfo.html
-  # From VirtualMachineConfigInfo
-  :hardware
-)
-
-MockVirtualMachineFileLayoutExFileInfo = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-  # From VirtualMachineFileLayoutExFileInfo
-  :key, :name, :size, :type, :uniqueSize
-)
-
-MockVirtualMachineGuestSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.GuestSummary.html
-  # From VirtualMachineGuestSummary
-  :hostName
-)
-
-MockVirtualMachineRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.RuntimeInfo.html
-  # From VirtualMachineRuntimeInfo
-  :bootTime, :cleanPowerOff, :connectionState, :faultToleranceState, :host, :maxCpuUsage, :maxMemoryUsage, :memoryOverhead,
-  :needSecondaryReason, :numMksConnections, :powerState, :question, :recordReplayState, :suspendInterval, :suspendTime, :toolsInstallerMounted
-)
-
-MockVirtualMachineSnapshotInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotInfo.html
-  # From MockVirtualMachineSnapshotInfo
-  :currentSnapshot, :rootSnapshotList
-)
-
-MockVirtualMachineSnapshotTree = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotTree.html
-  # From VirtualMachineSnapshotTree
-  :backupManifest, :childSnapshotList, :createTime, :description, :id, :name, :quiesced, :replaySupported,
-  :snapshot, :state, :vm
-)
-
-MockVirtualMachineSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.html
-  # From VirtualMachineSummary
-  :config, :customValue, :guest, :quickStats, :runtime, :storage, :vm
-)
-
-MockVirtualSCSIController = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-  # From VirtualSCSIController
-  :hotAddRemove, :scsiCtlrUnitNumber, :sharedBus,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-# --------------------
-# RBVMOMI only Objects
-# --------------------
-MockRbVmomiVIMConnection = Struct.new(
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  :serviceInstance, :serviceContent, :rootFolder, :root
-) do
-  # From https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  # Alias to serviceContent.searchIndex
-  def searchIndex
-    serviceContent.searchIndex
-  end
-  # Alias to serviceContent.propertyCollector
-  def propertyCollector
-    serviceContent.propertyCollector
-  end
-end
-
-# -------------------------------------------------------------------------------------------------------------
-# Mocking Methods
-# -------------------------------------------------------------------------------------------------------------
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-def mock_RbVmomi_VIM_ClusterComputeResource(options = {})
-  options[:name]  = 'Cluster' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockClusterComputeResource.new()
-
-  mock.name = options[:name]
-  # All cluster compute resources have a root Resource Pool
-  mock.resourcePool = mock_RbVmomi_VIM_ResourcePool({:name => options[:name]})
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ClusterComputeResource
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-def mock_RbVmomi_VIM_ContainerView(options = {})
-  mock = MockContainerView.new()
-  mock.container = options[:container]
-  mock.recursive = options[:recursive]
-  mock.type = options[:type]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-def mock_RbVmomi_VIM_ComputeResource(options = {})
-  options[:name]  = 'Compute' + rand(65536).to_s if options[:name].nil?
-  options[:hosts] = [{}] if options[:hosts].nil?
-
-  mock = MockComputeResource.new()
-
-  mock.name = options[:name]
-  mock.host = []
-
-  # A compute resource must have at least one host.
-  options[:hosts].each do |host_options|
-    mock_host = mock_RbVmomi_VIM_HostSystem(host_options)
-    mock_host.parent = mock
-    mock.host << mock_host
-  end
-
-  mock
-end
-
-# https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-def mock_RbVmomi_VIM_Connection(options = {})
-  options[:serviceInstance] = {} if options[:serviceInstance].nil?
-  options[:serviceContent]  = {} if options[:serviceContent].nil?
-
-  mock = MockRbVmomiVIMConnection.new()
-  mock.serviceContent = mock_RbVmomi_VIM_ServiceContent(options[:serviceContent])
-  options[:serviceInstance][:servicecontent] = mock.serviceContent if options[:serviceInstance][:servicecontent].nil?
-  mock.serviceInstance = mock_RbVmomi_VIM_ServiceInstance(options[:serviceInstance])
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datacenter(options = {})
-  options[:hostfolder_tree] = {} if options[:hostfolder_tree].nil?
-  options[:vmfolder_tree]   = {} if options[:vmfolder_tree].nil?
-  # Currently don't support mocking datastore tree
-  options[:datastores]      = [] if options[:datastores].nil?
-  options[:name]            = 'Datacenter' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockDatacenter.new()
-
-  mock.name = options[:name]
-  mock.hostFolder = mock_RbVmomi_VIM_Folder({ :name => 'hostFolderRoot'})
-  mock.vmFolder = mock_RbVmomi_VIM_Folder({ :name => 'vmFolderRoot'})
-  mock.datastore = []
-
-  # Create vmFolder hierarchy
-  recurse_folder_tree(options[:vmfolder_tree],mock.vmFolder.childEntity)
-
-  # Create hostFolder hierarchy
-  recurse_folder_tree(options[:hostfolder_tree],mock.hostFolder.childEntity)
-
-  # Create mock Datastores
-  options[:datastores].each do |datastorename|
-    mock_ds = mock_RbVmomi_VIM_Datastore({ :name => datastorename })
-    mock.datastore << mock_ds
-  end
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datacenter
-  end
-
-  mock
-end
-
-def recurse_folder_tree(tree, root_object)
-  tree.keys.each do |foldername|
-    folder_options = tree[foldername].nil? ? {} : tree[foldername]
-    folder_options[:name] = foldername if folder_options[:name].nil?
-
-    case folder_options[:object_type]
-    when 'vm'
-      child_object = mock_RbVmomi_VIM_VirtualMachine({ :name => folder_options[:name]})
-    when 'compute_resource'
-      child_object = mock_RbVmomi_VIM_ComputeResource({ :name => folder_options[:name]})
-    when 'cluster_compute_resource'
-      child_object = mock_RbVmomi_VIM_ClusterComputeResource({ :name => folder_options[:name]})
-    when 'resource_pool'
-      child_object =  mock_RbVmomi_VIM_ResourcePool({ :name => folder_options[:name]})
-    else
-      child_object = mock_RbVmomi_VIM_Folder({:name => foldername})
-    end
-
-    # Recursively create children - Default is the child_object is a Folder
-    case folder_options[:object_type]
-    when 'cluster_compute_resource'
-      # Append children into the root Resource Pool for a cluster, instead of directly into the cluster itself.
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool.resourcePool) unless folder_options[:children].nil?
-    when 'resource_pool'
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool) unless folder_options[:children].nil?
-    else
-      recurse_folder_tree(folder_options[:children],child_object.childEntity) unless folder_options[:children].nil?
-    end
-
-    root_object << child_object
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datastore(options = {})
-  options[:name] = 'Datastore' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockDatastore.new()
-
-  mock.name = options[:name]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datastore
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Folder(options = {})
-  options[:name] = 'Folder' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockFolder.new()
-
-  mock.name = options[:name]
-  mock.childEntity = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Folder
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-def mock_RbVmomi_VIM_HostSystem(options = {})
-  options[:memory_size]          = 4294967296 if options[:memory_size].nil? # 4GB RAM
-  options[:num_cpu]              = 1 if options[:num_cpu].nil?
-  options[:num_cores_per_cpu]    = 1 if options[:num_cores_per_cpu].nil?
-  options[:cpu_speed]            = 2048 if options[:cpu_speed].nil? # 2.0 GHz 
-  options[:cpu_model]            = 'Intel(R) Xeon(R) CPU E5-2697 v4 @ 2.0GHz' if options[:cpu_model].nil?
-  options[:maintenance_mode]     = false if options[:maintenance_mode].nil?
-  options[:overall_status]       = 'green' if options[:overall_status].nil?
-  options[:overall_cpu_usage]    = 1 if options[:overall_cpu_usage].nil?
-  options[:overall_memory_usage] = 1 if options[:overall_memory_usage].nil?
-  options[:name]                 = 'HOST' + rand(65536).to_s if options[:name].nil?
-  options[:config_issue]         = [] if options[:config_issue].nil?
-
-  mock = MockHostSystem.new()
-  mock.name = options[:name]
-  mock.summary = MockHostListSummary.new()
-  mock.summary.quickStats = MockHostListSummaryQuickStats.new()
-  mock.summary.hardware = MockHostHardwareSummary.new()
-  mock.hardware = MockHostSystemHostHardwareInfo.new()
-  mock.runtime = MockHostRuntimeInfo.new()
-
-  mock.hardware.cpuPkg = []
-  (1..options[:num_cpu]).each do |cpuid|
-    mockcpu = MockHostCpuPackage.new()
-    mockcpu.hz = options[:cpu_speed] * 1024 * 1024
-    mockcpu.description = options[:cpu_model]
-    mockcpu.index = 0
-    mock.hardware.cpuPkg << mockcpu
-  end
-
-  mock.runtime.inMaintenanceMode = options[:maintenance_mode]
-  mock.overallStatus = options[:overall_status]
-  mock.configIssue = options[:config_issue]
-
-  mock.summary.hardware.memorySize = options[:memory_size]
-  mock.hardware.memorySize = options[:memory_size]
-
-  mock.summary.hardware.cpuMhz = options[:cpu_speed]
-  mock.summary.hardware.numCpuCores = options[:num_cpu] * options[:num_cores_per_cpu]
-  mock.summary.hardware.numCpuPkgs = options[:num_cpu]
-  mock.summary.quickStats.overallCpuUsage = options[:overall_cpu_usage]
-  mock.summary.quickStats.overallMemoryUsage = options[:overall_memory_usage]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-def mock_RbVmomi_VIM_RetrieveResult(options = {})
-  options[:response] = [] if options[:response].nil?
-  mock = MockRetrieveResult.new()
-
-  mock.objects = []
-
-  options[:response].each do |response|
-    mock_objectdata = MockObjectContent.new()
-
-    mock_objectdata.propSet = []
-
-    mock_objectdata.obj = response[:object] 
-
-    # Mock the object properties
-    response.each do |key,value|
-      unless key == :object
-        mock_property = MockDynamicProperty.new()
-        mock_property.name = key
-        mock_property.val = value
-        mock_objectdata.propSet << mock_property
-      end
-    end
-    mock.objects << mock_objectdata
-  end
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-def mock_RbVmomi_VIM_PropertyCollector(options = {})
-  mock = MockPropertyCollector.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-def mock_RbVmomi_VIM_ServiceContent(options = {})
-  options[:propertyCollector] = {} if options[:propertyCollector].nil?
-  options[:datacenters]       = [] if options[:datacenters].nil?
-
-  mock = MockServiceContent.new()
-
-  mock.searchIndex = MockSearchIndex.new()
-  mock.viewManager = MockViewManager.new()
-  mock.virtualDiskManager = MockVirtualDiskManager.new()
-  mock.rootFolder = mock_RbVmomi_VIM_Folder({ :name => 'RootFolder' })
-
-  mock.propertyCollector = mock_RbVmomi_VIM_PropertyCollector(options[:propertyCollector])
-
-  # Create the DCs in this ServiceContent
-  options[:datacenters].each do |dc_options|
-    mock_dc = mock_RbVmomi_VIM_Datacenter(dc_options)
-    mock.rootFolder.childEntity << mock_dc
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-def mock_RbVmomi_VIM_ServiceInstance(options = {})
-  mock = MockServiceInstance.new()
-
-  mock.content = options[:servicecontent]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-def mock_RbVmomi_VIM_Task(options = {})
-  mock = MockTask.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-def mock_RbVmomi_VIM_ResourcePool(options = {})
-  options[:name] = 'ResourcePool' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockResourcePool.new()
-
-  mock.name = options[:name]
-  mock.resourcePool = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ResourcePool
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-def mock_RbVmomi_VIM_VirtualDisk(options = {})
-  options[:controllerKey] = rand(65536) if options[:controllerKey].nil?
-  options[:key]           = rand(65536) if options[:key].nil?
-  options[:label]         = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:unitNumber]    = rand(65536) if options[:unitNumber].nil?
-
-  mock = MockVirtualDisk.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.unitNumber = options[:unitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualDisk
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-def mock_RbVmomi_VIM_VirtualMachine(options = {})
-  options[:snapshot_tree] = nil if options[:snapshot_tree].nil?
-  options[:name] = 'VM' + rand(65536).to_s if options[:name].nil?
-  options[:path] = [] if options[:path].nil?
-
-  mock = MockVirtualMachine.new()
-  mock.config = MockVirtualMachineConfigInfo.new()
-  mock.config.hardware = MockVirtualHardware.new([])
-  mock.summary = MockVirtualMachineSummary.new()
-  mock.summary.runtime = MockVirtualMachineRuntimeInfo.new()
-  mock.summary.guest = MockVirtualMachineGuestSummary.new()
-  mock.runtime = mock.summary.runtime
-
-  mock.name = options[:name]
-  mock.summary.guest.hostName = options[:hostname]
-  mock.runtime.bootTime = options[:boottime]
-  mock.runtime.powerState = options[:powerstate]
-
-  unless options[:snapshot_tree].nil?
-    mock.snapshot = MockVirtualMachineSnapshotInfo.new()
-    mock.snapshot.rootSnapshotList = []
-    index = 0
-
-    # Create a recursive snapshot tree
-    recurse_snapshot_tree(options[:snapshot_tree],mock.snapshot.rootSnapshotList,index)
-  end
-
-  # Create an array of items that describe the path of the VM from the root folder
-  # all the way to the VM itself
-  mock.path = []
-  options[:path].each do |path_item|
-    mock_item = nil
-    case path_item[:type]
-    when 'folder'
-      mock_item = mock_RbVmomi_VIM_Folder({ :name => path_item[:name] })
-    when 'datacenter'
-      mock_item = mock_RbVmomi_VIM_Datacenter({ :name => path_item[:name] })
-    else
-      raise("Unknown mock type #{path_item[:type]} for mock_RbVmomi_VIM_VirtualMachine")
-    end
-    mock.path << [mock_item,path_item[:name]]
-  end
-  mock.path << [mock,options[:name]]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualMachine
-  end
-
-  mock
-end
-
-def recurse_snapshot_tree(tree, root_object, index)
-  tree.keys.each do |snapshotname|
-    snap_options = tree[snapshotname].nil? ? {} : tree[snapshotname]
-    snap = MockVirtualMachineSnapshotTree.new()
-    snap.id = index
-    snap.name = snapshotname
-    snap.childSnapshotList = []
-    snap.description = "Snapshot #{snapshotname}"
-    snap.snapshot = snap_options[:ref] unless snap_options[:ref].nil?
-
-    # Recursively create chilren
-    recurse_snapshot_tree(snap_options[:children],snap.childSnapshotList,index) unless snap_options[:children].nil?
-
-    root_object << snap
-    index += 1
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-def mock_RbVmomi_VIM_VirtualMachineDevice(options = {})
-  mock = MockVirtualDevice.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-def mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo(options = {})
-  options[:key] = rand(65536).to_s if options[:key].nil?
-
-  mock = MockVirtualMachineFileLayoutExFileInfo.new()
-
-  mock.key = options[:key]
-  mock.name = options[:name]
-  mock.size = options[:size]
-  mock.type = options[:type]
-  mock.uniqueSize = options[:uniqueSize]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-def mock_RbVmomi_VIM_VirtualMachineSnapshot(options = {})
-  mock = MockVirtualMachineSnapshot.new()
-  
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-def mock_RbVmomi_VIM_VirtualSCSIController(options = {})
-  options[:controllerKey]      = rand(65536) if options[:controllerKey].nil?
-  options[:key]                = rand(65536) if options[:key].nil?
-  options[:label]              = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:scsiCtlrUnitNumber] = 7 if options[:scsiCtlrUnitNumber].nil?
-
-  mock = MockVirtualSCSIController.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.scsiCtlrUnitNumber = options[:scsiCtlrUnitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualSCSIController
-  end
-
-  mock
-end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index b213bfb..dc94ee7 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -3,12 +3,10 @@ SimpleCov.start do
   add_filter '/spec/'
 end
 require 'helpers'
-require 'rbvmomi_helper'
-require 'rbvmomi'
 require 'rspec'
 require 'vmpooler'
 require 'redis'
-require 'vmpooler/statsd'
+require 'vmpooler/metrics'
 
 def project_root_dir
   File.dirname(File.dirname(__FILE__))
diff --git a/spec/unit/api/helpers_spec.rb b/spec/unit/api/helpers_spec.rb
index e026cf7..5788d5d 100644
--- a/spec/unit/api/helpers_spec.rb
+++ b/spec/unit/api/helpers_spec.rb
@@ -16,12 +16,12 @@ describe Vmpooler::API::Helpers do
 
   describe '#hostname_shorten' do
     [
-        ['example.com', 'not-example.com', 'example.com'],
-        ['example.com', 'example.com', 'example.com'],
-        ['sub.example.com', 'example.com', 'sub'],
-        ['example.com', nil, 'example.com']
-    ].each do |hostname, domain, expected|
-      it { expect(subject.hostname_shorten(hostname, domain)).to eq expected }
+        ['example.com', 'example'],
+        ['sub.example.com', 'sub'],
+        ['adjective-noun.example.com', 'adjective-noun'],
+        ['abc123.example.com', 'abc123']
+    ].each do |hostname, expected|
+      it { expect(subject.hostname_shorten(hostname)).to eq expected }
     end
   end
 
@@ -116,7 +116,7 @@ describe Vmpooler::API::Helpers do
       allow(redis).to receive(:pipelined).with(no_args).and_return [0]
       allow(redis).to receive(:get).and_return 0
 
-      expect(subject.get_queue_metrics([], redis)).to eq({pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
+      expect(subject.get_queue_metrics([], redis)).to eq({requested: 0, pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
     end
 
     it 'adds pool queues correctly' do
@@ -125,10 +125,14 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return 1
+      # Mock returns 7*2 + 2 = 16 results (7 queue types for 2 pools + 2 global counters)
+      # For each pool: [request, processing, odcreate, pending, ready, running, completed]
+      # Plus 2 global counters: clone (1), ondemandclone (0)
+      # Results array: [1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1, 0]
+      #                [req,  proc,  odc,   pend, rdy,  run,  comp, clone, odc]
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 14})
     end
 
     it 'sets booting to 0 when negative calculation' do
@@ -137,10 +141,10 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return 5
+      # Mock returns 7*2 + 2 = 16 results with clone=5 to cause negative booting
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 14})
     end
   end
 
@@ -262,22 +266,86 @@ describe Vmpooler::API::Helpers do
         }
       }
       let(:default_port) { 389 }
-      it 'should attempt ldap authentication' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str)
-
-        subject.authenticate(auth, username_str, password_str)
+      let(:default_encryption) do
+        {
+          :method => :start_tls,
+          :tls_options => { :ssl_version => 'TLSv1' }
+        }
       end
 
-      it 'should return true when authentication is successful' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(true)
+      context 'without a service account' do
+        it 'should attempt ldap authentication' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil)
 
-        expect(subject.authenticate(auth, username_str, password_str)).to be true
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when authentication is successful' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when authentication fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
       end
 
-      it 'should return false when authentication fails' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(false)
+      context 'with a service account' do
+        let(:service_account_hash) do
+          {
+            :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+            :password => 's3cr3t'
+          }
+        end
+        let(:auth) {
+          {
+            'provider' => 'ldap',
+            ldap: {
+              'host' => host,
+              'base' => base,
+              'user_object' => user_object,
+              'service_account_hash' => service_account_hash
+            }
+          }
+        }
+        it 'should attempt ldap authentication' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash)
 
-        expect(subject.authenticate(auth, username_str, password_str)).to be false
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when authentication is successful' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when authentication fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with an alternate ssl_version' do
+        let(:secure_encryption) do
+          {
+            :method => :start_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the alternate ssl_version when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
       end
 
       context 'with an alternate port' do
@@ -287,7 +355,27 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should specify the alternate port when authenticating' do
-          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, user_object, base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, default_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+      end
+
+      context 'with simple_tls and port 636' do
+        let(:secure_port) { 636 }
+        let(:secure_encryption) do
+          {
+            :method => :simple_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['port'] = secure_port
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the secure port and encryption options when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(secure_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
@@ -305,36 +393,165 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should attempt to bind with each base' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should not search the second base when the first binds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(true)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the second base when the first bind fails' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when any bind succeeds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when all bind attempts fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with multiple search user objects' do
+        let(:user_object) {
+          [
+            'uid',
+            'cn'
+          ]
+        }
+        before(:each) do
+          auth[:ldap]['user_object'] = user_object
+        end
+
+        it 'should attempt to bind with each user object' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should not search the second user object when the first binds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the second user object when the first bind fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when any bind succeeds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when all bind attempts fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with multiple search user objects and with multiple search bases' do
+        let(:user_object) {
+          [
+            'uid',
+            'cn'
+          ]
+        }
+        let(:base) {
+          [
+            'ou=user,dc=test,dc=com',
+            'ou=service,ou=user,dc=test,dc=com'
+          ]
+        }
+        before(:each) do
+          auth[:ldap]['base'] = base
+          auth[:ldap]['user_object'] = user_object
+        end
+
+        it 'should attempt to bind with each user object and base' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should not continue searching when the first combination binds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combinations when the first bind fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combinations when the first two binds fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should search the remaining combination when the first three binds fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when any bind succeeds' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when all bind attempts fail' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
@@ -362,16 +579,25 @@ describe Vmpooler::API::Helpers do
     let(:base) { 'ou=users,dc=example,dc=com' }
     let(:username_str) { 'admin' }
     let(:password_str) { 's3cr3t' }
+    let(:encryption) do
+      {
+        :method => :start_tls,
+        :tls_options => { :ssl_version => 'TLSv1' }
+      }
+    end
+    let(:service_account_hash) do
+      {
+        :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+        :password => 's3cr3t'
+      }
+    end
     let(:ldap) { double('ldap') }
     it 'should create a new ldap connection' do
       allow(ldap).to receive(:bind)
       expect(Net::LDAP).to receive(:new).with(
         :host => host,
         :port => port,
-        :encryption => {
-          :method => :start_tls,
-          :tls_options => { :ssl_version => 'TLSv1' }
-        },
+        :encryption => encryption,
         :base => base,
         :auth => {
           :method => :simple,
@@ -380,21 +606,35 @@ describe Vmpooler::API::Helpers do
         }
       ).and_return(ldap)
 
-      subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)
+      subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)
     end
 
     it 'should return true when a bind is successful' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(true)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be true
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be true
     end
 
     it 'should return false when a bind fails' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(false)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be false
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be false
+    end
+
+    it 'should return true when a bind_as is successful' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(true)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be true
+    end
+
+    it 'should return false when a bind_as fails' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(false)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be false
     end
   end
 
diff --git a/spec/unit/api/input_validator_spec.rb b/spec/unit/api/input_validator_spec.rb
new file mode 100644
index 0000000..24982ed
--- /dev/null
+++ b/spec/unit/api/input_validator_spec.rb
@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rack/test'
+require 'vmpooler/api/input_validator'
+
+describe Vmpooler::API::InputValidator do
+  let(:test_class) do
+    Class.new do
+      include Vmpooler::API::InputValidator
+    end
+  end
+  let(:validator) { test_class.new }
+
+  describe '#validate_hostname' do
+    it 'accepts valid hostnames' do
+      expect(validator.validate_hostname('test-host.example.com')).to be true
+      expect(validator.validate_hostname('host123')).to be true
+    end
+
+    it 'rejects invalid hostnames' do
+      result = validator.validate_hostname('invalid_host!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid hostname format')
+    end
+
+    it 'rejects hostnames that are too long' do
+      long_hostname = 'a' * 300
+      result = validator.validate_hostname(long_hostname)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+
+    it 'rejects empty hostnames' do
+      result = validator.validate_hostname('')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('required')
+    end
+  end
+
+  describe '#validate_pool_name' do
+    it 'accepts valid pool names' do
+      expect(validator.validate_pool_name('centos-7-x86_64')).to be true
+      expect(validator.validate_pool_name('ubuntu-2204')).to be true
+    end
+
+    it 'rejects invalid pool names' do
+      result = validator.validate_pool_name('invalid pool!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid pool name format')
+    end
+
+    it 'rejects pool names that are too long' do
+      result = validator.validate_pool_name('a' * 150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_tag' do
+    it 'accepts valid tags' do
+      expect(validator.validate_tag('project', 'test-123')).to be true
+      expect(validator.validate_tag('owner', 'user@example.com')).to be true
+    end
+
+    it 'rejects tags with invalid keys' do
+      result = validator.validate_tag('invalid key!', 'value')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid tag key format')
+    end
+
+    it 'rejects tags with invalid characters in value' do
+      result = validator.validate_tag('key', 'value<script>')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('invalid characters')
+    end
+
+    it 'rejects tags that are too long' do
+      result = validator.validate_tag('key', 'a' * 300)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_vm_count' do
+    it 'accepts valid VM counts' do
+      expect(validator.validate_vm_count(5)).to eq(5)
+      expect(validator.validate_vm_count('10')).to eq(10)
+    end
+
+    it 'rejects counts less than 1' do
+      result = validator.validate_vm_count(0)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at least 1')
+    end
+
+    it 'rejects counts greater than 100' do
+      result = validator.validate_vm_count(150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 100')
+    end
+
+    it 'rejects non-integer values' do
+      result = validator.validate_vm_count('abc')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('valid integer')
+    end
+  end
+
+  describe '#validate_disk_size' do
+    it 'accepts valid disk sizes' do
+      expect(validator.validate_disk_size(50)).to eq(50)
+      expect(validator.validate_disk_size('100')).to eq(100)
+    end
+
+    it 'rejects sizes less than 1' do
+      result = validator.validate_disk_size(0)
+      expect(result['ok']).to be false
+    end
+
+    it 'rejects sizes greater than 2048' do
+      result = validator.validate_disk_size(3000)
+      expect(result['ok']).to be false
+    end
+  end
+
+  describe '#validate_lifetime' do
+    it 'accepts valid lifetimes' do
+      expect(validator.validate_lifetime(24)).to eq(24)
+      expect(validator.validate_lifetime('48')).to eq(48)
+    end
+
+    it 'rejects lifetimes greater than 168 hours (1 week)' do
+      result = validator.validate_lifetime(200)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 168')
+    end
+  end
+
+  describe '#sanitize_json_body' do
+    it 'parses valid JSON' do
+      result = validator.sanitize_json_body('{"key": "value"}')
+      expect(result).to eq('key' => 'value')
+    end
+
+    it 'rejects invalid JSON' do
+      result = validator.sanitize_json_body('{invalid}')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid JSON')
+    end
+
+    it 'rejects non-object JSON' do
+      result = validator.sanitize_json_body('["array"]')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('must be a JSON object')
+    end
+
+    it 'rejects deeply nested JSON' do
+      deep_json = '{"a":{"b":{"c":{"d":{"e":{"f":"too deep"}}}}}}'
+      result = validator.sanitize_json_body(deep_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too complex')
+    end
+
+    it 'rejects bodies that are too large' do
+      large_json = '{"data":"' + ('a' * 20000) + '"}'
+      result = validator.sanitize_json_body(large_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too large')
+    end
+  end
+
+  describe '#validation_error?' do
+    it 'returns true for error responses' do
+      error = { 'ok' => false, 'error' => 'test error' }
+      expect(validator.validation_error?(error)).to be true
+    end
+
+    it 'returns false for successful responses' do
+      expect(validator.validation_error?(true)).to be false
+      expect(validator.validation_error?(5)).to be false
+    end
+  end
+end
diff --git a/spec/unit/collector_middleware_spec.rb b/spec/unit/collector_middleware_spec.rb
new file mode 100644
index 0000000..704ac5a
--- /dev/null
+++ b/spec/unit/collector_middleware_spec.rb
@@ -0,0 +1,191 @@
+
+require 'rack/test'
+require 'vmpooler/metrics/promstats/collector_middleware'
+
+
+describe Vmpooler::Metrics::Promstats::CollectorMiddleware do
+  include Rack::Test::Methods
+
+  # Reset the data store
+  before do
+    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+  end
+
+  let(:registry) do
+    Prometheus::Client::Registry.new
+  end
+
+  let(:original_app) do
+    ->(_) { [200, { 'Content-Type' => 'text/html' }, ['OK']] }
+  end
+
+  let!(:app) do
+    described_class.new(original_app, registry: registry)
+  end
+
+  let(:dummy_error) { RuntimeError.new("Dummy error from tests") }
+
+  it 'returns the app response' do
+    get '/foo'
+
+    expect(last_response).to be_ok
+    expect(last_response.body).to eql('OK')
+  end
+
+  it 'handles errors in the registry gracefully' do
+    counter = registry.get(:http_server_requests_total)
+    expect(counter).to receive(:increment).and_raise(dummy_error)
+
+    get '/foo'
+
+    expect(last_response).to be_ok
+  end
+
+  it 'traces request information' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.2)
+
+    get '/foo'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.25" => 1)
+  end
+
+  it 'normalizes paths containing /vm by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/foo/vm/bar-mumble-flame'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo/vm', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo/vm' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /ondemandvm by ' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/foo/ondemandvm/bar/fatman'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/foo/ondemandvm', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/foo/ondemandvm' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /token by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/token/secret-token-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/token', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/token' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /api/v1/token by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/api/v1/token/secret-token-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/api/v1/token', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/api/v1/token' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /img by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/img/image-name'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/img', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/img' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  it 'normalizes paths containing /lib by default' do
+    expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
+
+    get '/lib/xxxxx.js'
+
+    metric = :http_server_requests_total
+    labels = { method: 'get', path: '/lib', code: '200' }
+    expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+
+    metric = :http_server_request_duration_seconds
+    labels = { method: 'get', path: '/lib' }
+    expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
+  end
+
+  context 'when the app raises an exception' do
+    let(:original_app) do
+      lambda do |env|
+        raise dummy_error if env['PATH_INFO'] == '/broken'
+
+        [200, { 'Content-Type' => 'text/html' }, ['OK']]
+      end
+    end
+
+    before do
+      get '/foo'
+    end
+
+    it 'traces exceptions' do
+      expect { get '/broken' }.to raise_error RuntimeError
+
+      metric = :http_server_exceptions_total
+      labels = { exception: 'RuntimeError' }
+      expect(registry.get(metric).get(labels: labels)).to eql(1.0)
+    end
+  end
+
+  context 'when provided a custom metrics_prefix' do
+    let!(:app) do
+      described_class.new(
+        original_app,
+        registry: registry,
+        metrics_prefix: 'lolrus',
+      )
+    end
+
+    it 'provides alternate metric names' do
+      expect(
+        registry.get(:lolrus_requests_total),
+      ).to be_a(Prometheus::Client::Counter)
+      expect(
+        registry.get(:lolrus_request_duration_seconds),
+      ).to be_a(Prometheus::Client::Histogram)
+      expect(
+        registry.get(:lolrus_exceptions_total),
+      ).to be_a(Prometheus::Client::Counter)
+    end
+
+    it "doesn't register the default metrics" do
+      expect(registry.get(:http_server_requests_total)).to be(nil)
+      expect(registry.get(:http_server_request_duration_seconds)).to be(nil)
+      expect(registry.get(:http_server_exceptions_total)).to be(nil)
+    end
+  end
+end
diff --git a/spec/unit/dns/base_spec.rb b/spec/unit/dns/base_spec.rb
new file mode 100644
index 0000000..9fb656b
--- /dev/null
+++ b/spec/unit/dns/base_spec.rb
@@ -0,0 +1,172 @@
+require 'spec_helper'
+require 'vmpooler/dns/base'
+
+# This spec does not really exercise code paths but is merely used
+# to enforce that certain methods are defined in the base classes
+
+describe 'Vmpooler::PoolManager::Dns::Base' do
+  let(:logger) { MockLogger.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { {} }
+  let(:dns_plugin_name) { 'base' }
+  let(:dns_options) { { 'param' => 'value' } }
+
+  let(:fake_vm) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
+  }
+
+  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
+    metrics: metrics,
+    connpool_type: 'redis_connection_pool',
+    connpool_provider: 'testprovider',
+    size: 1,
+    timeout: 5
+  ) { MockRedis.new }
+  }
+
+  subject { Vmpooler::PoolManager::Dns::Base.new(config, logger, metrics, redis_connection_pool, dns_plugin_name, dns_options) }
+
+  # Helper attr_reader methods
+  describe '#logger' do
+    it 'should come from the provider initialization' do
+      expect(subject.logger).to be(logger)
+    end
+  end
+
+  describe '#metrics' do
+    it 'should come from the provider initialization' do
+      expect(subject.metrics).to be(metrics)
+    end
+  end
+
+  describe '#dns_options' do
+    it 'should come from the provider initialization' do
+      expect(subject.dns_options).to be(dns_options)
+    end
+  end
+
+  describe '#pool_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: '#{poolname}'
+    alias: [ 'mockpool' ]
+    template: 'Templates/pool1'
+    folder: 'Pooler/pool1'
+    datastore: 'datastore0'
+    size: 5
+    timeout: 10
+    ready_ttl: 1440
+    clone_target: 'cluster1'
+EOT
+      )
+    }
+    context 'Given a pool that does not exist' do
+      it 'should return nil' do
+        expect(subject.pool_config('missing_pool')).to be_nil
+      end
+    end
+
+    context 'Given a pool that does exist' do
+      it 'should return the pool\'s configuration' do
+        result = subject.pool_config(poolname)
+        expect(result['name']).to eq(poolname)
+      end
+    end
+  end
+
+  describe '#dns_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :#{dns_plugin_name}:
+    option1: 'value1'
+EOT
+      )
+    }
+
+    context 'Given a dns plugin with no configuration' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :bad_dns:
+    option1: 'value1'
+    option2: 'value1'
+EOT
+      )
+    }
+      it 'should return nil' do
+        expect(subject.dns_config).to be_nil
+      end
+    end
+
+    context 'Given a correct dns config name' do
+      it 'should return the dns\'s configuration' do
+        result = subject.dns_config
+        expect(result['option1']).to eq('value1')
+      end
+    end
+  end
+
+  describe '#global_config' do
+    it 'should come from the dns initialization' do
+      expect(subject.global_config).to be(config)
+    end
+  end
+
+  describe '#name' do
+    it "should come from the dns initialization" do
+      expect(subject.name).to eq(dns_plugin_name)
+    end
+  end
+
+  describe '#get_ip' do
+    it 'calls redis hget with vm name and ip' do
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with("vmpooler__vm__vm1", 'ip')
+      end
+      subject.get_ip(fake_vm['name'])
+    end
+  end
+
+  describe '#provided_pools' do
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: 'pool1'
+    dns_config: 'base'
+  - name: 'pool2'
+    dns_config: 'base'
+  - name: 'otherpool'
+    dns_config: 'other provider'
+  - name: 'no name'
+EOT
+      )
+    }
+
+    it "should return pools serviced by this provider" do
+      expect(subject.provided_pools).to eq(['pool1','pool2'])
+    end
+  end
+
+  describe '#create_or_replace_record' do
+    it 'should raise error' do
+      expect{subject.create_or_replace_record('pool')}.to raise_error(/does not implement create_or_replace_record/)
+    end
+  end
+
+  describe '#delete_record' do
+    it 'should raise error' do
+      expect{subject.delete_record('pool')}.to raise_error(/does not implement delete_record/)
+    end
+  end
+end
diff --git a/spec/unit/dns_spec.rb b/spec/unit/dns_spec.rb
new file mode 100644
index 0000000..3e02704
--- /dev/null
+++ b/spec/unit/dns_spec.rb
@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe 'Vmpooler::Dns' do
+  let(:dns_class) { 'mock-dnsservice' }
+  let(:dns_config_name) { 'mock' }
+  let(:pool) { 'pool1' }
+  let(:config) { YAML.load(<<~EOT
+  ---
+  :dns_configs:
+    :mock:
+      dns_class: 'mock'
+      domain: 'example.com'
+  :pools:
+    - name: 'pool1'
+      dns_plugin: 'mock'
+EOT
+  )}
+  subject { Vmpooler::Dns.new }
+
+  describe '.get_dns_plugin_class_by_name' do
+    it 'returns the plugin class for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_class_by_name(config, dns_config_name)
+      expect(result).to eq('mock')
+    end
+  end
+
+  describe '.get_domain_for_pool' do
+    it 'returns the domain for the specified pool' do
+      result = Vmpooler::Dns.get_domain_for_pool(config, pool)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_domain_by_name' do
+    it 'returns the domain for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_domain_by_name(config, dns_config_name)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_config_classes' do
+    it 'returns the list of dns plugin classes' do
+      result = Vmpooler::Dns.get_dns_plugin_config_classes(config)
+      expect(result).to eq(['mock'])
+    end
+  end
+
+  describe '#load_from_gems' do
+    let(:gem_name) { 'mock-dnsservice' }
+    let(:translated_gem_name) { 'mock/dnsservice' }
+
+    before(:each) do
+      allow(subject).to receive(:require).with(gem_name).and_return(true)
+    end
+
+    it 'loads the specified gem' do
+      expect(subject).to receive(:require).with("vmpooler/dns/#{translated_gem_name}")
+      result = subject.load_from_gems(gem_name)
+      expect(result).to eq("vmpooler/dns/#{translated_gem_name}")
+    end
+  end
+end
diff --git a/spec/unit/env_config.rb b/spec/unit/env_config.rb
index 1dd1bdf..1529cf7 100644
--- a/spec/unit/env_config.rb
+++ b/spec/unit/env_config.rb
@@ -22,14 +22,15 @@ describe 'Vmpooler' do
         ['prefix', test_string, ""],
         ['logfile', test_string, nil],
         ['site_name', test_string, nil],
-        ['domain', test_string, nil],
         ['clone_target', test_string, nil],
         ['create_folders', test_bool, nil],
         ['create_template_delta_disks', test_bool, nil],
         ['create_linked_clones', test_bool, nil],
         ['experimental_features', test_bool, nil],
-        ['purge_unconfigured_folders', test_bool, nil],
+        ['purge_unconfigured_resources', test_bool, nil],
         ['usage_stats', test_bool, nil],
+        ['request_logger', test_bool, nil],
+        ['extra_config', test_string, nil],
       ]
 
       test_cases.each do |key, value, default|
diff --git a/spec/unit/generic_connection_pool_spec.rb b/spec/unit/generic_connection_pool_spec.rb
index ff57472..4777795 100644
--- a/spec/unit/generic_connection_pool_spec.rb
+++ b/spec/unit/generic_connection_pool_spec.rb
@@ -1,16 +1,18 @@
 require 'spec_helper'
 
 describe 'GenericConnectionPool' do
-  let(:metrics) { Vmpooler::DummyStatsd.new }
-  let(:metric_prefix) { 'prefix' }
-  let(:default_metric_prefix) { 'connectionpool' }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:connpool_type) { 'test_connection_pool' }
+  let(:connpool_provider) { 'testprovider' }
+  let(:default_connpool_type) { 'connectionpool' }
   let(:connection_object) { double('connection') }
   let(:pool_size) { 1 }
   let(:pool_timeout) { 1 }
 
   subject { Vmpooler::PoolManager::GenericConnectionPool.new(
               metrics: metrics,
-              metric_prefix: metric_prefix,
+              connpool_type: connpool_type,
+              connpool_provider: connpool_provider,
               size: pool_size,
               timeout: pool_timeout
             ) { connection_object }
@@ -23,6 +25,13 @@ describe 'GenericConnectionPool' do
       connection: 'connection'
     }}
 
+    it 'should error out when connection pool could not establish no nothing' do
+      newsub = Vmpooler.redis_connection_pool("foo,bar", "1234", "fuba", 1, 1, metrics, 0)
+      expect { newsub.with_metrics do |conn_pool_object|
+        conn_pool_object.srem('foo', "bar")
+      end }.to raise_error Redis::CannotConnectError
+    end
+
     it 'should return a connection object when grabbing one from the pool' do
       subject.with_metrics do |conn_pool_object|
         expect(conn_pool_object).to be(connection_object)
@@ -65,7 +74,7 @@ describe 'GenericConnectionPool' do
 
     context 'When metrics are configured' do
       it 'should emit a gauge metric when the connection is grabbed and released' do
-        expect(metrics).to receive(:gauge).with(/\.available/,Integer).exactly(2).times
+        expect(metrics).to receive(:gauge).with(/connection_available/,Integer).exactly(2).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -73,7 +82,7 @@ describe 'GenericConnectionPool' do
       end
 
       it 'should emit a timing metric when the connection is grabbed' do
-        expect(metrics).to receive(:timing).with(/\.waited/,Integer).exactly(1).times
+        expect(metrics).to receive(:timing).with(/connection_waited/,Integer).exactly(1).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -81,8 +90,8 @@ describe 'GenericConnectionPool' do
       end
 
       it 'should emit metrics with the specified prefix' do
-        expect(metrics).to receive(:gauge).with(/#{metric_prefix}\./,Integer).at_least(1).times
-        expect(metrics).to receive(:timing).with(/#{metric_prefix}\./,Integer).at_least(1).times
+        expect(metrics).to receive(:gauge).with(/#{connpool_type}\./,Integer).at_least(1).times
+        expect(metrics).to receive(:timing).with(/#{connpool_type}\./,Integer).at_least(1).times
 
         subject.with_metrics do |conn1|
           # do nothing
@@ -90,11 +99,11 @@ describe 'GenericConnectionPool' do
       end
 
       context 'Metrix prefix is missing' do
-        let(:metric_prefix) { nil }
+        let(:connpool_type) { nil }
 
         it 'should emit metrics with default prefix' do
-          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
-          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:gauge).with(/#{default_connpool_type}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_connpool_type}\./,Integer).at_least(1).times
 
           subject.with_metrics do |conn1|
             # do nothing
@@ -103,11 +112,11 @@ describe 'GenericConnectionPool' do
       end
 
       context 'Metrix prefix is empty' do
-        let(:metric_prefix) { '' }
+        let(:connpool_type) { '' }
 
         it 'should emit metrics with default prefix' do
-          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
-          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:gauge).with(/#{default_connpool_type}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_connpool_type}\./,Integer).at_least(1).times
 
           subject.with_metrics do |conn1|
             # do nothing
diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb
index 3b41e62..1b2ccef 100644
--- a/spec/unit/pool_manager_spec.rb
+++ b/spec/unit/pool_manager_spec.rb
@@ -8,33 +8,48 @@ require 'mock_redis'
 RSpec::Matchers.define :a_pool_with_name_of do |value|
   match { |actual| actual['name'] == value }
 end
-
 describe 'Pool Manager' do
   let(:logger) { MockLogger.new }
-  let(:redis) { MockRedis.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
   let(:pool) { 'pool1' }
   let(:vm) { 'vm1' }
   let(:timeout) { 5 }
+  let(:timeout_notification) { 2 }
   let(:host) { double('host') }
-  let(:token) { 'token1234'}
+  let(:token) { 'token1234' }
+  let(:request_id) { '1234' }
+  let(:current_time) { Time.now }
 
   let(:provider_options) { {} }
-  let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, 'mock_provider', provider_options) }
+  let(:dns_options) { {} }
+  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
+    metrics: metrics,
+    connpool_type: 'redis_connection_pool',
+    connpool_provider: 'testprovider',
+    size: 1,
+    timeout: 5
+  ) { MockRedis.new }
+  }
 
+  let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'mock_provider', provider_options) }
+  let(:dns_plugin) { MockPoolManagerDnsBase.new }
   let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
+:dns_configs: {}
 :providers:
   :mock:
+    dns_class: base
+:redis: {}
 :pools:
   - name: '#{pool}'
     size: 1
+    dns_plugin: 'mock'
 EOT
     )
   }
 
-  subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
+  subject { Vmpooler::PoolManager.new(config, logger, redis_connection_pool, metrics) }
 
   describe '#config' do
     before do
@@ -60,14 +75,112 @@ EOT
     )
     }
     it do
-      files = ["#{project_root_dir}/lib/vmpooler/providers/vsphere.rb",
-               "#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
+      files = ["#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
       expect(subject.load_used_providers).to eq(files)
     end
   end
 
+  describe '#load_used_dns_plugins' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :base:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+    )
+    }
+    it do
+      files = ['vmpooler/dns/base']
+      expect(subject.load_used_dns_plugins).to eq(files)
+    end
+
+
+  end
+
+  describe '#used_providers' do
+    context 'with no named providers' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+    context 'with one named provider without a provider_class key' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+
+    context 'with one named provider with a provider_class key' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+    provider_class: 'mock_mock'
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock_mock", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+
+    context 'with one named provider with a provider_class key and one without' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+    provider_class: 'mock_mock'
+  :foo:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock_mock", "foo", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+  end
+
   it '#default_providers' do
-    expect(subject.default_providers).to eq(['vsphere', 'dummy'])
+    expect(subject.default_providers).to eq(['dummy'])
   end
 
   describe '#check_pending_vm' do
@@ -77,9 +190,9 @@ EOT
 
     it 'calls _check_pending_vm' do
       expect(Thread).to receive(:new).and_yield
-      expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,provider)
+      expect(subject).to receive(:_check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)
 
-      subject.check_pending_vm(vm, pool, timeout, provider)
+      subject.check_pending_vm(vm, pool, timeout, timeout_notification, provider)
     end
   end
 
@@ -91,17 +204,21 @@ EOT
     context 'host is in pool' do
 
       it 'calls move_pending_vm_to_ready if host is ready' do
-        expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(true)
-        expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool)
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(true)
+          expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool, redis, nil)
+        end
 
-        subject._check_pending_vm(vm, pool, timeout, provider)
+        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       end
 
       it 'calls fail_pending_vm if host is not ready' do
-        expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(false)
-        expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout)
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(false)
+          expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, timeout_notification, redis)
+        end
 
-        subject._check_pending_vm(vm, pool, timeout, provider)
+        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       end
     end
 
@@ -114,28 +231,33 @@ EOT
       it 'should return' do
         expect(subject).to receive(:vm_mutex).and_return(mutex)
 
-        expect(subject._check_pending_vm(vm, pool, timeout, provider)).to be_nil
+        expect(subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)).to be_nil
       end
     end
   end
 
   describe '#remove_nonexistent_vm' do
     before do
+      expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      expect(dns_plugin).to receive(:delete_record)
       expect(subject).not_to be_nil
     end
 
     it 'removes VM from pending in redis' do
-      create_pending_vm(pool,vm)
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool,vm,redis)
 
-      expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
-      subject.remove_nonexistent_vm(vm, pool)
-      expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+        subject.remove_nonexistent_vm(vm, pool, redis)
+        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+      end
     end
 
     it 'logs msg' do
       expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
 
-      subject.remove_nonexistent_vm(vm, pool)
+      redis_connection_pool.with do |redis|
+        subject.remove_nonexistent_vm(vm, pool, redis)
+      end
     end
   end
 
@@ -145,49 +267,198 @@ EOT
     end
 
     before(:each) do
-      create_pending_vm(pool,vm)
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool,vm,redis)
+        config[:config]['vm_checktime'] = 15
+      end
     end
 
     it 'takes no action if VM is not cloning' do
-      expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true)
-      expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
+      redis_connection_pool.with do |redis|
+        expect(logger).to_not receive(:log)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
+      end
     end
 
     it 'takes no action if VM is within timeout' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
-      expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true)
-      expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
+        expect(logger).to_not receive(:log)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
+        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
+      end
     end
 
     it 'moves VM to completed queue if VM has exceeded timeout and exists' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
-      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true)
-      expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
-      expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
+        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+        expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+      end
     end
 
     it 'logs message if VM has exceeded timeout and exists' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
-      expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
+        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: ")
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
+      end
     end
 
     it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
-      expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool)
-      expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(true)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
+        expect(logger).to_not receive(:log)
+        expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool, redis)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: false)).to eq(true)
+      end
     end
 
     it 'swallows error if an error is raised' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
-      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(false)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(false)
+      end
     end
 
     it 'logs message if an error is raised' do
-      redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
-      expect(logger).to receive(:log).with('d', String)
+      redis_connection_pool.with do |redis|
+        redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
+        expect(logger).to receive(:log).with('d', String)
 
-      subject.fail_pending_vm(vm, pool, timeout,true)
+        subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
+      end
+    end
+
+    context 'with request_id' do
+
+      it 'creates a new odcreate task' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'clone',(Time.now - 900).to_s)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
+          subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to eq(["#{pool}:#{pool}:1:#{request_id}"])
+        end
+      end
+    end
+  end
+
+  describe '#handle_timed_out_vm' do
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    before(:each) do
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool, vm, redis)
+        config[:config]['max_vm_retries'] = 3
+      end
+    end
+
+    context 'without request_id' do
+      it 'moves VM to completed queue and returns error' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'open_socket_error', 'connection failed')
+          result = subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+          expect(result).to eq('connection failed')
+        end
+      end
+    end
+
+    context 'with request_id and transient error' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'network timeout')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'Timeout::Error')
+        end
+      end
+
+      it 'retries on first failure' do
+        redis_connection_pool.with do |redis|
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count')).to eq('1')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+
+      it 'marks as failed after max retries' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__odrequest__#{request_id}", 'retry_count', '3')
+          
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to eq('Max retry attempts exceeded')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+    end
+
+    context 'with request_id and permanent error' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'template not found')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'RuntimeError')
+        end
+      end
+
+      it 'immediately marks as failed without retrying' do
+        redis_connection_pool.with do |redis|
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to include('Configuration error')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+    end
+  end
+
+  describe '#permanent_error?' do
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    it 'identifies template not found errors as permanent' do
+      expect(subject.permanent_error?('template not found', 'RuntimeError')).to be(true)
+    end
+
+    it 'identifies invalid path errors as permanent' do
+      expect(subject.permanent_error?('invalid path specified', 'ArgumentError')).to be(true)
+    end
+
+    it 'identifies permission denied errors as permanent' do
+      expect(subject.permanent_error?('permission denied', 'SecurityError')).to be(true)
+    end
+
+    it 'identifies ArgumentError class as permanent' do
+      expect(subject.permanent_error?('some argument error', 'ArgumentError')).to be(true)
+    end
+
+    it 'identifies network errors as transient' do
+      expect(subject.permanent_error?('connection timeout', 'Timeout::Error')).to be(false)
+    end
+
+    it 'identifies socket errors as transient' do
+      expect(subject.permanent_error?('connection refused', 'Errno::ECONNREFUSED')).to be(false)
+    end
+
+    it 'returns false for nil inputs' do
+      expect(subject.permanent_error?(nil, nil)).to be(false)
+      expect(subject.permanent_error?('error', nil)).to be(false)
+      expect(subject.permanent_error?(nil, 'Error')).to be(false)
     end
   end
 
@@ -199,61 +470,174 @@ EOT
     end
 
     before(:each) do
-      create_pending_vm(pool,vm)
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool,vm,redis)
+      end
     end
 
     context 'when hostname matches VM name' do
       it 'should move the VM from pending to ready pool' do
-        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
-        expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
-        subject.move_pending_vm_to_ready(vm, pool)
-        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
-        expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
+        redis_connection_pool.with do |redis|
+          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
+          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
+        end
       end
 
       it 'should log a message' do
         expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
 
-        subject.move_pending_vm_to_ready(vm, pool)
+        redis_connection_pool.with do |redis|
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+        end
       end
 
       it 'should receive time_to_ready_state metric' do
-        redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
-        expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/)
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
+          expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/)
 
-        subject.move_pending_vm_to_ready(vm, pool)
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+        end
       end
 
 
       it 'should set the boot time in redis' do
-        redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
-        expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
-        subject.move_pending_vm_to_ready(vm, pool)
-        expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil
-        # TODO Should we inspect the value to see if it's valid?
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
+          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil
+          # TODO Should we inspect the value to see if it's valid?
+        end
       end
 
       it 'should not determine boot timespan if clone start time not set' do
-        expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
-        subject.move_pending_vm_to_ready(vm, pool)
-        expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to eq("") # Possible implementation bug here. Should still be nil here
+        redis_connection_pool.with do |redis|
+          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to match(/\d\.\d{2}/)
+        end
       end
 
       it 'should raise error if clone start time is not parsable' do
-        redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
-        expect{subject.move_pending_vm_to_ready(vm, pool)}.to raise_error(/iamnotparsable_asdate/)
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
+          expect{subject.move_pending_vm_to_ready(vm, pool, redis)}.to raise_error(/iamnotparsable_asdate/)
+        end
       end
 
       it 'should save the last boot time' do
-        expect(redis.hget('vmpooler__lastboot', pool)).to be(nil)
-        subject.move_pending_vm_to_ready(vm, pool)
-        expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil)
+        redis_connection_pool.with do |redis|
+          expect(redis.hget('vmpooler__lastboot', pool)).to be(nil)
+          subject.move_pending_vm_to_ready(vm, pool, redis)
+          expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil)
+        end
+      end
+    end
+
+    context 'with request_id' do
+      context 'with a pending request' do
+        before(:each) do
+          allow(subject).to receive(:check_ondemand_request_ready)
+          config[:config]['ondemand_request_ttl'] = 20
+        end
+
+        it 'sets the vm as active' do
+          redis_connection_pool.with do |redis|
+            expect(Time).to receive(:now).and_return(current_time).at_least(:once)
+            redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+            expect(redis.hget("vmpooler__active__#{pool}", vm)).to eq(current_time.to_s)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to eq(current_time.to_s)
+            expect(redis.sismember("vmpooler__#{request_id}__#{pool}__#{pool}", vm)).to be true
+          end
+        end
+
+        it 'logs that the vm is ready for the request' do
+          redis_connection_pool.with do |redis|
+            redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+            expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'")
+
+            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+          end
+        end
+      end
+
+      context 'when the request has been marked as failed' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+          end
+        end
+
+        it 'moves the vm to completed' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
+            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+          end
+        end
+
+        it 'returns nil' do
+          redis_connection_pool.with do |redis|
+            result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+            expect(result).to be nil
+          end
+        end
+      end
+
+      context 'when the request has been marked as deleted' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+          end
+        end
+
+        it 'moves the vm to completed' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
+            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+          end
+        end
+
+        it 'returns nil' do
+          redis_connection_pool.with do |redis|
+            result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+            expect(result).to be nil
+          end
+        end
+      end
+
+      context 'with auth on the request' do
+        let(:user) { 'vmpuser' }
+        let(:platform_alias) { pool }
+        let(:platforms_string) { "#{platform_alias}:#{pool}:1" }
+        let(:score) { current_time.to_i }
+        before(:each) do
+          config[:config]['ondemand_request_ttl'] = 20
+          allow(subject).to receive(:check_ondemand_request_ready)
+          redis_connection_pool.with do |redis|
+            create_ondemand_request_for_test(request_id, score, platforms_string, redis, user, token)
+          end
+        end
+
+        it 'should specify auth data on the vm' do
+          redis_connection_pool.with do |redis|
+            allow(redis).to receive(:hset)
+
+            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'token:token')).to eq(token)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'token:user')).to eq(user)
+          end
+        end
       end
     end
   end
 
   describe '#check_ready_vm' do
-    let(:ttl) { 0 }
+    let(:ttl) { 5 }
     let(:poolconfig) { config[:pools][0] }
 
     before do
@@ -269,7 +653,7 @@ EOT
   end
 
   describe '#_check_ready_vm' do
-    let(:ttl) { 0 }
+    let(:ttl) { 5 }
     let(:host) { {} }
     let(:config) { YAML.load(<<-EOT
 ---
@@ -286,7 +670,9 @@ EOT
     }
 
     before(:each) do
-      create_ready_vm(pool,vm)
+      redis_connection_pool.with do |redis|
+        create_ready_vm(pool,vm,redis)
+      end
       config[:config]['vm_checktime'] = 15
 
       # Create a VM which is powered on
@@ -297,41 +683,43 @@ EOT
 
     context 'a VM that does not need to be checked' do
       it 'should do nothing' do
-        check_stamp = (Time.now - 60).to_s
-        redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp)
-        expect(provider).to receive(:get_vm).exactly(0).times
-        subject._check_ready_vm(vm, pool, ttl, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp)
+        redis_connection_pool.with do |redis|
+          check_stamp = (Time.now - 60).to_s
+          redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp)
+          expect(provider).to receive(:get_vm).exactly(0).times
+          subject._check_ready_vm(vm, pool, ttl, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp)
+        end
       end
     end
 
     context 'a VM that has never been checked' do
-      let(:last_check_date) { Date.new(2001,1,1).to_s }
-
       it 'should set the current check timestamp' do
-        expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
-        subject._check_ready_vm(vm, pool, ttl, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
+          subject._check_ready_vm(vm, pool, ttl, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil
+        end
       end
     end
 
     context 'a VM that needs to be checked' do
-      let(:last_check_date) { Date.new(2001,1,1).to_s }
+      let(:last_check_date) { Time.now - 901 }
       before(:each) do
-        redis.hset("vmpooler__vm__#{vm}", 'check',last_check_date)
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'check',last_check_date)
+        end
       end
 
       it 'should set the current check timestamp' do
-        expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date)
-        subject._check_ready_vm(vm, pool, ttl, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date)
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date.to_s)
+          subject._check_ready_vm(vm, pool, ttl, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date.to_s)
+        end
       end
 
       context 'and is ready' do
-        before(:each) do
-          expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(true)
-        end
-
         it 'should only set the next check interval' do
           subject._check_ready_vm(vm, pool, ttl, provider)
         end
@@ -339,25 +727,29 @@ EOT
 
       context 'has correct name and is not ready' do
         before(:each) do
-          expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(false)
+          redis_connection_pool.with do |redis|
+            redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now - 200)
+            redis.sadd("vmpooler__ready__#{pool}", vm)
+          end
         end
 
         it 'should move the VM to the completed queue' do
-          expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
+          redis_connection_pool.with do |redis|
+            expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
 
-          subject._check_ready_vm(vm, pool, ttl, provider)
+            subject._check_ready_vm(vm, pool, ttl, provider)
+          end
         end
 
         it 'should move the VM to the completed queue in Redis' do
-          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
-          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
-          subject._check_ready_vm(vm, pool, ttl, provider)
-          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
-          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+          redis_connection_pool.with do |redis|
+            subject._check_ready_vm(vm, pool, ttl, provider)
+            expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+          end
         end
 
         it 'should log messages about being unreachable' do
-          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
+          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' removed from 'ready' queue. vm unreachable with error: ")
 
           subject._check_ready_vm(vm, pool, ttl, provider)
         end
@@ -367,7 +759,9 @@ EOT
 
         context 'when less than 60 seconds since a VM moved to ready' do
           before(:each) do
-            redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
+            redis_connection_pool.with do |redis|
+              redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
+            end
           end
 
           it 'should return nil' do
@@ -377,29 +771,19 @@ EOT
 
         context 'with a hostname mismatch' do
           let(:different_hostname) { 'different_name' }
+          let(:longer_ttl) { 20 }
           before(:each) do
-            expect(provider).to receive(:get_vm).with(pool,vm).and_return(host)
             host['hostname'] = different_hostname
+            redis_connection_pool.with do |redis|
+              expect(subject).to receive(:mismatched_hostname?).with(vm, pool, provider, redis).and_return(true)
+              redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now - 300)
+            end
           end
 
-          it 'should move the VM to the completed queue' do
-            expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
+          it 'should not run vm_still_ready?' do
+            expect(subject).to_not receive(:vm_still_ready?)
 
-            subject._check_ready_vm(vm, pool, ttl, provider)
-          end
-
-          it 'should move the VM to the completed queue in Redis' do
-            expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
-            expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
-            subject._check_ready_vm(vm, pool, ttl, provider)
-            expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
-            expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
-          end
-
-          it 'should log messages about being misnamed' do
-            expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' has mismatched hostname #{different_hostname}, removed from 'ready' queue")
-
-            subject._check_ready_vm(vm, pool, ttl, provider)
+            subject._check_ready_vm(vm, pool, longer_ttl, provider)
           end
         end
       end
@@ -467,7 +851,9 @@ EOT
     end
 
     before(:each) do
-      create_running_vm(pool,vm)
+      redis_connection_pool.with do |redis|
+        create_running_vm(pool,vm, redis)
+      end
 
       # Create a VM which is powered on
       host['hostname'] = vm
@@ -476,36 +862,71 @@ EOT
     end
 
     it 'moves a missing VM to the completed queue' do
-      expect(provider).to receive(:vm_ready?).and_return(false)
-      expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
-      expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
-      subject._check_running_vm(vm, pool, timeout, provider)
-      expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
+      redis_connection_pool.with do |redis|
+        expect(provider).to receive(:vm_ready?).and_return(false)
+        expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
+        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+        subject._check_running_vm(vm, pool, timeout, provider)
+        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
+      end
     end
 
     context 'valid host' do
-      it 'should not move VM if it has no checkout time' do
-        expect(provider).to receive(:vm_ready?).and_return(true)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
-        subject._check_running_vm(vm, pool, 0, provider)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+      it 'should kill a VM if it has no checkout time' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+          subject._check_running_vm(vm, pool, 0, provider)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
+        end
+      end
+
+      it 'should log a message when the machine is removed due to no active data' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' is listed as running, but has no checkouttime data. Removing from running")
+          subject._check_running_vm(vm, pool, 0, provider)
+        end
       end
 
       it 'should not move VM if TTL is zero' do
-        expect(provider).to receive(:vm_ready?).and_return(true)
-        redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
-        subject._check_running_vm(vm, pool, 0, provider)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+          subject._check_running_vm(vm, pool, 0, provider)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+        end
       end
 
       it 'should move VM when past TTL' do
-        redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
-        expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
-        subject._check_running_vm(vm, pool, timeout, provider)
-        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
-        expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
+          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
+          subject._check_running_vm(vm, pool, timeout, provider)
+          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
+          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+        end
+      end
+
+      it 'should try to tag if it has not been done' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          expect(provider).to receive(:tag_vm_user)
+          subject._check_running_vm(vm, pool, 0, provider)
+        end
+      end
+
+      it 'should not try to tag if it has been done' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true')
+          expect(provider).not_to receive(:tag_vm_user)
+          subject._check_running_vm(vm, pool, 0, provider)
+        end
       end
     end
 
@@ -533,31 +954,43 @@ EOT
     end
 
     before(:each) do
-      create_pending_vm(pool, vm, token)
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool, vm, redis, token)
+      end
     end
 
     it 'VM should be in the "from queue" before the move' do
-      expect(redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
+      redis_connection_pool.with do |redis|
+        expect(redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
+      end
     end
 
     it 'VM should not be in the "from queue" after the move' do
-      subject.move_vm_queue(pool, vm, queue_from, queue_to, message)
-      expect(!redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
+      redis_connection_pool.with do |redis|
+        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
+        expect(!redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
+      end
     end
 
     it 'VM should not be in the "to queue" before the move' do
-      expect(!redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
+      redis_connection_pool.with do |redis|
+        expect(!redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
+      end
     end
 
     it 'VM should be in the "to queue" after the move' do
-      subject.move_vm_queue(pool, vm, queue_from, queue_to, message)
-      expect(redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
+      redis_connection_pool.with do |redis|
+        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
+        expect(redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
+      end
     end
 
     it 'should log a message' do
       allow(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' #{message}")
 
-      subject.move_vm_queue(pool, vm, queue_from, queue_to, message)
+      redis_connection_pool.with do |redis|
+        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
+      end
     end
   end
 
@@ -570,17 +1003,78 @@ EOT
     end
 
     it 'calls _clone_vm' do
-      expect(subject).to receive(:_clone_vm).with(pool_object,provider)
+      expect(subject).to receive(:_clone_vm).with(pool_object,provider,dns_plugin,nil,nil)
 
-      subject.clone_vm(pool_object,provider)
+      subject.clone_vm(pool_object,provider,dns_plugin)
     end
 
     it 'logs a message if an error is raised' do
       allow(logger).to receive(:log)
       expect(logger).to receive(:log).with('s',"[!] [#{pool}] failed while cloning VM with an error: MockError")
-      expect(subject).to receive(:_clone_vm).with(pool,provider).and_raise('MockError')
+      expect(subject).to receive(:_clone_vm).with(pool,provider,dns_plugin,nil,nil).and_raise('MockError')
 
-      expect{subject.clone_vm(pool,provider)}.to raise_error(/MockError/)
+      expect{subject.clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
+    end
+  end
+
+  describe '#find_unique_hostname' do
+    # it should return the hostname
+    before do
+      allow(subject).to receive(:generate_and_check_hostname).and_return(["spicy-proton", true])
+      allow(subject).to receive(:get_provider_for_pool).and_return(provider)
+    end
+
+    context 'with a setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:providers:
+  :mock_provider:
+    skip_dns_check_before_creating_vm: true
+:pools:
+  - name: '#{pool}'
+    size: 1
+    dns_config: 'mock'
+        EOT
+        )
+      }
+      it 'should skip the dns check' do
+        #method is skipped
+        expect(subject).not_to receive(:check_dns_available)
+        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+    end
+    context 'without the setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:providers:
+  :mock_provider:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    dns_config: 'mock'
+        EOT
+        )
+      }
+      it 'should run the dns check and pass' do
+        #method is skipped
+        expect(subject).to receive(:check_dns_available).and_return(["1.1.1.1",true])
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+
+      it 'should run the dns check and fail' do
+        #method is skipped
+        allow(subject).to receive(:check_dns_available).and_return(["1.1.1.1",false]).exactly(3).times
+        expect{subject.find_unique_hostname(pool)}.to raise_error(RuntimeError)
+      end
     end
   end
 
@@ -595,88 +1089,186 @@ EOT
     let(:config) {
       YAML.load(<<-EOT
 ---
+:dns_configs:
+  :mock:
+    dns_class: base
 :config:
   prefix: "prefix"
 :redis:
   ttl: #{redis_ttl}
+:pools:
+  - name: 'pool1'
+    size: 1
+    dns_config: 'mock'
 EOT
       )
     }
 
     context 'with no errors during cloning' do
       before(:each) do
-        expect(metrics).to receive(:timing).with(/clone\./,/0/)
+        allow(metrics).to receive(:timing)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
         expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).and_return(1)
+        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
         allow(logger).to receive(:log)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
+        end
       end
 
       it 'should create a cloning VM' do
-        expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
+        redis_connection_pool.with do |redis|
+          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
 
-        subject._clone_vm(pool,provider)
+          subject._clone_vm(pool,provider,dns_plugin)
 
-        expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1)
-        # Get the new VM Name from the pending pool queue as it should be the only entry
-        vm_name = redis.smembers("vmpooler__pending__#{pool}")[0]
-        expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil
-        expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(pool)
-        expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm_name}")).to_not be_nil
-        expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil
+          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil
+          expect(redis.hget("vmpooler__vm__#{vm}", 'template')).to eq(pool)
+          expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm}")).to_not be_nil
+          expect(redis.hget("vmpooler__vm__#{vm}", 'clone_time')).to_not be_nil
+        end
       end
 
       it 'should decrement the clone tasks counter' do
-        redis.incr('vmpooler__tasks__clone')
-        redis.incr('vmpooler__tasks__clone')
-        expect(redis.get('vmpooler__tasks__clone')).to eq('2')
-        subject._clone_vm(pool,provider)
-        expect(redis.get('vmpooler__tasks__clone')).to eq('1')
+        redis_connection_pool.with do |redis|
+          redis.incr('vmpooler__tasks__clone')
+          redis.incr('vmpooler__tasks__clone')
+          expect(redis.get('vmpooler__tasks__clone')).to eq('2')
+          subject._clone_vm(pool,provider,dns_plugin)
+          expect(redis.get('vmpooler__tasks__clone')).to eq('1')
+        end
       end
 
       it 'should log a message that is being cloned from a template' do
         expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] Starting to clone '(.+)'/)
 
-        subject._clone_vm(pool,provider)
+        subject._clone_vm(pool,provider,dns_plugin)
       end
 
       it 'should log a message that it completed being cloned' do
         expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned in [0-9.]+ seconds/)
 
-        subject._clone_vm(pool,provider)
+        subject._clone_vm(pool,provider,dns_plugin)
+      end
+    end
+
+    context 'with a failure to get ip address after cloning' do
+      it 'should log a message that it completed being cloned' do
+        allow(metrics).to receive(:timing)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
+        expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).and_return(nil)
+
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(StandardError)
       end
     end
 
     context 'with an error during cloning' do
       before(:each) do
         expect(provider).to receive(:create_vm).with(pool, String).and_raise('MockError')
+        allow(subject).to receive(:get_domain_for_pool)
         allow(logger).to receive(:log)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
+        end
       end
 
       it 'should not create a cloning VM' do
-        expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
+        redis_connection_pool.with do |redis|
+          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
 
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
 
-        expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
-        # Get the new VM Name from the pending pool queue as it should be the only entry
-        vm_name = redis.smembers("vmpooler__pending__#{pool}")[0]
-        expect(vm_name).to be_nil
+          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
+          # Get the new VM Name from the pending pool queue as it should be the only entry
+          vm_name = redis.smembers("vmpooler__pending__#{pool}")[0]
+          expect(vm_name).to be_nil
+        end
       end
 
       it 'should decrement the clone tasks counter' do
-        redis.incr('vmpooler__tasks__clone')
-        redis.incr('vmpooler__tasks__clone')
-        expect(redis.get('vmpooler__tasks__clone')).to eq('2')
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
-        expect(redis.get('vmpooler__tasks__clone')).to eq('1')
+        redis_connection_pool.with do |redis|
+          redis.incr('vmpooler__tasks__clone')
+          redis.incr('vmpooler__tasks__clone')
+          expect(redis.get('vmpooler__tasks__clone')).to eq('2')
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
+          expect(redis.get('vmpooler__tasks__clone')).to eq('1')
+        end
       end
 
       it 'should expire the vm metadata' do
-        expect(redis).to receive(:expire)
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire)
+          end
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
+        end
       end
 
       it 'should raise the error' do
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
+      end
+
+    end
+
+    context 'with request_id' do
+      before(:each) do
+        allow(metrics).to receive(:timing)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
+        expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).with(vm,pool).and_return(1)
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
+        expect(dns_plugin).to receive(:create_or_replace_record)
+        allow(logger).to receive(:log)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
+        end
+      end
+
+      it 'should set request_id and pool_alias on the vm data' do
+        redis_connection_pool.with do |redis|
+          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id)
+        end
+      end
+
+      it 'should reduce the clone count' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone')
+          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
+        end
+      end
+    end
+
+    context 'with #check_dns_available' do
+      before(:each) do
+        allow(logger).to receive(:log)
+        allow(provider).to receive(:get_vm_ip_address).and_return(true)
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
+      end
+      it 'should error out if DNS already exists' do
+        vm_name = "foo"
+        resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
+        expect(subject).to receive(:generate_and_check_hostname).exactly(3).times.and_return([vm_name, true]) #skip this, make it available all times
+        expect(resolv).to receive(:getaddress).exactly(3).times.and_return("1.2.3.4")
+        expect(metrics).to receive(:increment).with("vmpooler_errors.staledns.#{pool}").exactly(3).times
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/Unable to generate a unique hostname after/)
+      end
+      it 'should be successful if DNS does not exist' do
+        vm_name = "foo"
+        resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
+        expect(subject).to receive(:generate_and_check_hostname).and_return([vm_name, true])
+        expect(resolv).to receive(:getaddress).exactly(1).times.and_raise(Resolv::ResolvError)
+        expect(provider).to receive(:get_vm_ip_address)
+        expect(provider).to receive(:create_vm).with(pool, String)
+        subject._clone_vm(pool,provider,dns_plugin)
       end
     end
   end
@@ -688,17 +1280,17 @@ EOT
     end
 
     it 'calls _destroy_vm' do
-      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider)
+      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin)
 
-      subject.destroy_vm(vm,pool,provider)
+      subject.destroy_vm(vm,pool,provider,dns_plugin)
     end
 
     it 'logs a message if an error is raised' do
       allow(logger).to receive(:log)
       expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: MockError")
-      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider).and_raise('MockError')
+      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin).and_raise('MockError')
 
-      expect{subject.destroy_vm(vm,pool,provider)}.to raise_error(/MockError/)
+      expect{subject.destroy_vm(vm,pool,provider,dns_plugin)}.to raise_error(/MockError/)
     end
   end
 
@@ -706,7 +1298,9 @@ EOT
     before(:each) do
       expect(subject).not_to be_nil
 
-      create_completed_vm(vm,pool,true)
+      redis_connection_pool.with do |redis|
+        create_completed_vm(vm,pool,redis,true)
+      end
 
       allow(provider).to receive(:destroy_vm).with(pool,vm).and_return(true)
 
@@ -721,9 +1315,12 @@ EOT
       end
 
       it 'should call redis expire with 0' do
-        expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be_nil
-        subject._destroy_vm(vm,pool,provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be_nil
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__vm__#{vm}", 0)
+          end
+          subject._destroy_vm(vm,pool,provider,dns_plugin)
+        end
       end
     end
 
@@ -733,7 +1330,7 @@ EOT
       end
 
       it 'should raise an error' do
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(NoMethodError)
       end
     end
 
@@ -744,7 +1341,7 @@ EOT
       end
 
       it 'should not raise an error' do
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
     end
 
@@ -753,26 +1350,22 @@ EOT
         expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/)
         allow(logger).to receive(:log)
 
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
 
       it 'should emit a timing metric' do
         allow(subject).to receive(:get_vm_usage_labels)
-        expect(metrics).to receive(:timing).with("destroy.#{pool}", String)
+        allow(metrics).to receive(:timing)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with("vmpooler_destroy.#{pool}", String)
 
-        subject._destroy_vm(vm,pool,provider)
-      end
-
-      it 'should check usage labels' do
-        expect(subject).to receive(:get_vm_usage_labels).with(vm)
-
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
 
       it 'should dereference the mutex' do
         expect(subject).to receive(:dereference_mutex)
 
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
     end
 
@@ -786,13 +1379,13 @@ EOT
         expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/).exactly(0).times
         allow(logger).to receive(:log)
 
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
       end
 
       it 'should not emit a timing metric' do
         expect(metrics).to receive(:timing).with("destroy.#{pool}", String).exactly(0).times
 
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
       end
     end
 
@@ -805,214 +1398,12 @@ EOT
       it 'should return' do
         expect(subject).to receive(:vm_mutex).with(vm).and_return(mutex)
 
-        expect(subject._destroy_vm(vm,pool,provider)).to eq(nil)
+        expect(subject._destroy_vm(vm,pool,provider,dns_plugin)).to eq(nil)
       end
     end
   end
 
-  describe '#get_vm_usage_labels' do
-
-    let(:template) { 'pool1' }
-    let(:user) { 'vmpuser' }
-    let(:vm) { 'vm1' }
-
-    context 'when label evaluation is disabled' do
-      it 'should do nothing' do
-        subject.get_vm_usage_labels(vm)
-      end
-    end
-
-    context 'when label evaluation is enabled' do
-
-      before(:each) do
-        config[:config]['usage_stats'] = true
-      end
-
-      context 'when a VM has not been checked out' do
-        before(:each) do
-          create_ready_vm(template, vm)
-        end
-
-        it 'should return' do
-          expect(subject).to receive(:get_vm_usage_labels).and_return(nil)
-
-          subject.get_vm_usage_labels(vm)
-        end
-      end
-
-      context 'when a VM has been checked out' do
-
-        context 'without auth' do
-
-          before(:each) do
-            create_running_vm(template, vm)
-          end
-
-          it 'should emit a metric' do
-            expect(metrics).to receive(:increment).with("usage.unauthenticated.#{template}")
-
-            subject.get_vm_usage_labels(vm)
-          end
-        end
-
-        context 'with auth' do
-
-          before(:each) do
-            create_running_vm(template, vm, token, user)
-          end
-
-          it 'should emit a metric' do
-            expect(metrics).to receive(:increment).with("usage.#{user}.#{template}")
-
-            subject.get_vm_usage_labels(vm)
-          end
-
-          context 'with a user with period in name' do
-            let(:user) { 'test.user'.gsub('.', '_') }
-            let(:metric_string) { "usage.#{user}.#{template}" }
-            let(:metric_nodes) { metric_string.split('.') }
-
-            before(:each) do
-              create_running_vm(template, vm)
-            end
-
-            it 'should emit a metric with the character replaced' do
-              expect(metrics).to receive(:increment).with(metric_string)
-
-              subject.get_vm_usage_labels(vm)
-            end
-
-            it 'should include three nodes' do
-              expect(metric_nodes.count).to eq(3)
-            end
-
-          end
-
-          context 'with a jenkins_build_url label' do
-            let(:jenkins_build_url) { 'https://jenkins.example.com/job/enterprise_pe-acceptance-tests_integration-system_pe_full-agent-upgrade_weekend_2018.1.x/LAYOUT=centos6-64mcd-ubuntu1404-32f-64f,LEGACY_AGENT_VERSION=NONE,PLATFORM=NOTUSED,SCM_BRANCH=2018.1.x,UPGRADE_FROM=2018.1.0,UPGRADE_TO_VERSION=NONE,label=beaker/222/' }
-            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-            let(:instance) { url_parts[0] }
-            let(:value_stream_parts) { url_parts[2].split('_') }
-            let(:value_stream) { value_stream_parts.shift }
-            let(:branch) { value_stream_parts.pop }
-            let(:project) { value_stream_parts.shift }
-            let(:job_name) { value_stream_parts.join('_') }
-            let(:metric_string_nodes) {
-              [
-                'usage', user, instance, value_stream, branch, project, job_name, template
-              ]
-            }
-            let(:metric_string_sub) {
-              metric_string_nodes.map { |s| s.gsub('.', '_')
-              }
-            }
-            let(:metric_string) { metric_string_sub.join('.') }
-
-            before(:each) do
-              create_tag(vm, 'jenkins_build_url', jenkins_build_url)
-            end
-
-            it 'should emit a metric with information from the URL' do
-              expect(metrics).to receive(:increment).with(metric_string)
-
-              subject.get_vm_usage_labels(vm)
-            end
-          end
-
-          context 'with a jenkins_build_url that contains RMM_COMPONENT_TO_TEST_NAME' do
-            let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/' }
-            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-            let(:instance) { url_parts[0].gsub('.', '_') }
-            let(:value_stream_parts) { url_parts[2].split('_') }
-            let(:value_stream) { value_stream_parts.shift }
-            let(:branch) { value_stream_parts.pop }
-            let(:project) { value_stream_parts.shift }
-            let(:job_name) { value_stream_parts.join('_') }
-            let(:build_metadata) { url_parts[3] }
-            let(:build_component) { subject.component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata) }
-            let(:expected_string) { "usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{build_component}.#{template}" }
-            let(:metric_nodes) { expected_string.split('.') }
-
-            before(:each) do
-              create_tag(vm, 'jenkins_build_url', jenkins_build_url)
-            end
-
-            it 'should emit a metric with information from the URL' do
-              expect(metrics).to receive(:increment).with(expected_string)
-
-              subject.get_vm_usage_labels(vm)
-            end
-
-            it 'should contain exactly nine nodes' do
-              expect(metric_nodes.count).to eq(9)
-            end
-
-            context 'when there is no matrix job information' do
-
-              let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/824/' }
-              let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-              let(:instance) { url_parts[0].gsub('.', '_') }
-              let(:value_stream_parts) { url_parts[2].split('_') }
-              let(:value_stream) { value_stream_parts.shift }
-              let(:branch) { value_stream_parts.pop }
-              let(:project) { value_stream_parts.shift }
-              let(:job_name) { value_stream_parts.join('_') }
-
-              before(:each) do
-                create_tag(vm, 'jenkins_build_url', jenkins_build_url)
-              end
-
-              it 'should emit a metric with information from the URL without a build_component' do
-                expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{template}")
-
-                subject.get_vm_usage_labels(vm)
-              end
-            end
-          end
-
-        end
-      end
-    end
-  end
-
-  describe '#component_to_test' do
-    let(:matching_key) { 'LABEL_ONE' }
-    let(:matching_value) { 'test' }
-    let(:labels_string) { "#{matching_key}=#{matching_value},LABEL_TWO=test2,LABEL_THREE=test3" }
-    let(:nonmatrix_string) { 'test,stuff,and,things' }
-
-    context 'when string contains a matching key' do
-      it 'should print the corresponding value' do
-        expect(subject.component_to_test(matching_key, labels_string)).to eq(matching_value)
-      end
-
-      context 'when match contains no value' do
-        it 'should return nil' do
-          expect(subject.component_to_test(matching_key, matching_key)).to be nil
-        end
-      end
-    end
-
-    context 'when string contains no key value pairs' do
-      it 'should return' do
-        expect(subject.component_to_test(matching_key, nonmatrix_string)).to be nil
-      end
-    end
-
-    context 'when labels_string is a job number' do
-      it 'should return nil' do
-        expect(subject.component_to_test(matching_key, '25')).to be nil
-      end
-    end
-
-    context 'when labels_string is nil' do
-      it 'should return nil' do
-        expect(subject.component_to_test(matching_key, nil)).to be nil
-      end
-    end
-  end
-
-  describe '#purge_unused_vms_and_folders' do
+  describe '#purge_unused_vms_and_resources' do
     let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
@@ -1026,96 +1417,51 @@ EOT
     }
 
     it 'should return when purging is not enabled' do
-      expect(subject.purge_unused_vms_and_folders).to be_nil
+      expect(subject.purge_unused_vms_and_resources).to be_nil
     end
 
     context 'with purging enabled globally' do
       before(:each) do
-        config[:config]['purge_unconfigured_folders'] = true
+        config[:config]['purge_unconfigured_resources'] = true
         expect(Thread).to receive(:new).and_yield
       end
 
       it 'should run a purge for each provider' do
-        expect(subject).to receive(:purge_vms_and_folders)
+        expect(subject).to receive(:purge_vms_and_resources)
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
 
       it 'should log when purging fails' do
-        expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
+        expect(subject).to receive(:purge_vms_and_resources).and_raise(RuntimeError,'MockError')
         expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
     end
 
     context 'with purging enabled on the provider' do
       before(:each) do
-        config[:providers][:mock]['purge_unconfigured_folders'] = true
+        config[:providers][:mock]['purge_unconfigured_resources'] = true
         expect(Thread).to receive(:new).and_yield
       end
 
       it 'should run a purge for the provider' do
-        expect(subject).to receive(:purge_vms_and_folders)
+        expect(subject).to receive(:purge_vms_and_resources)
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
     end
   end
 
-  describe '#pool_folders' do
-    let(:folder_name) { 'myinstance' }
-    let(:folder_base) { 'vmpooler' }
-    let(:folder) { [folder_base,folder_name].join('/') }
-    let(:datacenter) { 'dc1' }
-    let(:provider_name) { 'mock_provider' }
-    let(:expected_response) {
-      {
-        folder_name => "#{datacenter}/vm/#{folder_base}"
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:providers:
-  :mock:
-:pools:
-  - name: '#{pool}'
-    folder: '#{folder}'
-    size: 1
-    datacenter: '#{datacenter}'
-    provider: '#{provider_name}'
-  - name: '#{pool}2'
-    folder: '#{folder}'
-    size: 1
-    datacenter: '#{datacenter}'
-    provider: '#{provider_name}2'
-EOT
-      )
-    }
-
-    it 'should return a list of pool folders' do
-      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
-
-      expect(subject.pool_folders(provider)).to eq(expected_response)
-    end
-
-    it 'should raise an error when the provider fails to get the datacenter' do
-      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
-
-      expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
-    end
-  end
-
-  describe '#purge_vms_and_folders' do
+  describe '#purge_vms_and_resources' do
     let(:folder_name) { 'myinstance' }
     let(:folder_base) { 'vmpooler' }
     let(:datacenter) { 'dc1' }
     let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
-    let(:configured_folders) { { folder_name => full_folder_path } }
-    let(:base_folders) { [ full_folder_path ] }
     let(:folder) { [folder_base,folder_name].join('/') }
     let(:provider_name) { 'mock_provider' }
-    let(:whitelist) { nil }
+    let(:allowlist) { nil }
     let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
@@ -1131,18 +1477,26 @@ EOT
       )
     }
 
-    it 'should run purge_unconfigured_folders' do
-      expect(subject).to receive(:pool_folders).and_return(configured_folders)
-      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
+    context 'when purging folders' do
+      before do
+        expect(subject).not_to be_nil
+        # Inject mock provider into global variable - Note this is a code smell
+        $providers = { provider_name => provider }
+      end
 
-      subject.purge_vms_and_folders(provider)
-    end
+      it 'should run purge_unconfigured_resources' do
+        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist)
+        expect(provider).to receive(:provider_config).and_return({}).twice
 
-    it 'should raise any errors' do
-      expect(subject).to receive(:pool_folders).and_return(configured_folders)
-      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
+        subject.purge_vms_and_resources(provider_name)
+      end
 
-      expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+      it 'should raise any errors' do
+        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist).and_raise('mockerror')
+        expect(provider).to receive(:provider_config).and_return({}).twice
+
+        expect{ subject.purge_vms_and_resources(provider_name) }.to raise_error(RuntimeError, 'mockerror')
+      end
     end
   end
 
@@ -1169,7 +1523,9 @@ EOT
       expect(subject).not_to be_nil
       allow(logger).to receive(:log)
 
-      create_running_vm(pool,vm,token)
+      redis_connection_pool.with do |redis|
+        create_running_vm(pool,vm,redis,token)
+      end
     end
 
     context 'Given a VM that does not exist' do
@@ -1179,8 +1535,10 @@ EOT
       end
 
       it 'should not update redis if the VM does not exist' do
-        expect(redis).to receive(:hset).exactly(0).times
-        expect{ subject._create_vm_disk(pool, vm, disk_size, provider) }.to raise_error(RuntimeError)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hset).exactly(0).times
+          expect{ subject._create_vm_disk(pool, vm, disk_size, provider) }.to raise_error(RuntimeError)
+        end
       end
     end
 
@@ -1197,8 +1555,10 @@ EOT
       end
 
       it 'should raise an error if the disk size is a Fixnum' do
-        expect(redis).to receive(:hset).exactly(0).times
-        expect{ subject._create_vm_disk(pool, vm, 10, provider) }.to raise_error(NoMethodError,/empty?/)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hset).exactly(0).times
+          expect{ subject._create_vm_disk(pool, vm, 10, provider) }.to raise_error(NoMethodError,/empty?/)
+        end
       end
     end
 
@@ -1215,16 +1575,20 @@ EOT
       end
 
       it 'should update redis information when attaching the first disk' do
-        subject._create_vm_disk(pool, vm, disk_size, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("+#{disk_size}gb")
+        redis_connection_pool.with do |redis|
+          subject._create_vm_disk(pool, vm, disk_size, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("+#{disk_size}gb")
+        end
       end
 
       it 'should update redis information when attaching the additional disks' do
-        initial_disks = '+10gb:+20gb'
-        redis.hset("vmpooler__vm__#{vm}", 'disk', initial_disks)
+        redis_connection_pool.with do |redis|
+          initial_disks = '+10gb:+20gb'
+          redis.hset("vmpooler__vm__#{vm}", 'disk', initial_disks)
 
-        subject._create_vm_disk(pool, vm, disk_size, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("#{initial_disks}:+#{disk_size}gb")
+          subject._create_vm_disk(pool, vm, disk_size, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("#{initial_disks}:+#{disk_size}gb")
+        end
       end
     end
 
@@ -1234,10 +1598,12 @@ EOT
       end
 
       it 'should not update redis information' do
-        expect(redis).to receive(:hset).exactly(0).times
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hset).exactly(0).times
 
-        subject._create_vm_disk(pool, vm, disk_size, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to be_nil
+          subject._create_vm_disk(pool, vm, disk_size, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to be_nil
+        end
       end
 
       it 'should log a message' do
@@ -1271,7 +1637,9 @@ EOT
     end
 
     before(:each) do
-      create_running_vm(pool,vm,token)
+      redis_connection_pool.with do |redis|
+        create_running_vm(pool,vm,redis,token)
+      end
     end
 
     context 'Given a Pool that does not exist' do
@@ -1282,9 +1650,11 @@ EOT
       end
 
       it 'should not update redis' do
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
-        expect{ subject._create_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found")
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+          expect{ subject._create_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found")
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        end
       end
     end
 
@@ -1295,9 +1665,11 @@ EOT
       end
 
       it 'should not update redis' do
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
-        expect{ subject._create_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found")
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+          expect{ subject._create_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found")
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        end
       end
     end
 
@@ -1314,9 +1686,11 @@ EOT
       end
 
       it 'should add snapshot redis information' do
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
-        subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to_not be_nil
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+          subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to_not be_nil
+        end
       end
     end
 
@@ -1333,9 +1707,11 @@ EOT
       end
 
       it 'should not update redis' do
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
-        subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
-        expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        redis_connection_pool.with do |redis|
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+          subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
+          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
+        end
       end
     end
   end
@@ -1429,17 +1805,23 @@ EOT
   describe '#get_pool_name_for_vm' do
     context 'Given a valid VM' do
       before(:each) do
-        create_running_vm(pool, vm, token)
+        redis_connection_pool.with do |redis|
+          create_running_vm(pool, vm, redis, token)
+        end
       end
 
       it 'should return the pool name' do
-        expect(subject.get_pool_name_for_vm(vm)).to eq(pool)
+        redis_connection_pool.with do |redis|
+          expect(subject.get_pool_name_for_vm(vm,redis)).to eq(pool)
+        end
       end
     end
 
     context 'Given an invalid VM' do
       it 'should return nil' do
-        expect(subject.get_pool_name_for_vm('does_not_exist')).to be_nil
+        redis_connection_pool.with do |redis|
+          expect(subject.get_pool_name_for_vm('does_not_exist',redis)).to be_nil
+        end
       end
     end
   end
@@ -1531,6 +1913,51 @@ EOT
     end
   end
 
+  describe '#get_dns_plugin_class_name_for_pool' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+
+    )}
+    before(:each) do
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+    end
+
+    it 'calls Vmpooler::Dns.get_dns_plugin_class_by_name' do
+      expect(Vmpooler::Dns).to receive(:get_dns_plugin_class_by_name).with(config, 'mock')
+
+      subject.get_dns_plugin_class_name_for_pool(pool)
+    end
+  end
+
+  describe '#get_dns_plugin_domain_for_pool' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+  )}
+    before(:each) do
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+    end
+
+    it 'calls Vmpooler::Dns.get_dns_plugin_domain_for_pool' do
+      expect(Vmpooler::Dns).to receive(:get_dns_plugin_domain_by_name).with(config, 'mock')
+
+      subject.get_dns_plugin_domain_for_pool(pool)
+    end
+  end
+
   describe '#check_disk_queue' do
     let(:threads) {[]}
 
@@ -1608,7 +2035,9 @@ EOT
 
     context 'when VM in the queue does not exist' do
       before(:each) do
-        disk_task_vm(vm,"snapshot_#{vm}")
+        redis_connection_pool.with do |redis|
+          disk_task_vm(vm,"snapshot_#{vm}",redis)
+        end
       end
 
       it 'should log an error' do
@@ -1626,9 +2055,11 @@ EOT
 
     context 'when specified provider does not exist' do
       before(:each) do
-        disk_task_vm(vm,"snapshot_#{vm}")
-        create_running_vm(pool, vm, token)
-        expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+        redis_connection_pool.with do |redis|
+          disk_task_vm(vm,"snapshot_#{vm}",redis)
+          create_running_vm(pool, vm, redis, token)
+          expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+        end
       end
 
       it 'should log an error' do
@@ -1647,8 +2078,10 @@ EOT
     context 'when multiple VMs in the queue' do
       before(:each) do
         ['vm1', 'vm2', 'vm3'].each do |vm_name|
-          disk_task_vm(vm_name,"snapshot_#{vm_name}")
-          create_running_vm(pool, vm_name, token)
+          redis_connection_pool.with do |redis|
+            disk_task_vm(vm_name,"snapshot_#{vm_name}",redis)
+            create_running_vm(pool, vm_name, redis, token)
+          end
         end
 
         allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
@@ -1677,6 +2110,7 @@ EOT
 
     before(:each) do
       expect(Thread).to receive(:new).and_yield
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
       allow(subject).to receive(:_check_snapshot_queue).with(no_args)
     end
 
@@ -1722,6 +2156,10 @@ EOT
       let(:maxloop) { 5 }
       # Note a maxloop of zero can not be tested as it never terminates
 
+      before(:each) do
+        allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+      end
+
       after(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = nil
@@ -1750,7 +2188,9 @@ EOT
 
       context 'when VM in the queue does not exist' do
         before(:each) do
-          snapshot_vm(vm,"snapshot_#{vm}")
+          redis_connection_pool.with do |redis|
+            snapshot_vm(vm,"snapshot_#{vm}",redis)
+          end
         end
 
         it 'should log an error' do
@@ -1768,9 +2208,11 @@ EOT
 
       context 'when specified provider does not exist' do
         before(:each) do
-          snapshot_vm(vm,"snapshot_#{vm}")
-          create_running_vm(pool, vm, token)
-          expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+          redis_connection_pool.with do |redis|
+            snapshot_vm(vm,"snapshot_#{vm}",redis)
+            create_running_vm(pool, vm, redis, token)
+            expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+          end
         end
 
         it 'should log an error' do
@@ -1789,8 +2231,10 @@ EOT
       context 'when multiple VMs in the queue' do
         before(:each) do
           ['vm1', 'vm2', 'vm3'].each do |vm_name|
-            snapshot_vm(vm_name,"snapshot_#{vm_name}")
-            create_running_vm(pool, vm_name, token)
+            redis_connection_pool.with do |redis|
+              snapshot_vm(vm_name,"snapshot_#{vm_name}",redis)
+              create_running_vm(pool, vm_name, redis, token)
+            end
           end
 
           allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
@@ -1824,7 +2268,9 @@ EOT
 
       context 'when VM in the queue does not exist' do
         before(:each) do
-          snapshot_revert_vm(vm,"snapshot_#{vm}")
+          redis_connection_pool.with do |redis|
+            snapshot_revert_vm(vm,"snapshot_#{vm}",redis)
+          end
         end
 
         it 'should log an error' do
@@ -1842,9 +2288,11 @@ EOT
 
       context 'when specified provider does not exist' do
         before(:each) do
-          snapshot_revert_vm(vm,"snapshot_#{vm}")
-          create_running_vm(pool, vm, token)
-          expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+          redis_connection_pool.with do |redis|
+            snapshot_revert_vm(vm,"snapshot_#{vm}",redis)
+            create_running_vm(pool, vm, redis, token)
+            expect(subject).to receive(:get_provider_for_pool).and_return(nil)
+          end
         end
 
         it 'should log an error' do
@@ -1863,8 +2311,10 @@ EOT
       context 'when multiple VMs in the queue' do
         before(:each) do
           ['vm1', 'vm2', 'vm3'].each do |vm_name|
-            snapshot_revert_vm(vm_name,"snapshot_#{vm_name}")
-            create_running_vm(pool, vm_name, token)
+            redis_connection_pool.with do |redis|
+              snapshot_revert_vm(vm_name,"snapshot_#{vm_name}",redis)
+              create_running_vm(pool, vm_name, redis, token)
+            end
           end
 
           allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
@@ -1978,15 +2428,18 @@ EOT
     end
 
     it 'returns when a template is set and matches the configured template' do
-      redis.hset('vmpooler__config__template', pool, old_template)
-
+      redis_connection_pool.with do |redis|
+        redis.hset('vmpooler__config__template', pool, old_template)
+      end
       subject.sync_pool_template(config[:pools][0])
 
       expect(config[:pools][0]['template']).to eq(old_template)
     end
 
     it 'updates a pool template when the redis provided value is different' do
-      redis.hset('vmpooler__config__template', pool, new_template)
+      redis_connection_pool.with do |redis|
+        redis.hset('vmpooler__config__template', pool, new_template)
+      end
 
       subject.sync_pool_template(config[:pools][0])
 
@@ -2026,7 +2479,9 @@ EOT
     end
 
     it 'should set the pool template to match the configured template' do
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+      redis_connection_pool.with do |redis|
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
 
       expect(poolconfig['template']).to eq(new_template)
     end
@@ -2034,31 +2489,41 @@ EOT
     it 'should log that the template is updated' do
       expect(logger).to receive(:log).with('s', "[*] [#{pool}] template updated from #{current_template} to #{new_template}")
 
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+      redis_connection_pool.with do |redis|
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
     end
 
     it 'should run drain_pool' do
-      expect(subject).to receive(:drain_pool).with(pool)
+      redis_connection_pool.with do |redis|
+        expect(subject).to receive(:drain_pool).with(pool, redis)
 
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
     end
 
     it 'should log that a template is being prepared' do
       expect(logger).to receive(:log).with('s', "[*] [#{pool}] preparing pool template for deployment")
 
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+      redis_connection_pool.with do |redis|
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
     end
 
     it 'should run prepare_template' do
-      expect(subject).to receive(:prepare_template).with(poolconfig, provider)
+      redis_connection_pool.with do |redis|
+        expect(subject).to receive(:prepare_template).with(poolconfig, provider, redis)
 
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
     end
 
     it 'should log that the pool is ready for use' do
       expect(logger).to receive(:log).with('s', "[*] [#{pool}] is ready for use")
 
-      subject.update_pool_template(poolconfig, provider, new_template, current_template)
+      redis_connection_pool.with do |redis|
+        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
+      end
     end
   end
 
@@ -2088,8 +2553,11 @@ EOT
     context 'when the mutex is locked' do
       let(:mutex) { Mutex.new }
       before(:each) do
-        expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(2)
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool,'vm1',redis)
+          create_ready_vm(pool,'vm2',redis)
+          create_pending_vm(pool,'vm3',redis)
+        end
         mutex.lock
         expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
       end
@@ -2101,28 +2569,35 @@ EOT
 
     context 'with a total size less than the pool size' do
       it 'should return nil' do
-        expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
+        end
         expect(subject.remove_excess_vms(config[:pools][0])).to be_nil
       end
     end
 
     context 'with a total size greater than the pool size' do
       it 'should remove excess ready vms' do
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(4)
-        expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(0)
+        redis_connection_pool.with do |redis|
+          ['vm1','vm2','vm3','vm4'].each do |v|
+            create_ready_vm(pool,v,redis)
+          end
+        end
         expect(subject).to receive(:move_vm_queue).exactly(2).times
 
         subject.remove_excess_vms(config[:pools][0])
       end
 
       it 'should remove excess pending vms' do
-        create_pending_vm(pool,'vm1')
-        create_pending_vm(pool,'vm2')
-        create_ready_vm(pool, 'vm3')
-        create_ready_vm(pool, 'vm4')
-        create_ready_vm(pool, 'vm5')
-        expect(subject).to receive(:move_vm_queue).exactly(3).times
+        redis_connection_pool.with do |redis|
+          create_pending_vm(pool,'vm1',redis)
+          create_pending_vm(pool,'vm2',redis)
+          create_ready_vm(pool,'vm3',redis)
+          create_ready_vm(pool,'vm4',redis)
+          create_ready_vm(pool,'vm5',redis)
+          expect(subject).to receive(:move_vm_queue).exactly(3).times
+        end
 
         subject.remove_excess_vms(config[:pools][0])
       end
@@ -2146,33 +2621,43 @@ EOT
 
     context 'when creating the template delta disks' do
       before(:each) do
-        allow(redis).to receive(:hset)
+        redis_connection_pool.with do |redis|
+          allow(redis).to receive(:hset)
+        end
         allow(provider).to receive(:create_template_delta_disks)
       end
 
       it 'should run create template delta disks' do
-        expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])
 
-        subject.prepare_template(config[:pools][0], provider)
+          subject.prepare_template(config[:pools][0], provider, redis)
+        end
       end
 
       it 'should mark the template as prepared' do
-        expect(redis).to receive(:hset).with('vmpooler__template__prepared', pool, config[:pools][0]['template'])
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hset).with('vmpooler__template__prepared', pool, config[:pools][0]['template'])
 
-        subject.prepare_template(config[:pools][0], provider)
+          subject.prepare_template(config[:pools][0], provider, redis)
+        end
       end
     end
 
     context 'when template delta disk creation fails' do
       before(:each) do
-        allow(redis).to receive(:hset)
+        redis_connection_pool.with do |redis|
+          allow(redis).to receive(:hset)
+        end
         expect(provider).to receive(:create_template_delta_disks).and_raise("MockError")
       end
 
       it 'should log a message when delta disk creation returns an error' do
-        expect(logger).to receive(:log).with('s', "[!] [#{pool}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: MockError")
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "[!] [#{pool}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: MockError")
 
-        subject.prepare_template(config[:pools][0], provider)
+          subject.prepare_template(config[:pools][0], provider, redis)
+        end
       end
     end
   end
@@ -2196,18 +2681,24 @@ EOT
     }
 
     before(:each) do
-      allow(redis).to receive(:hget)
+      redis_connection_pool.with do |redis|
+        allow(redis).to receive(:hget)
+      end
       expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
     end
 
     it 'should retreive the prepared template' do
-      expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+      end
 
       subject.evaluate_template(config[:pools][0], provider)
     end
 
     it 'should retrieve the redis configured template' do
-      expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
+      end
 
       subject.evaluate_template(config[:pools][0], provider)
     end
@@ -2225,14 +2716,18 @@ EOT
     context 'when prepared template is nil' do
 
       it 'should prepare the template' do
-        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(nil)
-        expect(subject).to receive(:prepare_template).with(config[:pools][0], provider)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(nil)
+          expect(subject).to receive(:prepare_template).with(config[:pools][0], provider, redis)
+        end
 
         subject.evaluate_template(config[:pools][0], provider)
       end
 
       it 'should not prepare the template again' do
-        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+        end
         expect(subject).to_not receive(:prepare_template).with(config[:pools][0], provider)
 
         subject.evaluate_template(config[:pools][0], provider)
@@ -2241,19 +2736,25 @@ EOT
 
     context 'when the configured pool template does not match the prepared template' do
       before(:each) do
-        config[:pools][0]['template'] = new_template
-        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+        redis_connection_pool.with do |redis|
+          config[:pools][0]['template'] = new_template
+          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+        end
       end
 
       it 'should prepare the template' do
-        expect(subject).to receive(:prepare_template).with(config[:pools][0], provider)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:prepare_template).with(config[:pools][0], provider, redis)
+        end
 
         subject.evaluate_template(config[:pools][0], provider)
       end
 
       context 'if configured_template is provided' do
         it 'should not run prepare_template' do
-          expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(current_template)
+          redis_connection_pool.with do |redis|
+            expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(current_template)
+          end
           expect(subject).to_not receive(:prepare_template)
 
           subject.evaluate_template(config[:pools][0], provider)
@@ -2263,12 +2764,16 @@ EOT
 
     context 'when a new template is requested' do
       before(:each) do
-        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
-        expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
+          expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
+        end
       end
 
       it 'should update the template' do
-        expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, new_template, current_template)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, new_template, current_template, redis)
+        end
 
         subject.evaluate_template(config[:pools][0], provider)
       end
@@ -2282,57 +2787,75 @@ EOT
 
     context 'with no vms' do
       it 'should return nil' do
-        expect(subject.drain_pool(pool)).to be_nil
+        redis_connection_pool.with do |redis|
+          expect(subject.drain_pool(pool, redis)).to be_nil
+        end
       end
 
       it 'should not log any messages' do
         expect(logger).to_not receive(:log)
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
 
       it 'should not try to move any vms' do
         expect(subject).to_not receive(:move_vm_queue)
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
     end
 
     context 'with ready vms' do
       before(:each) do
-        create_ready_vm(pool, 'vm1')
-        create_ready_vm(pool, 'vm2')
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool, 'vm1', redis)
+          create_ready_vm(pool, 'vm2', redis)
+        end
       end
 
       it 'removes the ready instances' do
         expect(subject).to receive(:move_vm_queue).twice
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
 
       it 'logs that ready instances are being removed' do
         expect(logger).to receive(:log).with('s', "[*] [#{pool}] removing ready instances")
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
     end
 
     context 'with pending instances' do
       before(:each) do
-        create_pending_vm(pool, 'vm1')
-        create_pending_vm(pool, 'vm2')
+        redis_connection_pool.with do |redis|
+          create_pending_vm(pool, 'vm1', redis)
+          create_pending_vm(pool, 'vm2', redis)
+        end
       end
 
       it 'removes the pending instances' do
         expect(subject).to receive(:move_vm_queue).twice
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
 
       it 'logs that pending instances are being removed' do
         expect(logger).to receive(:log).with('s', "[*] [#{pool}] removing pending instances")
 
-        subject.drain_pool(pool)
+        redis_connection_pool.with do |redis|
+          subject.drain_pool(pool, redis)
+        end
       end
     end
   end
@@ -2364,25 +2887,33 @@ EOT
     end
 
     it 'should get the pool size configuration from redis' do
-      expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool)
+      end
 
       subject.update_pool_size(poolconfig)
     end
 
     it 'should return when poolsize is not set in redis' do
-      expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(nil)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(nil)
+      end
 
       expect(subject.update_pool_size(poolconfig)).to be_nil
     end
 
     it 'should return when no change in configuration is required' do
-      expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return('2')
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return('2')
+      end
 
       expect(subject.update_pool_size(poolconfig)).to be_nil
     end
 
     it 'should update the pool size' do
-      expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(newsize)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(newsize)
+      end
 
       subject.update_pool_size(poolconfig)
 
@@ -2417,25 +2948,33 @@ EOT
     end
 
     it 'should get the pool clone target configuration from redis' do
-      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool)
+      end
 
       subject.update_clone_target(poolconfig)
     end
 
     it 'should return when clone_target is not set in redis' do
-      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(nil)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(nil)
+      end
 
       expect(subject.update_clone_target(poolconfig)).to be_nil
     end
 
     it 'should return when no change in configuration is required' do
-      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return('cluster1')
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return('cluster1')
+      end
 
       expect(subject.update_clone_target(poolconfig)).to be_nil
     end
 
     it 'should update the clone target' do
-      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(newtarget)
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(newtarget)
+      end
 
       subject.update_clone_target(poolconfig)
 
@@ -2459,6 +2998,8 @@ EOT
       allow(subject).to receive(:check_disk_queue)
       allow(subject).to receive(:check_snapshot_queue)
       allow(subject).to receive(:check_pool)
+      allow(subject).to receive(:check_ondemand_requests)
+      allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
 
       allow(logger).to receive(:log)
     end
@@ -2469,6 +3010,23 @@ EOT
     end
 
     context 'on startup' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+        )}
+
+      before(:each) do
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+      end
+
       it 'should log a message that VMPooler has started' do
         expect(logger).to receive(:log).with('d', 'starting vmpooler')
 
@@ -2476,15 +3034,19 @@ EOT
       end
 
       it 'should set clone tasks to zero' do
-        redis.set('vmpooler__tasks__clone', 1)
-        subject.execute!(1,0)
-        expect(redis.get('vmpooler__tasks__clone')).to eq('0')
+        redis_connection_pool.with do |redis|
+          redis.set('vmpooler__tasks__clone', 1)
+          subject.execute!(1,0)
+          expect(redis.get('vmpooler__tasks__clone')).to eq('0')
+        end
       end
 
       it 'should clear migration tasks' do
-        redis.set('vmpooler__migration', 1)
-        subject.execute!(1,0)
-        expect(redis.get('vmpooler__migration')).to be_nil
+        redis_connection_pool.with do |redis|
+          redis.set('vmpooler__migration', 1)
+          subject.execute!(1,0)
+          expect(redis.get('vmpooler__migration')).to be_nil
+        end
       end
 
       it 'should run the check_disk_queue method' do
@@ -2505,6 +3067,52 @@ EOT
         subject.execute!(1,0)
       end
 
+      it 'should run the check_ondemand_requests method' do
+        expect(subject).to receive(:check_ondemand_requests)
+
+        subject.execute!(1,0)
+      end
+
+      context 'creating Dns plugins' do
+        let(:mock_dns_plugin) { double('mock_dns_plugin') }
+        let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+  - name: 'dummy'
+    dns_plugin: 'mock'
+  - name: 'dummy2'
+    dns_plugin: 'mock'
+EOT
+        )}
+
+        it 'should call create_dns_object idempotently' do
+          # Even though there are two pools using the mock dns plugin, it should only
+          # create the dns object once.
+          expect(subject).to receive(:create_dns_object).and_return(mock_dns_plugin)
+
+          subject.execute!(1,0)
+        end
+
+        it 'should raise an error if the dns plugin cannot be created' do
+          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")
+
+          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
+        end
+
+        it 'should log a message if the dns plugin cannot be created' do
+          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")
+          expect(logger).to receive(:log).with('s',"Error while creating dns plugin for pool #{pool}: MockError")
+
+          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
+        end
+      end
+
       context 'creating Providers' do
         let(:vsphere_provider) { double('vsphere_provider') }
         let(:config) {
@@ -2512,17 +3120,27 @@ EOT
 ---
 :providers:
   :vsphere: {}
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    provider: 'vsphere'
+    dns_plugin: 'mock'
   - name: 'dummy'
     provider: 'vsphere'
+    dns_plugin: 'mock'
 EOT
         )}
 
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
+
         it 'should call create_provider_object idempotently' do
-          # Even though there are two pools using the vsphere provider (the default), it should only
+          # Even though there are two pools using the vsphere provider, it should only
           # create the provider object once.
-          expect(subject).to receive(:create_provider_object).with(Object, Object, Object, 'vsphere', 'vsphere', Object).and_return(vsphere_provider)
+          expect(subject).to receive(:create_provider_object).with(Object, Object, Object, redis_connection_pool, 'vsphere', 'vsphere', Object).and_return(vsphere_provider)
 
           subject.execute!(1,0)
         end
@@ -2542,62 +3160,25 @@ EOT
       end
     end
 
-    context 'creating multiple vsphere Providers' do
-      let(:vsphere_provider) { double('vsphere_provider') }
-      let(:vsphere_provider2) { double('vsphere_provider') }
-      let(:provider1) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, 'vsphere', provider_options) }
-      let(:provider2) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, 'secondvsphere', provider_options) }
-      let(:config) {
-        YAML.load(<<-EOT
----
-:providers:
-  :vsphere:
-    server: 'blah1'
-    provider_class: 'vsphere'
-  :secondvsphere:
-    server: 'blah2'
-    provider_class: 'vsphere'
-:pools:
-  - name: #{pool}
-    provider: 'vsphere'
-  - name: 'secondpool'
-    provider: 'secondvsphere'
-EOT
-        )}
-
-      it 'should call create_provider_object twice' do
-        # The two pools use a different provider name, but each provider_class is vsphere
-        expect(subject).to receive(:create_provider_object).with(Object, Object, Object, "vsphere", "vsphere", Object).and_return(vsphere_provider)
-        expect(subject).to receive(:create_provider_object).with(Object, Object, Object, "vsphere", "secondvsphere", Object).and_return(vsphere_provider2)
-        subject.execute!(1,0)
-      end
-
-      it 'should have vsphere providers with different servers' do
-        allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider1)
-        result = subject.get_provider_for_pool(pool)
-        allow(provider1).to receive(:provider_config).and_call_original
-        expect(result.provider_config['server']).to eq('blah1')
-
-        allow(subject).to receive(:get_provider_for_pool).with('secondpool').and_return(provider2)
-        result = subject.get_provider_for_pool('secondpool')
-        allow(provider1).to receive(:provider_config).and_call_original
-        expect(result.provider_config['server']).to eq('blah2')
-        subject.execute!(1,0)
-      end
-    end
-
     context 'modify configuration on startup' do
       context 'move vSphere configuration to providers location' do
         let(:config) {
         YAML.load(<<-EOT
+:dns_configs:
+  :mock:
+    dns_class: base
 :vsphere:
   server: 'vsphere.example.com'
   username: 'vmpooler'
   password: 'password'
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
 EOT
         )}
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
 
         it 'should set providers with no provider to vsphere' do
           expect(subject.config[:providers]).to be nil
@@ -2616,29 +3197,38 @@ EOT
       end
 
 
-      context 'default to the vphere provider' do
+      context 'default to the dummy provider' do
         let(:config) {
         YAML.load(<<-EOT
 ---
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
   - name: 'dummy'
     provider: 'dummy'
+    dns_plugin: 'mock'
 EOT
         )}
 
-        it 'should set providers with no provider to vsphere' do
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
+
+        it 'should set providers with no provider to dummy' do
           expect(subject.config[:pools][0]['provider']).to be_nil
           expect(subject.config[:pools][1]['provider']).to eq('dummy')
 
           subject.execute!(1,0)
 
-          expect(subject.config[:pools][0]['provider']).to eq('vsphere')
+          expect(subject.config[:pools][0]['provider']).to eq('dummy')
           expect(subject.config[:pools][1]['provider']).to eq('dummy')
         end
 
         it 'should log a message' do
-          expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'vsphere' as default")
+          expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'dummy' as default")
 
           subject.execute!(1,0)
         end
@@ -2647,11 +3237,31 @@ EOT
 
     context 'with dead disk_manager thread' do
       let(:disk_manager_thread) { double('thread', :alive? => false) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
 
       before(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = {}
         $threads['disk_manager'] = disk_manager_thread
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'should run the check_disk_queue method and log a message' do
@@ -2664,10 +3274,30 @@ EOT
 
     context 'with dead snapshot_manager thread' do
       let(:snapshot_manager_thread) { double('thread', :alive? => false) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       before(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = {}
         $threads['snapshot_manager'] = snapshot_manager_thread
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'should run the check_snapshot_queue method and log a message' do
@@ -2685,10 +3315,27 @@ EOT
         let(:default_check_loop_delay_min) { 5 }
         let(:default_check_loop_delay_max) { 60 }
         let(:default_check_loop_delay_decay) { 2.0 }
+        let(:check_loop_delay_min) { 7 }
+        let(:check_loop_delay_max) { 20 }
+        let(:check_loop_delay_decay) { 2.1 }
+        let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
         before(:each) do
           # Reset the global variable - Note this is a code smell
           $threads = {}
           $threads[pool] = pool_thread
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
         end
 
         it 'should run the check_pool method and log a message' do
@@ -2702,6 +3349,37 @@ EOT
         end
       end
 
+      context 'with dead ondemand provisioner thread' do
+        let(:ondemand_provisioner_thread) { double('thread', :alive? => false) }
+        let(:default_check_loop_delay_min) { 5 }
+        let(:default_check_loop_delay_max) { 60 }
+        let(:default_check_loop_delay_decay) { 2.0 }
+        let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
+        before(:each) do
+          # Reset the global variable - Note this is a code smell
+          $threads = {}
+          $threads['ondemand_provisioner'] = ondemand_provisioner_thread
+        end
+
+        it 'should run the process_ondemand_requests method' do
+          expect(subject).to receive(:check_ondemand_requests).with(default_check_loop_delay_min, default_check_loop_delay_max, default_check_loop_delay_decay)
+          subject.execute!(1,0)
+        end
+
+      end
+
       context 'with check_loop_delay_xxx settings' do
         let(:pool_thread) { double('thread', :alive? => false) }
         let(:check_loop_delay_min) { 7 }
@@ -2715,8 +3393,12 @@ EOT
   check_loop_delay_min: #{check_loop_delay_min}
   check_loop_delay_max: #{check_loop_delay_max}
   check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
 EOT
           )
         }
@@ -2724,6 +3406,8 @@ EOT
           # Reset the global variable - Note this is a code smell
           $threads = {}
           $threads[pool] = pool_thread
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+          # allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
         end
 
         it 'should run the check_pool method and log a message' do
@@ -2741,12 +3425,34 @@ EOT
     context 'delays between loops' do
       let(:maxloop) { 2 }
       let(:loop_delay) { 1 }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:maxloop) { 5 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       # Note a maxloop of zero can not be tested as it never terminates
       before(:each) do
-  
+
         allow(subject).to receive(:check_disk_queue)
         allow(subject).to receive(:check_snapshot_queue)
         allow(subject).to receive(:check_pool)
+        allow(subject).to receive(:check_ondemand_requests)
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'when a non-default loop delay is specified' do
@@ -2758,14 +3464,38 @@ EOT
 
     context 'loops specified number of times (5)' do
       let(:alive_thread) { double('thread', :alive? => true) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
       let(:maxloop) { 5 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       # Note a maxloop of zero can not be tested as it never terminates
       before(:each) do
-        end
+        allow(subject).to receive(:get_domain_for_pool)
+        allow(subject).to receive(:get_dns_plugin_config_classes)
+      end
 
       it 'should run startup tasks only once' do
-        expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once
-        expect(redis).to receive(:del).with('vmpooler__migration').once
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once
+          expect(redis).to receive(:set).with('vmpooler__tasks__ondemandclone', 0).once
+          expect(redis).to receive(:del).with('vmpooler__migration').once
+        end
 
         subject.execute!(maxloop,0)
       end
@@ -2774,6 +3504,7 @@ EOT
         expect(subject).to receive(:check_disk_queue).exactly(maxloop).times
         expect(subject).to receive(:check_snapshot_queue).exactly(maxloop).times
         expect(subject).to receive(:check_pool).exactly(maxloop).times
+        expect(subject).to receive(:check_ondemand_requests).exactly(maxloop).times
 
         subject.execute!(maxloop,0)
       end
@@ -2782,14 +3513,28 @@ EOT
         expect(subject).to receive(:check_disk_queue).exactly(0).times
         expect(subject).to receive(:check_snapshot_queue).exactly(0).times
         expect(subject).to receive(:check_pool).exactly(0).times
+        expect(subject).to receive(:check_ondemand_requests).exactly(0).times
 
         $threads[pool] = alive_thread
         $threads['disk_manager'] = alive_thread
         $threads['snapshot_manager'] = alive_thread
+        $threads['ondemand_provisioner'] = alive_thread
 
         subject.execute!(maxloop,0)
       end
     end
+
+    context 'when redis server connection is not available' do
+      let(:maxloop) { 2 }
+      it 'should log a failure and raise the error' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).and_raise(Redis::CannotConnectError)
+        end
+        expect(logger).to receive(:log).with('s', 'Cannot connect to the redis server: Redis::CannotConnectError')
+
+        expect{subject.execute!(maxloop,0)}.to raise_error Redis::CannotConnectError
+      end
+    end
   end
 
   describe "#sleep_with_wakeup_events" do
@@ -2824,22 +3569,28 @@ EOT
       let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
 
       it 'should check the number of VMs ready in Redis' do
-        expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").once
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").once
+        end
 
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
       end
 
       it 'should sleep until the number of VMs ready in Redis increases' do
-        expect(subject).to receive(:sleep).exactly(3).times
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1,1,1,2)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1,1,1,2)
+        end
 
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
       end
 
       it 'should sleep until the number of VMs ready in Redis decreases' do
-        expect(subject).to receive(:sleep).exactly(3).times
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(2,2,2,1)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(2,2,2,1)
+        end
 
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
       end
@@ -2855,25 +3606,160 @@ EOT
 
       context 'with a template configured' do
         before(:each) do
-          redis.hset('vmpooler__config__template', pool, new_template)
-          allow(redis).to receive(:hget)
+          redis_connection_pool.with do |redis|
+            redis.hset('vmpooler__config__template', pool, new_template)
+            allow(redis).to receive(:hget)
+          end
         end
 
         it 'should check if a template is configured in redis' do
-          expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
-          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).once
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
+            expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).once
+          end
 
           subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
         end
 
         it 'should sleep until a template change is detected' do
-          expect(subject).to receive(:sleep).exactly(3).times
-          expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(nil,nil,new_template)
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(nil,nil,new_template)
+          end
 
           subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
         end
       end
     end
+
+    describe 'with the pool_reset wakeup option' do
+      let(:wakeup_option) {{
+        :pool_reset => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a pool reset is requested' do
+
+        it 'should sleep until the reset request is detected' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:sismember).with('vmpooler__poolreset', pool).and_return(false,false,true)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
+
+    describe 'with the undo_override wakeup option' do
+      let(:wakeup_option) {{
+        :undo_override => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a undoing a template override is requested' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.sadd('vmpooler__pool__undo_template_override', pool)
+            allow(redis).to receive(:hget)
+          end
+        end
+
+        it 'should sleep until the undo override request is detected' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).at_least(2).times
+            expect(subject).to receive(:sleep).at_most(3).times
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,true)
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+
+      context 'when a undoing a size override is requested' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.sadd('vmpooler__pool__undo_size_override', pool)
+            allow(redis).to receive(:hget)
+          end
+        end
+
+        it 'should sleep until the undo override request is detected' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,false)
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false,true)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
+
+    describe 'with the pending_vm wakeup option' do
+      let(:wakeup_option) {{
+        :pending_vm => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a pending_vm is detected' do
+
+        it 'should sleep until the pending instance' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(0,0,1)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
+
+    describe 'with the ondemand_request wakeup option' do
+      let(:wakeup_option) {{ :ondemand_request => true }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      it 'should sleep until the provisioning request is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,1)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
+        end
+
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+
+      it 'should sleep until provisioning processing is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,1)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
+        end
+
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+
+      it 'should sleep until ondemand creation task is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,1)
+        end
+
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+    end
   end
 
   describe "#check_pool" do
@@ -2963,6 +3849,18 @@ EOT
       end
     end
 
+    context 'when redis connection fails' do
+      let(:maxloop) { 2 }
+      let(:loop_delay) { 1 }
+
+      it 'should raise the error' do
+        allow(subject).to receive(:_check_pool).and_raise(Redis::CannotConnectError)
+        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")
+
+        expect{subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)}.to raise_error Redis::CannotConnectError
+      end
+    end
+
     context 'delays between loops configured in the pool configuration' do
       let(:maxloop) { 2 }
       let(:loop_delay) { 1 }
@@ -3125,6 +4023,54 @@ EOT
     end
   end
 
+  describe 'undo_override' do
+    let(:mutex) { Mutex.new }
+    let(:original_template) { 'templates/template1' }
+    let(:override_template) { 'templates/template2' }
+    let(:original_size) { 2 }
+    let(:override_size) { 10 }
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+  task_limit: 5
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: #{override_size}
+    template: '#{override_template}'
+:pool_index:
+  '#{pool}': 0
+:pools_at_startup:
+  - name: '#{pool}'
+    size: #{original_size}
+    template: '#{original_template}'
+EOT
+      )
+    }
+
+    before(:each) do
+      redis_connection_pool.with do |redis|
+        redis.sadd('vmpooler__pool__undo_template_override', pool)
+        redis.sadd('vmpooler__pool__undo_size_override', pool)
+        # allow(redis).to receive(:hget)
+      end
+    end
+
+    it 'should revert to the original template and pool size' do
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(true)
+        expect(redis).to receive(:srem).with('vmpooler__pool__undo_template_override', pool).and_return(true)
+        expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, original_template, override_template, redis)
+
+        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(true)
+        expect(redis).to receive(:srem).with('vmpooler__pool__undo_size_override', pool).and_return(true)
+      end
+
+      subject.undo_override(config[:pools][0], provider)
+    end
+  end
+
   describe '#create_inventory' do
 
     it 'should log an error if one occurs' do
@@ -3149,7 +4095,9 @@ EOT
         {}
       }
       before(:each) do
-        create_running_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          create_running_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should not call check_running_vm' do
@@ -3159,7 +4107,9 @@ EOT
       end
 
       it 'should move the VM to completed queue' do
-        expect(subject).to receive(:move_vm_queue).with(pool,vm,'running','completed',String).and_call_original
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:move_vm_queue).with(pool,vm,'running','completed',redis,String).and_call_original
+        end
 
         subject.check_running_pool_vms(pool,provider, pool_check_response, inventory)
       end
@@ -3172,8 +4122,10 @@ EOT
         { vm => 1 }
       }
       before(:each) do
-        allow(subject).to receive(:check_running_vm)
-        create_running_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          allow(subject).to receive(:check_running_vm)
+          create_running_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should log an error if one occurs' do
@@ -3191,8 +4143,9 @@ EOT
 
       it 'should use the VM lifetime in preference to defaults' do
         big_lifetime = 2000
-
-        redis.hset("vmpooler__vm__#{vm}", 'lifetime',big_lifetime)
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'lifetime',big_lifetime)
+        end
         # The lifetime comes in as string
         expect(subject).to receive(:check_running_vm).with(vm,pool,"#{big_lifetime}",provider)
 
@@ -3216,6 +4169,7 @@ EOT
 
   describe '#check_ready_pool_vms' do
     let(:provider) { double('provider') }
+    let(:pool_ttl) { 5 }
     let(:pool_check_response) {
       {:checked_ready_vms => 0}
     }
@@ -3226,19 +4180,23 @@ EOT
         {}
       }
       before(:each) do
-        create_ready_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should not call check_ready_vm' do
         expect(subject).to receive(:check_ready_vm).exactly(0).times
 
-        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
       end
 
       it 'should move the VM to completed queue' do
-        expect(subject).to receive(:move_vm_queue).with(pool,vm,'ready','completed',String).and_call_original
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:move_vm_queue).with(pool,vm,'ready','completed',redis,String).and_call_original
+        end
 
-        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
       end
     end
 
@@ -3249,12 +4207,14 @@ EOT
       }
       let(:big_lifetime) { 2000 }
       before(:each) do
-        allow(subject).to receive(:check_ready_vm)
-        create_ready_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          allow(subject).to receive(:check_ready_vm)
+          create_ready_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should return the number of checked ready VMs' do
-        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
 
         expect(pool_check_response[:checked_ready_vms]).to be(1)
       end
@@ -3273,15 +4233,16 @@ EOT
       end
 
       it 'should use a pool TTL of zero if none set' do
-        expect(subject).to receive(:check_ready_vm).with(vm,pool,0,provider)
+        expect(subject).to receive(:check_ready_vm).with(vm,pool,pool_ttl,provider)
 
-        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
       end
     end
   end
 
   describe '#check_pending_pool_vms' do
     let(:provider) { double('provider') }
+    let(:timeout) { 10 }
     let(:pool_check_response) {
       {:checked_pending_vms => 0}
     }
@@ -3293,13 +4254,17 @@ EOT
       }
 
       before(:each) do
-        create_pending_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          create_pending_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should call fail_pending_vm' do
-        expect(subject).to receive(:fail_pending_vm).with(vm,pool,Integer,false)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:fail_pending_vm).with(vm, pool, Integer, Integer, redis, exists: false)
+        end
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
     end
 
@@ -3310,12 +4275,14 @@ EOT
       }
 
       before(:each) do
-        create_pending_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          create_pending_vm(pool,vm,redis,token)
+        end
       end
 
       it 'should return the number of checked pending VMs' do
         allow(subject).to receive(:check_pending_vm)
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
 
         expect(pool_check_response[:checked_pending_vms]).to be(1)
       end
@@ -3324,31 +4291,31 @@ EOT
         expect(subject).to receive(:check_pending_vm).and_raise(RuntimeError,'MockError')
         expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating pending VMs: MockError")
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
 
       it 'should use the pool timeout if set' do
         big_lifetime = 2000
 
         config[:pools][0]['timeout'] = big_lifetime
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,big_lifetime,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm,pool, big_lifetime, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
       end
 
       it 'should use the configuration setting if the pool timeout is not set' do
         big_lifetime = 2000
 
         config[:config]['timeout'] = big_lifetime
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,big_lifetime,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm, pool, big_lifetime, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
       end
 
       it 'should use a pool timeout of 15 if nothing is set' do
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,15,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
     end
   end
@@ -3367,7 +4334,9 @@ EOT
       }
 
       before(:each) do
-        create_completed_vm(vm,pool,true)
+        redis_connection_pool.with do |redis|
+          create_completed_vm(vm,pool,redis,true)
+        end
       end
 
       it 'should log a message' do
@@ -3381,15 +4350,17 @@ EOT
       end
 
       it 'should remove redis information' do
-        expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
-        expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)
+        redis_connection_pool.with do |redis|
+          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
+          expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)
 
-        subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
+          subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
 
-        expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
-        expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
-        expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
+          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
+          expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
+        end
       end
     end
 
@@ -3400,7 +4371,9 @@ EOT
       }
 
       before(:each) do
-        create_completed_vm(vm,pool,true)
+        redis_connection_pool.with do |redis|
+          create_completed_vm(vm,pool,redis,true)
+        end
       end
 
       it 'should call destroy_vm' do
@@ -3426,15 +4399,17 @@ EOT
         end
 
         it 'should remove redis information' do
-          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
-          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
-          expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)
+          redis_connection_pool.with do |redis|
+            expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
+            expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)
 
-          subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
+            subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
 
-          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
-          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
-          expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
+            expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
+            expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
+          end
         end
       end
     end
@@ -3443,17 +4418,23 @@ EOT
   describe "#check_discovered_pool_vms" do
     context 'Discovered VM' do
       before(:each) do
-        create_discovered_vm(vm,pool)
+        redis_connection_pool.with do |redis|
+          create_discovered_vm(vm,pool,redis)
+        end
       end
 
       it 'should be moved to the completed queue' do
         subject.check_discovered_pool_vms(pool)
 
-        expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+        redis_connection_pool.with do |redis|
+          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+        end
       end
 
       it 'should log a message if an error occurs' do
-        expect(redis).to receive(:smove).with("vmpooler__discovered__#{pool}", "vmpooler__completed__#{pool}", vm).and_raise(RuntimeError,'MockError')
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:smove).with("vmpooler__discovered__#{pool}", "vmpooler__completed__#{pool}", vm).and_raise(RuntimeError,'MockError')
+        end
         expect(logger).to receive(:log).with("d", "[!] [#{pool}] _check_pool failed with an error while evaluating discovered VMs: MockError")
 
         subject.check_discovered_pool_vms(pool)
@@ -3471,25 +4452,31 @@ EOT
           end
 
           it "should remain in the #{queue_name} queue" do
-            redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
-            allow(logger).to receive(:log)
+            redis_connection_pool.with do |redis|
+              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
+              allow(logger).to receive(:log)
 
-            subject.check_discovered_pool_vms(pool)
+              subject.check_discovered_pool_vms(pool)
 
-            expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", vm)).to be(true)
+              expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", vm)).to be(true)
+            end
           end
 
           it "should be removed from the discovered queue" do
-            redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
-            allow(logger).to receive(:log)
+            redis_connection_pool.with do |redis|
+              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
+              allow(logger).to receive(:log)
 
-            expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(true)
-            subject.check_discovered_pool_vms(pool)
-            expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(false)
+              expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(true)
+              subject.check_discovered_pool_vms(pool)
+              expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(false)
+            end
           end
 
           it "should log a message" do
-            redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
+            redis_connection_pool.with do |redis|
+              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
+            end
             expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' found in '#{queue_name}', removed from 'discovered' queue")
 
             subject.check_discovered_pool_vms(pool)
@@ -3513,7 +4500,9 @@ EOT
       }
 
       before(:each) do
-        create_migrating_vm(vm,pool)
+        redis_connection_pool.with do |redis|
+          create_migrating_vm(vm,pool,redis)
+        end
       end
 
       it 'should not do anything' do
@@ -3530,7 +4519,9 @@ EOT
       }
 
       before(:each) do
-        create_migrating_vm(vm,pool)
+        redis_connection_pool.with do |redis|
+          create_migrating_vm(vm,pool,redis)
+        end
       end
 
       it 'should return the number of migrated VMs' do
@@ -3556,7 +4547,7 @@ EOT
   end
 
   describe '#repopulate_pool_vms' do
-    let(:pool_size) { 0 }
+    let(:pool_size) { 1 }
     let(:config) {
       YAML.load(<<-EOT
 ---
@@ -3572,14 +4563,20 @@ EOT
       }
     }
 
+    before(:each) do
+      allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+    end
+
     it 'should not call clone_vm when number of VMs is equal to the pool size' do
       expect(subject).to receive(:clone_vm).exactly(0).times
 
-      subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
+      subject.repopulate_pool_vms(pool, provider, pool_check_response, 0)
     end
 
     it 'should not call clone_vm when number of VMs is greater than the pool size' do
-      create_ready_vm(pool,vm,token)
+      redis_connection_pool.with do |redis|
+        create_ready_vm(pool,vm,redis,token)
+      end
       expect(subject).to receive(:clone_vm).exactly(0).times
 
       subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
@@ -3589,9 +4586,11 @@ EOT
       it "should use VMs in #{queue_name} queue to calculate pool size" do
         expect(subject).to receive(:clone_vm).exactly(0).times
         # Modify the pool size to 1 and add a VM in the queue
-        redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
+        redis_connection_pool.with do |redis|
+          redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
+        end
         pool_size = 1
-  
+
         subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
       end
     end
@@ -3600,7 +4599,9 @@ EOT
       it "should not use VMs in #{queue_name} queue to calculate pool size" do
         expect(subject).to receive(:clone_vm)
         # Modify the pool size to 1 and add a VM in the queue
-        redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
+        redis_connection_pool.with do |redis|
+          redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
+        end
         pool_size = 1
 
         subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
@@ -3613,10 +4614,21 @@ EOT
       subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
     end
 
+    it 'should not log a message when the pool size is 0' do
+      expect(logger).to_not receive(:log).with('s', "[!] [#{pool}] is empty")
+
+      subject.repopulate_pool_vms(pool, provider, pool_check_response, 0)
+    end
+
     context 'when pool is marked as empty' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
 
       before(:each) do
-        redis.set("vmpooler__empty__#{pool}", 'true')
+        redis_connection_pool.with do |redis|
+          redis.set("vmpooler__empty__#{pool}", 'true')
+        end
       end
 
       it 'should not log a message when the pool remains empty' do
@@ -3626,15 +4638,20 @@ EOT
       end
 
       it 'should remove the empty pool mark if it is no longer empty' do
-        create_ready_vm(pool,vm,token)
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool,vm,redis,token)
 
-        expect(redis.get("vmpooler__empty__#{pool}")).to be_truthy
-        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
-        expect(redis.get("vmpooler__empty__#{pool}")).to be_falsey
+          expect(redis.get("vmpooler__empty__#{pool}")).to be_truthy
+          subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
+          expect(redis.get("vmpooler__empty__#{pool}")).to be_falsey
+        end
       end
     end
 
     context 'when number of VMs is less than the pool size' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
 
       it 'should return the number of cloned VMs' do
         pool_size = 5
@@ -3663,11 +4680,13 @@ EOT
       end
 
       it 'log a message if a cloning error occurs' do
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool,'vm',redis)
+        end
         pool_size = 2
 
         expect(subject).to receive(:clone_vm).and_raise(RuntimeError,"MockError")
         expect(logger).to receive(:log).with("s", "[!] [#{pool}] clone failed during check_pool with an error: MockError")
-        create_ready_vm(pool,'vm')
         expect{ subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size) }.to raise_error(RuntimeError,'MockError')
 
       end
@@ -3689,10 +4708,16 @@ EOT
     end
 
     context 'export metrics' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       it 'increments metrics for ready queue' do
-        create_ready_vm(pool,'vm1')
-        create_ready_vm(pool,'vm2')
-        create_ready_vm(pool,'vm3')
+        redis_connection_pool.with do |redis|
+          create_ready_vm(pool,'vm1',redis)
+          create_ready_vm(pool,'vm2',redis)
+          create_ready_vm(pool,'vm3',redis)
+        end
 
         expect(metrics).to receive(:gauge).with("ready.#{pool}", 3)
         allow(metrics).to receive(:gauge)
@@ -3701,9 +4726,11 @@ EOT
       end
 
       it 'increments metrics for running queue' do
-        create_running_vm(pool,'vm1',token)
-        create_running_vm(pool,'vm2',token)
-        create_running_vm(pool,'vm3',token)
+        redis_connection_pool.with do |redis|
+          create_running_vm(pool,'vm1',redis,token)
+          create_running_vm(pool,'vm2',redis,token)
+          create_running_vm(pool,'vm3',redis,token)
+        end
 
         expect(metrics).to receive(:gauge).with("running.#{pool}", 3)
         allow(metrics).to receive(:gauge)
@@ -3713,6 +4740,7 @@ EOT
 
       it 'increments metrics with 0 when pool empty' do
 
+        allow(metrics).to receive(:gauge)
         expect(metrics).to receive(:gauge).with("ready.#{pool}", 0)
         expect(metrics).to receive(:gauge).with("running.#{pool}", 0)
 
@@ -3774,6 +4802,7 @@ EOT
         allow(subject).to receive(:destroy_vm)
         allow(subject).to receive(:clone_vm)
         allow(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response)
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'calls inventory correctly' do
@@ -3822,30 +4851,34 @@ EOT
       end
 
       it 'should add undiscovered VMs to the completed queue' do
-        allow(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue")
+        redis_connection_pool.with do |redis|
+          allow(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue")
 
-        expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
-        expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(false)
+          expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
+          expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(false)
 
-        subject._check_pool(pool_object,provider)
+          subject._check_pool(pool_object,provider)
 
-        expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
-        expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
+          expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
+          expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
+        end
       end
 
       ['running','ready','pending','completed','discovered','migrating'].each do |queue_name|
         it "should not discover VMs in the #{queue_name} queue" do
-          expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue").exactly(0).times
-          expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
-          redis.sadd("vmpooler__#{queue_name}__#{pool}", new_vm)
+          redis_connection_pool.with do |redis|
+            expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue").exactly(0).times
+            expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
+            redis.sadd("vmpooler__#{queue_name}__#{pool}", new_vm)
 
-          subject._check_pool(pool_object,provider)
+            subject._check_pool(pool_object,provider)
 
-          if queue_name == 'discovered'
-            # Discovered VMs end up in the completed queue
-            expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
-          else
-            expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", new_vm)).to be(true)
+            if queue_name == 'discovered'
+              # Discovered VMs end up in the completed queue
+              expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
+            else
+              expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", new_vm)).to be(true)
+            end
           end
         end
       end
@@ -3874,6 +4907,10 @@ EOT
 
     # RUNNING
     context 'when checking running VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -3896,6 +4933,10 @@ EOT
 
     # READY
     context 'when checking ready VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -3918,6 +4959,10 @@ EOT
 
     # PENDING
     context 'when checking ready VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -3932,7 +4977,7 @@ EOT
 
       it 'should call #check_ready_pool_vms' do
         allow(subject).to receive(:create_inventory).and_return({})
-        expect(subject).to receive(:check_pending_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['timeout'])
+        expect(subject).to receive(:check_pending_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['timeout'], pool_object['timeout_notification'])
 
         subject._check_pool(pool_object,provider)
       end
@@ -3941,6 +4986,10 @@ EOT
 
     # COMPLETED
     context 'when checking completed VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
             discovered_vms: 0,
@@ -3963,6 +5012,10 @@ EOT
 
     # DISCOVERED
     context 'when checking discovered VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
             discovered_vms: 0,
@@ -3985,6 +5038,10 @@ EOT
 
     # MIGRATIONS
     context 'when checking migrating VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -4011,7 +5068,10 @@ EOT
       let(:newpoolsize) { 3 }
       before(:each) do
         config[:pools][0]['size'] = poolsize
-        redis.hset('vmpooler__config__poolsize', pool, newpoolsize)
+        redis_connection_pool.with do |redis|
+          redis.hset('vmpooler__config__poolsize', pool, newpoolsize)
+        end
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'should change the pool size configuration' do
@@ -4048,9 +5108,12 @@ EOT
     context 'when an excess number of ready vms exist' do
 
       before(:each) do
-        allow(redis).to receive(:scard)
-        expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
-        expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
+        redis_connection_pool.with do |redis|
+          allow(redis).to receive(:scard)
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
+          expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
+        end
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'should call remove_excess_vms' do
@@ -4060,9 +5123,530 @@ EOT
         subject._check_pool(config[:pools][0],provider)
       end
     end
+  end
 
-    #
+  describe 'process_ondemand_requests' do
+    context 'with no requests' do
+      it 'returns 0' do
+        result = subject.process_ondemand_requests
+        expect(result).to eq(0)
+      end
 
+      it 'runs process_ondemand_vms' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:process_ondemand_vms).with(redis).and_return(0)
+          subject.process_ondemand_requests
+        end
+      end
 
+      it 'checks ready requests' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:check_ondemand_requests_ready).with(redis).and_return(0)
+          subject.process_ondemand_requests
+        end
+      end
+    end
+
+    context 'with provisioning requests' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.zadd('vmpooler__provisioning__request', current_time, request_id)
+        end
+      end
+
+      it 'returns the number of requests processed' do
+        result = subject.process_ondemand_requests
+        expect(result).to eq(1)
+      end
+
+      it 'runs create_ondemand_vms for each request' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:create_ondemand_vms).with(request_id, redis)
+          subject.process_ondemand_requests
+        end
+      end
+    end
+  end
+
+  describe '#create_ondemand_vms' do
+    context 'when requested does not have corresponding data' do
+      it 'logs an error' do
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "Failed to find odrequest for request_id '1111'")
+          subject.create_ondemand_vms('1111', redis)
+        end
+      end
+    end
+
+    context 'when request is already marked as failed' do
+      let(:request_string) { "#{pool}:#{pool}:1" }
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+          set_ondemand_request_status(request_id, 'failed', redis)
+        end
+      end
+
+      it 'logs that the request is already failed' do
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "Request '#{request_id}' already marked as failed, skipping VM creation")
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+
+      it 'removes the request from provisioning__request queue' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zscore('vmpooler__provisioning__request', request_id)).to be_nil
+        end
+      end
+
+      it 'does not create VM tasks' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zcard('vmpooler__odcreate__task')).to eq(0)
+        end
+      end
+
+      it 'does not add to provisioning__processing queue' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zscore('vmpooler__provisioning__processing', request_id)).to be_nil
+        end
+      end
+    end
+
+    context 'with a request that has data' do
+      let(:request_string) { "#{pool}:#{pool}:1" }
+      before(:each) do
+        expect(Time).to receive(:now).and_return(current_time).at_least(:once)
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+        end
+      end
+
+      it 'creates tasks for instances to be provisioned' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            allow(pipe).to receive(:zadd)
+            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}")
+          end
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+
+      it 'adds a member to provisioning__processing' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            allow(pipe).to receive(:zadd)
+            expect(pipe).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id)
+          end
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+    end
+  end
+
+  describe '#process_ondemand_vms' do
+    it 'returns the length of the queue' do
+      redis_connection_pool.with do |redis|
+        result = subject.process_ondemand_vms(redis)
+        expect(result).to eq(0)
+      end
+    end
+
+    context 'with a request to create a single vm' do
+      let(:request_string) { "#{pool}:#{pool}:1" }
+      let(:pool_alias) { pool }
+      before(:each) do
+        config[:config]['ondemand_clone_limit'] = 10
+        expect(subject).to receive(:get_provider_for_pool).and_return(provider)
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+        redis_connection_pool.with do |redis|
+          create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
+        end
+      end
+
+      it 'creates the vm' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias)
+          subject.process_ondemand_vms(redis)
+        end
+      end
+    end
+
+    context 'with a request to create more instances than the limit' do
+      let(:request_string) { "#{pool}:#{pool}:5" }
+      let(:request_string_remaining) { "#{pool}:#{pool}:2" }
+      let(:pool_alias) { pool }
+      before(:each) do
+        config[:config]['ondemand_clone_limit'] = 3
+        expect(subject).to receive(:get_provider_for_pool).and_return(provider)
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+        redis_connection_pool.with do |redis|
+          create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
+        end
+      end
+
+      it 'should create the maximum number of vms' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias).exactly(3).times
+          subject.process_ondemand_vms(redis)
+        end
+      end
+
+      it 'should add the remaining number back as a new create task with the same score' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string_remaining}:#{request_id}")
+          end
+          subject.process_ondemand_vms(redis)
+        end
+      end
+
+      it 'should return the number of requests processed' do
+        redis_connection_pool.with do |redis|
+          result = subject.process_ondemand_vms(redis)
+          expect(result).to eq(1)
+        end
+      end
+    end
+
+    context 'when the limit has been reached' do
+      let(:clone_count) { { 'ondemand_clone_count' => 3 } }
+      before(:each) do
+        config[:config]['ondemand_clone_limit'] = 3
+        subject.instance_variable_set(:@tasks, clone_count)
+      end
+
+      it 'does not create any instances' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to_not receive(:clone_vm)
+          subject.process_ondemand_vms(redis)
+        end
+      end
+    end
+  end
+
+  describe '#vms_ready?' do
+    let(:request_string) { "#{pool}:#{pool}:5" }
+    let(:platform_alias) { pool }
+    before(:each) do
+      redis_connection_pool.with do |redis|
+        create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+      end
+    end
+
+    it 'fails if the request is not valid' do
+      redis_connection_pool.with do |redis|
+        request_id = "#{request_id}-wrong"
+        create_ondemand_request_for_test(request_id, current_time.to_i, "#{pool}:5", redis)
+        expect{subject.vms_ready?(request_id, redis)}.to raise_error(ArgumentError)
+      end
+    end
+
+    it 'returns false when vms for request_id are not ready' do
+      redis_connection_pool.with do |redis|
+        result = subject.vms_ready?(request_id, redis)
+        expect(result).to be false
+      end
+    end
+
+    context 'with a request that has all instances ready' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          ['vm1','vm2','vm3','vm4','vm5'].each do |v|
+            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
+          end
+        end
+      end
+
+      it 'returns true' do
+        redis_connection_pool.with do |redis|
+          result = subject.vms_ready?(request_id, redis)
+          expect(result).to be true
+        end
+      end
+    end
+
+    context 'with a request that has some instances ready' do
+      let(:request_string) { "#{pool}:#{pool}:3,#{pool}2:#{pool}2:3" }
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+          ['vm1','vm2','vm3'].each do |v|
+            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
+          end
+        end
+      end
+
+      it 'returns false' do
+        redis_connection_pool.with do |redis|
+          result = subject.vms_ready?(request_id, redis)
+          expect(result).to be false
+        end
+      end
+    end
+  end
+
+  describe '#check_ondemand_requests_ready' do
+    before(:each) do
+      config[:config]['ondemand_request_ttl'] = 5
+    end
+
+    it 'returns 0 when no provisoning requests are in progress' do
+      redis_connection_pool.with do |redis|
+        result = subject.check_ondemand_requests_ready(redis)
+        expect(result).to eq(0)
+      end
+    end
+
+    context 'with requests in progress' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_processing(request_id, current_time, redis)
+        end
+      end
+
+      it 'returns the number of requests processed' do
+        expect(subject).to receive(:vms_ready?).and_return(false)
+        redis_connection_pool.with do |redis|
+          result = subject.check_ondemand_requests_ready(redis)
+          expect(result).to eq(1)
+        end
+      end
+    end
+  end
+
+  describe '#check_ondemand_request_ready' do
+    let(:score) { current_time.to_f }
+    before(:each) do
+      config[:config]['ondemand_request_ttl'] = 5
+    end
+
+    context 'when the request is ready' do
+      before(:each) do
+        expect(subject).to receive(:vms_ready?).and_return(true)
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:zscore).and_return(score)
+        end
+      end
+
+      it 'sets the request as ready' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'ready')
+          subject.check_ondemand_request_ready(request_id, redis)
+        end
+      end
+
+      it 'marks the ondemand request hash key for expiration in one month' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{request_id}", 2592000)
+          subject.check_ondemand_request_ready(request_id, redis)
+        end
+      end
+
+      it 'removes the request from processing' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
+          subject.check_ondemand_request_ready(request_id, redis)
+        end
+      end
+    end
+
+    context 'with the score provided' do
+      it 'should not request the score' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to_not receive(:zscore)
+          expect(subject).to receive(:vms_ready?).and_return(true)
+          expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
+          subject.check_ondemand_request_ready(request_id, redis, score)
+        end
+      end
+    end
+
+    context 'when a request has taken too long to be filled' do
+      it 'should return true for request_expired?' do
+        redis_connection_pool.with do |redis|
+          expect(redis).to receive(:zscore).and_return(score)
+          expect(subject).to receive(:request_expired?).with(request_id, Float, redis).and_return(true)
+          subject.check_ondemand_request_ready(request_id, redis)
+        end
+      end
+    end
+  end
+
+  describe '#request_expired?' do
+    let(:ondemand_request_ttl) { 5 }
+    let(:expiration_ttl) { 10 }
+    before(:each) do
+      config[:config]['ondemand_request_ttl'] = ondemand_request_ttl
+      config[:redis]['data_ttl'] = expiration_ttl
+    end
+
+    context 'with a request that has taken too long to be filled' do
+      let(:expired_time) { (Time.now - 960).to_i }
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:remove_vms_for_failed_request)
+          create_ondemand_processing(request_id, expired_time, redis)
+        end
+      end
+
+      it 'returns true when the request is expired' do
+        redis_connection_pool.with do |redis|
+          result = subject.request_expired?(request_id, expired_time, redis)
+          expect(result).to be true
+        end
+      end
+
+      it 'logs a message that the request has expired' do
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
+          subject.request_expired?(request_id, expired_time, redis)
+        end
+      end
+
+      it 'removes the request from processing requests' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
+          end
+          subject.request_expired?(request_id, expired_time, redis)
+        end
+      end
+
+      it 'sets the status as failed on the request hash' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+          end
+          subject.request_expired?(request_id, expired_time, redis)
+        end
+      end
+
+      it 'marks the request hash for expiration' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__odrequest__#{request_id}", expiration_ttl * 60 * 60)
+          end
+          subject.request_expired?(request_id, expired_time, redis)
+        end
+      end
+    end
+
+    context 'with a request that has been made within the ttl' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_processing(request_id, current_time, redis)
+        end
+      end
+
+      it 'should return false' do
+        redis_connection_pool.with do |redis|
+          result = subject.request_expired?(request_id, current_time, redis)
+          expect(result).to be false
+        end
+      end
+    end
+  end
+
+  describe '#remove_vms_for_failed_request)' do
+    let(:expiration_ttl) { 100 * 60 * 60 }
+    let(:platform_alias) { pool }
+    let(:platforms_string) { "#{platform_alias}:#{pool}:3" }
+    context 'with two vms marked as ready for the request' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time, platforms_string, redis)
+          [vm,"#{vm}2"].each do |v|
+            create_running_vm(pool, v, redis)
+            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
+          end
+        end
+      end
+
+      it 'should remove the associated vms' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(subject).to receive(:move_vm_queue).with(pool, String, 'running', 'completed', pipe, "moved to completed queue. '#{request_id}' could not be filled in time").twice
+          end
+          subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
+        end
+      end
+
+      it 'should mark the ready set for expiration' do
+        redis_connection_pool.with do |redis|
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+          end
+          subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
+        end
+      end
+    end
+  end
+
+  describe 'check_ondemand_requests' do
+    let(:threads) {[]}
+    let(:maxloop) { 0 }
+    let(:loop_delay_min) { 5 }
+    let(:loop_delay_max) { 60 }
+    let(:loop_delay_decay) { 2.0 }
+
+    before(:each) do
+      expect(Thread).to receive(:new).and_yield
+    end
+
+    it 'should log the ondemand provisioner is starting' do
+      expect(subject).to receive(:_check_ondemand_requests).with(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
+      expect(logger).to receive(:log).with('d', "[*] [ondemand_provisioner] starting worker thread")
+
+      expect($threads.count).to be(0)
+      subject.check_ondemand_requests
+      expect($threads.count).to be(1)
+    end
+
+    context' delays between loops' do
+      let(:maxloop) { 2 }
+      let(:loop_delay) { 1 }
+
+      it 'when a non-default loop delay is specified' do
+        expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay, Numeric, Hash).exactly(maxloop).times
+
+        subject.check_ondemand_requests(maxloop,loop_delay,loop_delay)
+      end
+    end
+
+    context 'delays between loops with a specified min and max value' do
+      let(:maxloop) { 5 }
+      let(:loop_delay_min) { 1 }
+      let(:loop_delay_max) { 60 }
+      let(:loop_decay) { 3.0 }
+
+      it 'delay values increase with a decay' do
+        expect(subject).to receive(:sleep_with_wakeup_events).with(3, Numeric, Hash).once
+        expect(subject).to receive(:sleep_with_wakeup_events).with(9, Numeric, Hash).once
+        expect(subject).to receive(:sleep_with_wakeup_events).with(27, Numeric, Hash).once
+        expect(subject).to receive(:sleep_with_wakeup_events).with(60, Numeric, Hash).twice
+
+        subject.check_ondemand_requests(maxloop,loop_delay_min,loop_delay_max,loop_decay)
+      end
+    end
+
+    context 'loops specified number of times (5)' do
+      let(:maxloop) { 5 }
+
+      it 'should run startup tasks only once' do
+        expect(logger).to receive(:log).with('d', "[*] [ondemand_provisioner] starting worker thread")
+
+        subject.check_ondemand_requests(maxloop,0)
+      end
+
+      it 'should run per thread tasks 5 times' do
+        expect(subject).to receive(:process_ondemand_requests).and_return(0).exactly(maxloop).times
+
+        subject.check_ondemand_requests(maxloop,0)
+      end
+    end
   end
 end
diff --git a/spec/unit/promstats_spec.rb b/spec/unit/promstats_spec.rb
new file mode 100644
index 0000000..9075536
--- /dev/null
+++ b/spec/unit/promstats_spec.rb
@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'prometheus' do
+  logger = MockLogger.new
+  params = { 'prefix' => 'test', 'prometheus_prefix'  => 'mtest', 'prometheus_endpoint'  => 'eptest' }
+  subject = Vmpooler::Metrics::Promstats.new(logger, params)
+  let(:logger) { MockLogger.new }
+
+  describe '#initialise' do
+    it 'returns a Metrics object' do
+      expect(Vmpooler::Metrics::Promstats.new(logger)).to be_a(Vmpooler::Metrics)
+    end
+  end
+
+  describe '#find_metric' do
+    context "Single Value Parameters" do
+      let!(:foo_metrics) do
+        { metric_suffixes: { bar: 'baz' },
+          param_labels: %i[first second last] }
+      end
+      let!(:labels_hash) { { labels: { :first => nil, :second => nil, :last => nil } } }
+      before { 
+        subject.instance_variable_set(:@p_metrics, { foo: foo_metrics, torun: %i[api] }) 
+        subject.instance_variable_set(:@torun, [ :api ]) 
+      }
+  
+      it 'returns the metric for a given label including parsed labels' do
+        expect(subject.find_metric('foo.bar')).to include(metric_name: 'mtest_foo_bar')
+        expect(subject.find_metric('foo.bar')).to include(foo_metrics)
+        expect(subject.find_metric('foo.bar')).to include(labels_hash)
+      end
+
+      it 'raises an error when the given label is not present in metrics' do
+        expect { subject.find_metric('bogus') }.to raise_error(RuntimeError, 'Invalid Metric bogus for bogus')
+      end
+  
+      it 'raises an error when the given label specifies metric_suffixes but the following suffix not present in metrics' do
+        expect { subject.find_metric('foo.metric_suffixes.bogus') }.to raise_error(RuntimeError, 'Invalid Metric foo_metric_suffixes for foo.metric_suffixes.bogus')
+      end
+    end
+
+    context "Node Name Handling" do
+      let!(:node_metrics) do
+        { metric_name: 'connection_to',
+          param_labels: %i[node],
+          torun: %i[api] 
+        }
+      end
+      let!(:nodename_hash) { { labels: { :node => 'test.bar.net'}}}
+      before { 
+        subject.instance_variable_set(:@p_metrics, { connection_to: node_metrics })
+        subject.instance_variable_set(:@torun, [ :api ]) 
+      }
+
+      it 'Return final remaining fields (e.g. fqdn) in last label' do
+        expect(subject.find_metric('connection_to.test.bar.net')).to include(nodename_hash)
+      end
+    end
+  end
+
+  context 'setup_prometheus_metrics' do
+    before(:all) do
+      Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+      subject.setup_prometheus_metrics(%i[api manager])
+    end
+
+    describe '#setup_prometheus_metrics' do
+      it 'calls add_prometheus_metric for each item in list' do
+        Prometheus::Client.config.data_store = Prometheus::Client::DataStores::Synchronized.new
+        expect(subject).to receive(:add_prometheus_metric).at_least(subject.vmpooler_metrics_table.size).times
+        subject.setup_prometheus_metrics(%i[api manager])
+      end
+    end
+
+    describe '#increment' do
+      it 'Increments checkout.nonresponsive.#{template_backend}' do
+        template_backend = 'test'
+        expect { subject.increment("checkout.nonresponsive.#{template_backend}") }.to change {
+          metric, po = subject.get("checkout.nonresponsive.#{template_backend}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.empty. + requested' do
+        requested = 'test'
+        expect { subject.increment('checkout.empty.' + requested) }.to change {
+          metric, po = subject.get('checkout.empty.' + requested)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.success. + vmtemplate' do
+        vmtemplate = 'test-template'
+        expect { subject.increment('checkout.success.' + vmtemplate) }.to change {
+          metric, po = subject.get('checkout.success.' + vmtemplate)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.invalid. + bad_template' do
+        bad_template = 'test-template'
+        expect { subject.increment('checkout.invalid.' + bad_template) }.to change {
+          metric, po = subject.get('checkout.invalid.' + bad_template)
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments checkout.invalid.unknown' do
+        expect { subject.increment('checkout.invalid.unknown') }.to change {
+          metric, po = subject.get('checkout.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments delete.failed' do
+        bad_template = 'test-template'
+        expect { subject.increment('delete.failed') }.to change {
+          metric, po = subject.get('delete.failed')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments delete.success' do
+        bad_template = 'test-template'
+        expect { subject.increment('delete.success') }.to change {
+          metric, po = subject.get('delete.success')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_generate.duplicaterequests' do
+        bad_template = 'test-template'
+        expect { subject.increment('ondemandrequest_generate.duplicaterequests') }.to change {
+          metric, po = subject.get('ondemandrequest_generate.duplicaterequests')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_generate.success' do
+        bad_template = 'test-template'
+        expect { subject.increment('ondemandrequest_generate.success') }.to change {
+          metric, po = subject.get('ondemandrequest_generate.success')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_fail.toomanyrequests.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("ondemandrequest_fail.toomanyrequests.#{bad_template}") }.to change {
+          metric, po = subject.get("ondemandrequest_fail.toomanyrequests.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments ondemandrequest_fail.invalid.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("ondemandrequest_fail.invalid.#{bad_template}") }.to change {
+          metric, po = subject.get("ondemandrequest_fail.invalid.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments config.invalid.#{bad_template}' do
+        bad_template = 'test-template'
+        expect { subject.increment("config.invalid.#{bad_template}") }.to change {
+          metric, po = subject.get("config.invalid.#{bad_template}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments config.invalid.unknown' do
+        expect { subject.increment('config.invalid.unknown') }.to change {
+          metric, po = subject.get('config.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments poolreset.invalid.#{bad_pool}' do
+        bad_pool = 'test-pool'
+        expect { subject.increment("poolreset.invalid.#{bad_pool}") }.to change {
+          metric, po = subject.get("poolreset.invalid.#{bad_pool}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments poolreset.invalid.unknown' do
+        expect { subject.increment('poolreset.invalid.unknown') }.to change {
+          metric, po = subject.get('poolreset.invalid.unknown')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.markedasfailed.#{pool}' do
+        pool = 'test-pool'
+        expect { subject.increment("errors.markedasfailed.#{pool}") }.to change {
+          metric, po = subject.get("errors.markedasfailed.#{pool}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.duplicatehostname.#{pool_name}' do
+        pool_name = 'test-pool'
+        expect { subject.increment("errors.duplicatehostname.#{pool_name}") }.to change {
+          metric, po = subject.get("errors.duplicatehostname.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments errors.staledns.#{pool_name}' do
+        pool_name = 'test-pool'
+        expect { subject.increment("errors.staledns.#{pool_name}") }.to change {
+          metric, po = subject.get("errors.staledns.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments user.#{user}.#{operation}.#{poolname}' do
+        user = 'myuser'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("user.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("user.#{user}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments usage_litmus.#{user}.#{operation}.#{poolname}' do
+        user = 'myuser'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_litmus.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_litmus.#{user}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}' do
+        jenkins_instance = 'jenkins_test_instance'
+        value_stream = 'notional_value'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}' do
+        branch = 'treetop'
+        project = 'test-project'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}' do
+        job_name = 'a-job'
+        component_to_test = 'component-name'
+        operation = 'allocate'
+        poolname = 'test-pool'
+        expect { subject.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments connect.open' do
+        expect { subject.increment('connect.open') }.to change {
+          metric, po = subject.get('connect.open')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments connect.fail' do
+        expect { subject.increment('connect.fail') }.to change {
+          metric, po = subject.get('connect.fail')
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments migrate_from.#{vm_hash[\'host_name\']}' do
+        vm_hash = { 'host_name': 'testhost.testdomain' }
+        expect { subject.increment("migrate_from.#{vm_hash['host_name']}") }.to change {
+          metric, po = subject.get("migrate_from.#{vm_hash['host_name']}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments "migrate_to.#{dest_host_name}"' do
+        dest_host_name = 'testhost.testdomain'
+        expect { subject.increment("migrate_to.#{dest_host_name}") }.to change {
+          metric, po = subject.get("migrate_to.#{dest_host_name}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+      it 'Increments label http_requests_vm_total.#{method}.#{subpath}.#{operation}' do
+        method = 'get'
+        subpath = 'template'
+        operation = 'something'
+        expect { subject.increment("http_requests_vm_total.#{method}.#{subpath}.#{operation}") }.to change {
+          metric, po = subject.get("http_requests_vm_total.#{method}.#{subpath}.#{operation}")
+          po.get(labels: metric[:labels])
+        }.by(1)
+      end
+
+    end
+
+    describe '#gauge' do
+      # metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
+      it 'sets value of ready.#{pool_name} to $redis.scard("vmpooler__ready__#{pool_name}"))' do
+        # is there a specific redis value that should be tested?
+        pool_name = 'test-pool'
+        test_value = 42
+        expect { subject.gauge("ready.#{pool_name}", test_value) }.to change {
+          metric, po = subject.get("ready.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.from(0).to(42)
+      end
+      # metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
+      it 'sets value of running.#{pool_name} to $redis.scard("vmpooler__running__#{pool_name}"))' do
+        # is there a specific redis value that should be tested?
+        pool_name = 'test-pool'
+        test_value = 42
+        expect { subject.gauge("running.#{pool_name}", test_value) }.to change {
+          metric, po = subject.get("running.#{pool_name}")
+          po.get(labels: metric[:labels])
+        }.from(0).to(42)
+      end
+    end
+
+    describe '#timing' do
+      it 'sets histogram value of time_to_ready_state.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("time_to_ready_state.#{pool}", finish) }.to change {
+          metric, po = subject.get("time_to_ready_state.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of clone.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("clone.#{pool}", finish) }.to change {
+          metric, po = subject.get("clone.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of migrate.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("migrate.#{pool}", finish) }.to change {
+          metric, po = subject.get("migrate.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+      it 'sets histogram value of destroy.#{pool} to finish' do
+        pool = 'test-pool'
+        finish = 42
+        expect { subject.timing("destroy.#{pool}", finish) }.to change {
+          metric, po = subject.get("destroy.#{pool}")
+          po.get(labels: metric[:labels])
+        }
+      end
+    end
+  end
+end
diff --git a/spec/unit/providers/base_spec.rb b/spec/unit/providers/base_spec.rb
index 5dc71c5..1cc2090 100644
--- a/spec/unit/providers/base_spec.rb
+++ b/spec/unit/providers/base_spec.rb
@@ -6,7 +6,7 @@ require 'vmpooler/providers/base'
 
 describe 'Vmpooler::PoolManager::Provider::Base' do
   let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
   let(:config) { {} }
   let(:provider_name) { 'base' }
   let(:provider_options) { { 'param' => 'value' } }
@@ -22,7 +22,9 @@ describe 'Vmpooler::PoolManager::Provider::Base' do
     fake_vm
   }
 
-  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, provider_name, provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, provider_name, provider_options) }
 
   # Helper attr_reader methods
   describe '#logger' do
diff --git a/spec/unit/providers/dummy_spec.rb b/spec/unit/providers/dummy_spec.rb
index 13006ce..f4e4849 100644
--- a/spec/unit/providers/dummy_spec.rb
+++ b/spec/unit/providers/dummy_spec.rb
@@ -3,7 +3,7 @@ require 'vmpooler/providers/dummy'
 
 describe 'Vmpooler::PoolManager::Provider::Dummy' do
   let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
   let(:pool_name) { 'pool1' }
   let(:other_pool_name) { 'pool2' }
   let(:vm_name) { 'vm1' }
@@ -91,7 +91,9 @@ EOT
     )
   }
 
-  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, 'dummy', provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, redis_connection_pool, 'dummy', provider_options) }
 
   describe '#name' do
     it 'should be dummy' do
diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb
deleted file mode 100644
index 7e445c2..0000000
--- a/spec/unit/providers/vsphere_spec.rb
+++ /dev/null
@@ -1,3464 +0,0 @@
-require 'spec_helper'
-require 'mock_redis'
-require 'vmpooler/providers/vsphere'
-
-RSpec::Matchers.define :relocation_spec_with_host do |value|
-  match { |actual| actual[:spec].host == value }
-end
-
-RSpec::Matchers.define :create_virtual_disk_with_size do |value|
-  match { |actual| actual[:spec].capacityKb == value * 1024 * 1024 }
-end
-
-RSpec::Matchers.define :create_vm_spec do |new_name,target_folder_name,datastore|
-  match { |actual|
-    # Should have the correct new name
-    actual[:name] == new_name &&
-    # Should be in the new folder
-    actual[:folder].name == target_folder_name &&
-    # Should be poweredOn after clone
-    actual[:spec].powerOn == true &&
-    # Should be on the correct datastore
-    actual[:spec][:location].datastore.name == datastore &&
-    # Should contain annotation data
-    actual[:spec][:config].annotation != '' &&
-    # Should contain VIC information
-    actual[:spec][:config].extraConfig[0][:key] == 'guestinfo.hostname' &&
-    actual[:spec][:config].extraConfig[0][:value] == new_name
-  }
-end
-
-RSpec::Matchers.define :create_snapshot_spec do |new_snapshot_name|
-  match { |actual|
-    # Should have the correct new name
-    actual[:name] == new_snapshot_name &&
-    # Should snapshot the memory too
-    actual[:memory] == true &&
-    # Should quiesce the disk
-    actual[:quiesce] == true
-  }
-end
-
-describe 'Vmpooler::PoolManager::Provider::VSphere' do
-  let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::DummyStatsd.new }
-  let(:poolname) { 'pool1'}
-  let(:provider_options) { { 'param' => 'value' } }
-  let(:datacenter_name) { 'MockDC' }
-  let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_tries: 3
-  retry_factor: 10
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    insecure: true
-    # Drop the connection pool timeout way down for spec tests so they fail fast
-    connection_pool_timeout: 1
-    datacenter: MockDC
-    create_linked_clones: true
-:pools:
-  - name: '#{poolname}'
-    alias: [ 'mockpool' ]
-    template: 'Templates/pool1'
-    folder: 'Pooler/pool1'
-    datastore: 'datastore0'
-    size: 5
-    timeout: 10
-    ready_ttl: 1440
-    clone_target: 'cluster1'
-EOT
-    )
-  }
-
-  let(:connection_options) {{}}
-  let(:connection) { mock_RbVmomi_VIM_Connection(connection_options) }
-  let(:vmname) { 'vm1' }
-  let(:redis) { MockRedis.new }
-
-  subject { Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, 'vsphere', provider_options) }
-
-  before(:each) do
-    allow(subject).to receive(:vsphere_connection_ok?).and_return(true)
-  end
-
-  describe '#name' do
-    it 'should be vsphere' do
-      expect(subject.name).to eq('vsphere')
-    end
-  end
-
-  describe '#folder_configured?' do
-    let(:folder_title) { 'folder1' }
-    let(:other_folder) { 'folder2' }
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:configured_folders) { { folder_title => base_folder } }
-    let(:whitelist) { nil }
-    it 'should return true when configured_folders includes the folder_title' do
-      expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true
-    end
-
-    it 'should return false when title is not in configured_folders' do
-      expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false
-    end
-
-    context 'with another base folder' do
-      let(:base_folder) { 'dc2/vm/base' }
-      let(:configured_folders) { { folder_title => 'dc1/vm/base' } }
-      it 'should return false' do
-        expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false
-      end
-    end
-
-    context 'with a whitelist set' do
-      let(:whitelist) { [ other_folder ] }
-      it 'should return true' do
-        expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true
-      end
-    end
-
-    context 'with string whitelist value' do
-      let(:whitelist) { 'whitelist' }
-      it 'should raise an error' do
-        expect(whitelist).to receive(:include?).and_raise('mockerror')
-
-        expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror')
-      end
-    end
-  end
-
-  describe '#destroy_vm_and_log' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-        :powerstate => 'poweredOn',
-      })
-    }
-    let(:pool) { 'pool1' }
-    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-    let(:data_ttl) { 1 }
-    let(:finish) { '0.00' }
-    let(:now) { Time.now }
-
-    context 'when destroying a vm' do
-      before(:each) do
-        allow(power_off_task).to receive(:wait_for_completion)
-        allow(destroy_task).to receive(:wait_for_completion)
-        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-        $redis = redis
-      end
-
-      it 'should remove redis data and expire the vm key' do
-        allow(Time).to receive(:now).and_return(now)
-        expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname)
-        expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname)
-        expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now)
-        expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-
-      it 'should log a message that the vm is destroyed' do
-        expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds")
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-
-      it 'should record metrics' do
-        expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-
-      it 'should power off and destroy the vm' do
-        allow(destroy_task).to receive(:wait_for_completion)
-        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-    end
-
-    context 'with a powered off vm' do
-      before(:each) do
-        vm_object.runtime.powerState = 'poweredOff'
-      end
-
-      it 'should destroy the vm without attempting to power it off' do
-        allow(destroy_task).to receive(:wait_for_completion)
-        expect(vm_object).to_not receive(:PowerOffVM_Task)
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-    end
-
-    context 'with a folder object' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) }
-
-      it 'should log that a folder object was received' do
-        expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying")
-
-        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
-      end
-    end
-
-    context 'with an error that is not a RuntimeError' do
-      it 'should retry three times' do
-        expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times
-
-        expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/)
-      end
-    end
-  end
-
-  describe '#destroy_folder_and_children' do
-    let(:data_ttl) { 1 }
-    let(:config) {
-      {
-        redis: {
-          'data_ttl' => data_ttl
-        }
-      }
-    }
-    let(:foldername) { 'pool1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-
-    before(:each) do
-      $config = config
-    end
-
-    context 'with an empty folder' do
-      it 'should destroy the folder' do
-        expect(subject).to_not receive(:destroy_vm_and_log)
-        expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil)
-
-        subject.destroy_folder_and_children(folder_object)
-      end
-    end
-
-    context 'with a folder containing vms' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-      before(:each) do
-        folder_object.childEntity << vm_object
-      end
-
-      it 'should destroy the vms' do
-        allow(subject).to receive(:destroy_vm_and_log).and_return(nil)
-        allow(subject).to receive(:destroy_folder).and_return(nil)
-        expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl)
-
-        subject.destroy_folder_and_children(folder_object)
-      end
-    end
-
-    it 'should raise any errors' do
-      expect(subject).to receive(:destroy_folder).and_throw('mockerror')
-
-      expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#destroy_folder' do
-    let(:foldername) { 'pool1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
-      allow(destroy_task).to receive(:wait_for_completion)
-    end
-
-    it 'should destroy the folder' do
-      expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-      subject.destroy_folder(folder_object)
-    end
-
-    it 'should log that the folder is being destroyed' do
-      expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder")
-
-      subject.destroy_folder(folder_object)
-    end
-
-    it 'should retry three times when failing' do
-      expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times
-
-      expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#purge_unconfigured_folders' do
-    let(:folder_title) { 'folder1' }
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
-    let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) }
-    let(:whitelist) { nil }
-    let(:base_folders) { [ base_folder ] }
-    let(:configured_folders) { { folder_title => base_folder } }
-    let(:folder_children) { [ folder_title => child_folder ] }
-    let(:empty_list) { [] }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'with an empty folder' do
-      it 'should not attempt to destroy any folders' do
-        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list)
-        expect(subject).to_not receive(:destroy_folder_and_children)
-
-        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-      end
-    end
-
-    it 'should retrieve the folder children' do
-      expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
-      allow(subject).to receive(:folder_configured?).and_return(true)
-
-      subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-    end
-
-    context 'with a folder that is not configured' do
-      before(:each) do
-        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
-        allow(subject).to receive(:folder_configured?).and_return(false)
-      end
-
-      it 'should destroy the folder and children' do
-        expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil)
-
-        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-      end
-    end
-
-    it 'should raise any errors' do
-      expect(subject).to receive(:get_folder_children).and_throw('mockerror')
-
-      expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#get_folder_children' do
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
-    let(:foldername) { 'folder1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-    let(:folder_return) { [ { foldername => folder_object } ] }
-
-    before(:each) do
-      base_folder_object.childEntity << folder_object
-    end
-
-    it 'should return an array of configured folder hashes' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object)
-
-      result = subject.get_folder_children(foldername, connection)
-
-      expect(result).to eq(folder_return)
-    end
-
-    it 'should raise any errors' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror')
-
-      expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#vms_in_pool' do
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-    let(:pool_config) { config[:pools][0] }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'Given a pool folder that is missing' do
-      before(:each) do
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(nil)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should return an empty array' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq([])
-      end
-    end
-
-    context 'Given an empty pool folder' do
-      before(:each) do
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should return an empty array' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq([])
-      end
-    end
-
-    context 'Given a pool folder with many VMs' do
-      let(:expected_vm_list) {[
-        { 'name' => 'vm1'},
-        { 'name' => 'vm2'},
-        { 'name' => 'vm3'}
-      ]}
-      before(:each) do
-        expected_vm_list.each do |vm_hash|
-          mock_vm = mock_RbVmomi_VIM_VirtualMachine({ :name => vm_hash['name'] })
-          # Add the mocked VM to the folder
-          folder_object.childEntity << mock_vm
-        end
-
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should list all VMs in the VM folder for the pool' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq(expected_vm_list)
-      end
-    end
-
-    context 'given a pool folder with a folder and vms' do
-      let(:expected_vm_list) {[
-        { 'name' => 'vm1'},
-        { 'name' => 'vm2'},
-        { 'name' => 'vm3'}
-      ]}
-      let(:folder_object2) { mock_RbVmomi_VIM_Folder({ :name => 'pool2'}) }
-      before(:each) do
-        expected_vm_list.each do |vm_hash|
-          mock_vm = mock_RbVmomi_VIM_VirtualMachine({ :name => vm_hash['name'] })
-          # Add the mocked VM to the folder
-          folder_object.childEntity << mock_vm
-        end
-        folder_object.childEntity << folder_object2
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should return the vms without the folder' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq(expected_vm_list)
-      end
-    end
-  end
-
-  describe '#get_vm' do
-    let(:vm_object) { nil }
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      expect(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      it 'should return nil' do
-        expect(subject.get_vm(poolname,vmname)).to be_nil
-      end
-    end
-
-    context 'when VM exists but is missing information' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-        })
-      }
-
-      it 'should return a hash' do
-        expect(subject.get_vm(poolname,vmname)).to be_kind_of(Hash)
-      end
-
-      it 'should return the VM name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-
-      ['hostname','boottime','powerstate'].each do |testcase|
-        it "should return nil for #{testcase}" do
-          result = subject.get_vm(poolname,vmname)
-
-          expect(result[testcase]).to be_nil
-        end
-      end
-    end
-
-    context 'when VM exists and contains all information' do
-      let(:vm_hostname) { "#{vmname}.demo.local" }
-      let(:boot_time) { Time.now }
-      let(:power_state) { 'MockPowerState' }
-
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :hostname => vm_hostname,
-          :powerstate => power_state,
-          :boottime => boot_time,
-          # This path should match the folder in the mocked pool in the config above
-          :path => [
-            { :type => 'folder',     :name => 'Datacenters' },
-            { :type => 'datacenter', :name => 'DC01' },
-            { :type => 'folder',     :name => 'vm' },
-            { :type => 'folder',     :name => 'Pooler' },
-            { :type => 'folder',     :name => 'pool1'},
-          ]
-        })
-      }
-      let(:pool_info) { config[:pools][0]}
-
-      it 'should return a hash' do
-        expect(subject.get_vm(poolname,vmname)).to be_kind_of(Hash)
-      end
-
-      it 'should return the VM name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-
-      it 'should return the VM hostname' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['hostname']).to eq(vm_hostname)
-      end
-
-      it 'should return the template name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['template']).to eq(pool_info['template'])
-      end
-
-      it 'should return the pool name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['poolname']).to eq(pool_info['name'])
-      end
-
-      it 'should return the boot time' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['boottime']).to eq(boot_time)
-      end
-
-      it 'should return the powerstate' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['powerstate']).to eq(power_state)
-      end
-
-    end
-  end
-
-  describe '#create_vm' do
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-      :datastores => ['datastore0'],
-      :vmfolder_tree => {
-        'Templates' => { :children => {
-          'pool1' => { :object_type => 'vm', :name => 'pool1' },
-        }},
-        'Pooler' => { :children => {
-          'pool1' => nil,
-        }},
-      },
-      :hostfolder_tree => {
-        'cluster1' =>  {:object_type => 'compute_resource'},
-      }
-    })
-    }
-
-    let(:clone_vm_task) { mock_RbVmomi_VIM_Task() }
-    let(:new_vm_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-    let(:new_template_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
-    context 'Given an invalid pool name' do
-      it 'should raise an error' do
-        expect{ subject.create_vm('missing_pool', vmname) }.to raise_error(/missing_pool does not exist/)
-      end
-    end
-
-    context 'Given an invalid template path in the pool config' do
-      before(:each) do
-        config[:pools][0]['template'] = 'bad_template'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template that starts with /' do
-      before(:each) do
-        config[:pools][0]['template'] = '/bad_template'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template that ends with /' do
-      before(:each) do
-        config[:pools][0]['template'] = 'bad_template/'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template VM that does not exist' do
-      before(:each) do
-        config[:pools][0]['template'] = 'Templates/missing_template'
-        expect(subject).to receive(:find_template_vm).and_raise("specifies a template VM of #{vmname} which does not exist")
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/specifies a template VM of .+ which does not exist/)
-      end
-    end
-
-    context 'Given a successful creation' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-      before(:each) do
-        template_vm = new_template_object
-        allow(subject).to receive(:find_template_vm).and_return(new_template_object)
-        allow(template_vm).to receive(:CloneVM_Task).and_return(clone_vm_task)
-        allow(clone_vm_task).to receive(:wait_for_completion).and_return(new_vm_object)
-        allow(subject).to receive(:find_vm_folder).and_return(folder_object)
-      end
-
-      it 'should return a hash' do
-        result = subject.create_vm(poolname, vmname)
-
-        expect(result.is_a?(Hash)).to be true
-      end
-
-      it 'should use the appropriate Create_VM spec' do
-        template_vm = new_template_object
-        expect(template_vm).to receive(:CloneVM_Task)
-          .with(create_vm_spec(vmname,'pool1','datastore0'))
-          .and_return(clone_vm_task)
-
-        subject.create_vm(poolname, vmname)
-      end
-
-      it 'should have the new VM name' do
-        result = subject.create_vm(poolname, vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-    end
-  end
-
-  describe '#create_disk' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-    let(:datastorename) { 'datastore0' }
-    let(:disk_size) { 10 }
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-    end
-
-    context 'Given an invalid pool name' do
-      it 'should raise an error' do
-        expect{ subject.create_disk('missing_pool',vmname,disk_size) }.to raise_error(/missing_pool does not exist/)
-      end
-    end
-
-    context 'Given a missing datastore in the pool config' do
-      before(:each) do
-        config[:pools][0]['datastore'] = nil
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/does not have a datastore defined/)
-      end
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when adding the disk raises an error' do
-      before(:each) do
-        expect(subject).to receive(:add_disk).and_raise(RuntimeError,'Mock Disk Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/Mock Disk Error/)
-      end
-    end
-
-    context 'when adding the disk succeeds' do
-      before(:each) do
-        expect(subject).to receive(:add_disk).with(vm_object, disk_size, datastorename, connection, datacenter_name)
-      end
-
-      it 'should return true' do
-        expect(subject.create_disk(poolname,vmname,disk_size)).to be true
-      end
-    end
-  end
-
-  describe '#create_snapshot' do
-    let(:snapshot_task) { mock_RbVmomi_VIM_Task() }
-    let(:snapshot_name) { 'snapshot' }
-    let(:snapshot_tree) {{}}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname, vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when snapshot already exists' do
-      let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-      let(:snapshot_tree) { { snapshot_name => { :ref => snapshot_object } } }
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Snapshot #{snapshot_name} .+ already exists /)
-      end
-    end
-
-    context 'when snapshot raises an error' do
-      before(:each) do
-        expect(vm_object).to receive(:CreateSnapshot_Task).and_raise(RuntimeError,'Mock Snapshot Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Mock Snapshot Error/)
-      end
-    end
-
-    context 'when snapshot succeeds' do
-      before(:each) do
-        expect(vm_object).to receive(:CreateSnapshot_Task)
-          .with(create_snapshot_spec(snapshot_name))
-          .and_return(snapshot_task)
-        expect(snapshot_task).to receive(:wait_for_completion).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.create_snapshot(poolname,vmname,snapshot_name)).to be true
-      end
-    end
-  end
-
-  describe '#revert_snapshot' do
-    let(:snapshot_task) { mock_RbVmomi_VIM_Task() }
-    let(:snapshot_name) { 'snapshot' }
-    let(:snapshot_tree) { { snapshot_name => { :ref => snapshot_object } } }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when snapshot does not exist' do
-      let(:snapshot_tree) {{}}
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Snapshot #{snapshot_name} .+ does not exist /)
-      end
-    end
-
-    context 'when revert to snapshot raises an error' do
-      before(:each) do
-        expect(snapshot_object).to receive(:RevertToSnapshot_Task).and_raise(RuntimeError,'Mock Snapshot Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Mock Snapshot Error/)
-      end
-    end
-
-    context 'when revert to snapshot succeeds' do
-      before(:each) do
-        expect(snapshot_object).to receive(:RevertToSnapshot_Task).and_return(snapshot_task)
-        expect(snapshot_task).to receive(:wait_for_completion).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.revert_snapshot(poolname,vmname,snapshot_name)).to be true
-      end
-    end
-  end
-
-  describe '#destroy_vm' do
-    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'Given a missing VM name' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, 'missing_vm')).to be true
-      end
-    end
-
-    context 'Given a powered on VM' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :powerstate => 'poweredOn',
-        })
-      }
-
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(vm_object)
-        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        allow(power_off_task).to receive(:wait_for_completion)
-        allow(destroy_task).to receive(:wait_for_completion)
-      end
-
-      it 'should call PowerOffVM_Task on the VM' do
-        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should call Destroy_Task on the VM' do
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, vmname)).to be true
-      end
-    end
-
-    context 'Given a powered off VM' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :powerstate => 'poweredOff',
-        })
-      }
-
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(vm_object)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        allow(destroy_task).to receive(:wait_for_completion)
-      end
-
-      it 'should not call PowerOffVM_Task on the VM' do
-        expect(vm_object).to receive(:PowerOffVM_Task).exactly(0).times
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should call Destroy_Task on the VM' do
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, vmname)).to be true
-      end
-    end
-  end
-
-  describe '#vm_ready?' do
-    let(:domain) { nil }
-    context 'When a VM is ready' do
-      before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname, domain)
-      end
-
-      it 'should return true' do
-        expect(subject.vm_ready?(poolname,vmname)).to be true
-      end
-    end
-
-    context 'When a VM is ready but the pool does not exist' do
-      # TODO not sure how to handle a VM that is passed in but
-      # not located in the pool.  Is that ready or not?
-      before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname, domain)
-      end
-
-      it 'should return true' do
-        expect(subject.vm_ready?('missing_pool',vmname)).to be true
-      end
-    end
-
-    context 'When an error occurs connecting to the VM' do
-      # TODO not sure how to handle a VM that is passed in but
-      # not located in the pool.  Is that ready or not?
-      before(:each) do
-        expect(subject).to receive(:open_socket).and_raise(RuntimeError,'MockError')
-      end
-
-      it 'should return false' do
-        expect(subject.vm_ready?(poolname,vmname)).to be false
-      end
-    end
-  end
-
-  describe '#vm_exists?' do
-    it 'should return true when get_vm returns an object' do
-      allow(subject).to receive(:get_vm).with(poolname,vmname).and_return(mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }))
-
-      expect(subject.vm_exists?(poolname,vmname)).to eq(true)
-    end
-
-    it 'should return false when get_vm returns nil' do
-      allow(subject).to receive(:get_vm).with(poolname,vmname).and_return(nil)
-
-      expect(subject.vm_exists?(poolname,vmname)).to eq(false)
-    end
-  end
-
-  # vSphere helper methods
-  describe '#get_target_datacenter_from_config' do
-    let(:pool_dc) { 'PoolDC'}
-    let(:provider_dc) { 'ProvDC'}
-
-    context 'when not specified' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-:pools:
-  - name: '#{poolname}'
-EOT
-        )
-      }
-      it 'returns nil' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to be_nil
-      end
-    end
-
-    context 'when specified only in the pool' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{pool_dc}'
-EOT
-        )
-      }
-      it 'returns the pool datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
-      end
-    end
-
-    context 'when specified only in the provider' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    datacenter: '#{provider_dc}'
-:pools:
-  - name: '#{poolname}'
-EOT
-        )
-      }
-      it 'returns the provider datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(provider_dc)
-      end
-    end
-
-    context 'when specified in the provider and pool' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    datacenter: '#{provider_dc}'
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{pool_dc}'
-EOT
-        )
-      }
-      it 'returns the pool datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
-      end
-    end
-  end
-
-  # vSphere helper methods
-  describe '#ensured_vsphere_connection' do
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    # Drop the connection pool timeout way down for spec tests so they fail fast
-    connection_pool_timeout: 1
-    connection_pool_size: 1
-:pools:
-EOT
-      )
-    }
-    let(:connection1) { mock_RbVmomi_VIM_Connection(connection_options) }
-    let(:connection2) { mock_RbVmomi_VIM_Connection(connection_options) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection1)
-    end
-
-    # This is to ensure that the pool_size of 1 is in effect
-    it 'should return the same connection object when calling the pool multiple times' do
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-    end
-
-    context 'when the connection breaks' do
-      before(:each) do
-        # Emulate the connection state being good, then bad, then good again
-        expect(subject).to receive(:vsphere_connection_ok?).and_return(true, false, true)
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection1, connection2)
-      end
-
-      it 'should restore the connection' do
-        subject.connection_pool.with_metrics do |pool_object|
-          # This line needs to be added to all instances of the connection_pool allocation
-          connection = subject.ensured_vsphere_connection(pool_object)
-
-          expect(connection).to be(connection1)
-        end
-
-        subject.connection_pool.with_metrics do |pool_object|
-          connection = subject.ensured_vsphere_connection(pool_object)
-          # The second connection would have failed.  This test ensures that a
-          # new connection object was created.
-          expect(connection).to be(connection2)
-        end
-
-        subject.connection_pool.with_metrics do |pool_object|
-          connection = subject.ensured_vsphere_connection(pool_object)
-          expect(connection).to be(connection2)
-        end
-      end
-    end
-  end
-
-  describe '#connect_to_vsphere' do
-    before(:each) do
-      allow(RbVmomi::VIM).to receive(:connect).and_return(connection)
-    end
-
-    let (:credentials) { config[:providers][:vsphere] }
-
-    context 'succesful connection' do
-      it 'should use the supplied credentials' do
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => credentials['insecure']
-        }).and_return(connection)
-        subject.connect_to_vsphere
-      end
-
-      it 'should honor the insecure setting' do
-        config[:providers][:vsphere][:insecure] = true
-
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => true,
-        }).and_return(connection)
-        subject.connect_to_vsphere
-      end
-
-      it 'should default to an insecure connection' do
-        config[:providers][:vsphere][:insecure] = nil
-
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => true
-        }).and_return(connection)
-
-        subject.connect_to_vsphere
-      end
-
-      it 'should return the connection object' do
-        result = subject.connect_to_vsphere
-
-        expect(result).to be(connection)
-      end
-
-      it 'should increment the connect.open counter' do
-        expect(metrics).to receive(:increment).with('connect.open')
-        subject.connect_to_vsphere
-      end
-    end
-
-    context 'connection is initially unsuccessful' do
-      before(:each) do
-        # Simulate a failure and then success
-        expect(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError').ordered
-        expect(RbVmomi::VIM).to receive(:connect).and_return(connection).ordered
-
-        allow(subject).to receive(:sleep)
-      end
-
-      it 'should return the connection object' do
-        result = subject.connect_to_vsphere
-
-        expect(result).to be(connection)
-      end
-
-      it 'should increment the connect.fail and then connect.open counter' do
-        expect(metrics).to receive(:increment).with('connect.fail').exactly(1).times
-        expect(metrics).to receive(:increment).with('connect.open').exactly(1).times
-        subject.connect_to_vsphere
-      end
-    end
-
-    context 'connection is always unsuccessful' do
-      before(:each) do
-        allow(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError')
-        allow(subject).to receive(:sleep)
-      end
-
-      it 'should retry the connection attempt config.max_tries times' do
-        expect(RbVmomi::VIM).to receive(:connect).exactly(config[:config]['max_tries']).times
-
-        begin
-          # Swallow any errors
-          subject.connect_to_vsphere
-        rescue
-        end
-      end
-
-      it 'should increment the connect.fail counter config.max_tries times' do
-        expect(metrics).to receive(:increment).with('connect.fail').exactly(config[:config]['max_tries']).times
-
-        begin
-          # Swallow any errors
-          subject.connect_to_vsphere
-        rescue
-        end
-      end
-
-      [{:max_tries => 5, :retry_factor => 1},
-      {:max_tries => 8, :retry_factor => 5},
-      ].each do |testcase|
-        context "Configuration set for max_tries of #{testcase[:max_tries]} and retry_facter of #{testcase[:retry_factor]}" do
-          it "should sleep #{testcase[:max_tries] - 1} times between attempts with increasing timeout" do
-            config[:config]['max_tries'] = testcase[:max_tries]
-            config[:config]['retry_factor'] = testcase[:retry_factor]
-
-            (1..testcase[:max_tries] - 1).each do |try|
-              expect(subject).to receive(:sleep).with(testcase[:retry_factor] * try).ordered
-            end
-
-            begin
-              # Swallow any errors
-              subject.connect_to_vsphere
-            rescue
-            end
-          end
-        end
-      end
-    end
-  end
-
-  describe '#open_socket' do
-    let(:TCPSocket) { double('tcpsocket') }
-    let(:socket) { double('tcpsocket') }
-    let(:hostname) { 'host' }
-    let(:domain) { 'domain.local'}
-    let(:default_socket) { 22 }
-
-    before do
-      expect(subject).not_to be_nil
-      allow(socket).to receive(:close)
-    end
-
-    it 'opens socket with defaults' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-
-      expect(subject.open_socket(hostname)).to eq(nil)
-    end
-
-    it 'yields the socket if a block is given' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-
-      expect{ |socket| subject.open_socket(hostname,nil,nil,default_socket,&socket) }.to yield_control.exactly(1).times 
-    end
-
-    it 'closes the opened socket' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-      expect(socket).to receive(:close)
-
-      expect(subject.open_socket(hostname)).to eq(nil)
-    end
-
-    it 'opens a specific socket' do
-      expect(TCPSocket).to receive(:new).with(hostname,80).and_return(socket)
-
-      expect(subject.open_socket(hostname,nil,nil,80)).to eq(nil)
-    end
-
-    it 'uses a specific domain with the hostname' do
-      expect(TCPSocket).to receive(:new).with("#{hostname}.#{domain}",default_socket).and_return(socket)
-
-      expect(subject.open_socket(hostname,domain)).to eq(nil)
-    end
-
-    it 'raises error if host is not resolvable' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'getaddrinfo: No such host is known')
-
-      expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError)
-    end
-
-    it 'raises error if socket is not listening' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'No connection could be made because the target machine actively refused it')
-
-      expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError)
-    end
-  end
-
-  describe '#get_vm_folder_path' do
-    [
-      { :path_description => 'Datacenters/DC01/vm/Pooler/pool1/vm1',
-        :expected_path => 'Pooler/pool1',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'Pooler' },
-          { :type => 'folder',     :name => 'pool1'},
-        ],
-      },
-      { :path_description => 'Datacenters/DC01/vm/something/subfolder/pool/vm1',
-        :expected_path => 'something/subfolder/pool',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'something' },
-          { :type => 'folder',     :name => 'subfolder' },
-          { :type => 'folder',     :name => 'pool'},
-        ],
-      },
-      { :path_description => 'Datacenters/DC01/vm/vm1',
-        :expected_path => '',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-        ],
-      },
-    ].each do |testcase|
-      context "given a path of #{testcase[:path_description]}" do
-        it "should return '#{testcase[:expected_path]}'" do
-          vm_object = mock_RbVmomi_VIM_VirtualMachine({
-            :name => 'vm1',
-            :path => testcase[:vm_object_path],
-          })
-          expect(subject.get_vm_folder_path(vm_object)).to eq(testcase[:expected_path])
-        end
-      end
-    end
-
-    [
-      { :path_description => 'a path missing a Datacenter',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'Pooler' },
-          { :type => 'folder',     :name => 'pool1'},
-        ],
-      },
-      { :path_description => 'a path missing root VM folder',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-        ],
-      },
-    ].each do |testcase|
-      context "given #{testcase[:path_description]}" do
-        it "should return nil" do
-          vm_object = mock_RbVmomi_VIM_VirtualMachine({
-            :name => 'vm1',
-            :path => testcase[:vm_object_path],
-          })
-          expect(subject.get_vm_folder_path(vm_object)).to be_nil
-        end
-      end
-    end
-  end
-
-  describe '#add_disk' do
-    let(:datastorename) { 'datastore' }
-    let(:disk_size) { 30 }
-    let(:collectMultiple_response) { {} }
-
-    let(:vm_scsi_controller) { mock_RbVmomi_VIM_VirtualSCSIController() }
-
-    # Require at least one SCSI Controller
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-      })
-      mock_vm.config.hardware.device << vm_scsi_controller
-
-      mock_vm
-    }
-
-    # Require at least one DC with the required datastore
-    let(:connection_options) {{
-      :serviceContent => {
-        :datacenters => [
-          { :name => datacenter_name, :datastores => [datastorename] }
-        ]
-      }
-    }}
-
-    let(:create_virtual_disk_task) { mock_RbVmomi_VIM_Task() }
-    let(:reconfig_vm_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      # Mocking for find_vmdks
-      allow(connection.serviceContent.propertyCollector).to receive(:collectMultiple).and_return(collectMultiple_response)
-
-      # Mocking for creating the disk
-      allow(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task) do |options|
-        if options[:datacenter][:name] == datacenter_name
-          create_virtual_disk_task
-        end
-      end
-      allow(create_virtual_disk_task).to receive(:wait_for_completion).and_return(true)
-
-      # Mocking for adding disk to the VM
-      allow(vm_object).to receive(:ReconfigVM_Task).and_return(reconfig_vm_task)
-      allow(reconfig_vm_task).to receive(:wait_for_completion).and_return(true)
-    end
-
-    context 'Succesfully addding disk' do
-      it 'should return true' do
-        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
-      end
-
-      it 'should request a disk of appropriate size' do
-        expect(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task)
-          .with(create_virtual_disk_with_size(disk_size))
-          .and_return(create_virtual_disk_task)
-
-
-        subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)
-      end
-    end
-
-    context 'Requested disk size is 0' do
-      it 'should raise an error' do
-        expect(subject.add_disk(vm_object,0,datastorename,connection,datacenter_name)).to be false
-      end
-    end
-
-    context 'No datastores or datastore missing' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => ['missing_datastore'] }
-          ]
-        }
-      }}
-
-      it 'should raise error' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(/does not exist/)
-      end
-    end
-
-    context 'Multiple datacenters with multiple datastores' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC', :datastores => ['dc1','dc2'] },
-            { :name => datacenter_name, :datastores => ['dc3',datastorename,'dc4'] },
-          ]
-        }
-      }}
-
-      it 'should return true' do
-        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
-      end
-    end
-
-    context 'VM does not have a SCSI Controller' do
-      let(:vm_object) {
-        mock_vm = mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-        })
-
-        mock_vm
-      }
-
-      it 'should raise an error' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(NoMethodError)
-      end
-    end
-  end
-
-  describe '#find_datastore' do
-    let(:datastorename) { 'datastore' }
-    let(:datastore_list) { [] }
-
-    context 'No datastores in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => [] }
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        expect(result).to be_nil
-      end
-    end
-
-    context 'Many datastores in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => ['ds1','ds2',datastorename,'ds3'] }
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore('missing_datastore',connection,datacenter_name)
-        expect(result).to be_nil
-      end
-
-      it 'should find the datastore in the datacenter' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        
-        expect(result).to_not be_nil
-        expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
-        expect(result.name).to eq(datastorename)
-      end
-    end
-
-    context 'Many datastores in many datacenters' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC', :datastores => ['ds1','ds2','ds3'] },
-            { :name => datacenter_name, :datastores => ['ds3','ds4',datastorename,'ds5'] },
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore(datastorename,connection,'AnotherDC')
-        expect(result).to be_nil
-      end
-
-      it 'should find the datastore in the datacenter' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        
-        expect(result).to_not be_nil
-        expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
-        expect(result.name).to eq(datastorename)
-      end
-    end
-  end
-
-  describe '#find_device' do
-    let(:devicename) { 'device1' }
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-      mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      mock_vm
-    }
-
-    it 'should return a device if the device name matches' do
-      result = subject.find_device(vm_object,devicename)
-
-      expect(result.deviceInfo.label).to eq(devicename)
-    end
-
-    it 'should return nil if the device name does not match' do
-      result = subject.find_device(vm_object,'missing_device')
-
-      expect(result).to be_nil
-    end
-  end
-
-  describe '#find_disk_controller' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-
-    it 'should return nil when there are no devices' do
-      result = subject.find_disk_controller(vm_object)
-
-      expect(result).to be_nil
-    end
-
-    [0,1,14].each do |testcase|
-      it "should return a device for a single VirtualSCSIController with #{testcase} attached disks" do
-        mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-        vm_object.config.hardware.device << mock_scsi
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-        # Add the disks
-        (1..testcase).each do
-          vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key })
-        end
-
-        result = subject.find_disk_controller(vm_object)
-
-        expect(result).to eq(mock_scsi)
-      end
-    end
-
-    [15].each do |testcase|
-      it "should return nil for a single VirtualSCSIController with #{testcase} attached disks" do
-        mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-        vm_object.config.hardware.device << mock_scsi
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-        # Add the disks
-        (1..testcase).each do
-          vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key })
-        end
-
-        result = subject.find_disk_controller(vm_object)
-
-        expect(result).to be_nil
-      end
-    end
-
-    it 'should raise if a VirtualDisk is missing a controller' do
-      # Note - Typically this is not possible as a VirtualDisk requires a controller (SCSI, PVSCSI or IDE)
-      mock_scsi = mock_RbVmomi_VIM_VirtualDisk()
-      vm_object.config.hardware.device << mock_scsi
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      expect{subject.find_disk_controller(vm_object)}.to raise_error(NoMethodError)
-    end
-  end
-
-  describe '#find_disk_devices' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-
-    it 'should return empty hash when there are no devices' do
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result).to eq({})
-    end
-
-    it 'should return empty hash when there are no VirtualSCSIController or VirtualDisk devices' do
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result).to eq({})
-    end
-
-    it 'should return a device for a VirtualSCSIController device with no children' do
-      mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-      vm_object.config.hardware.device << mock_scsi
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(1)
-      expect(result[mock_scsi.key]).to_not be_nil
-      expect(result[mock_scsi.key]['children']).to eq([])
-      expect(result[mock_scsi.key]['device']).to eq(mock_scsi)
-    end
-
-    it 'should return a device for a VirtualDisk device' do
-      mock_disk = mock_RbVmomi_VIM_VirtualDisk()
-      vm_object.config.hardware.device << mock_disk
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(1)
-      expect(result[mock_disk.controllerKey]).to_not be_nil
-      expect(result[mock_disk.controllerKey]['children'][0]).to eq(mock_disk)
-    end
-
-    it 'should return one device for many VirtualDisk devices on the same controller' do
-      controller1Key = rand(2000)
-      controller2Key = controller1Key + 1
-      mock_disk1 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key})
-      mock_disk2 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key})
-      mock_disk3 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller2Key})
-
-      vm_object.config.hardware.device << mock_disk2
-      vm_object.config.hardware.device << mock_disk1
-      vm_object.config.hardware.device << mock_disk3
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(2)
-
-      expect(result[controller1Key]).to_not be_nil
-      expect(result[controller2Key]).to_not be_nil
-
-      expect(result[controller1Key]['children']).to contain_exactly(mock_disk1,mock_disk2)
-      expect(result[controller2Key]['children']).to contain_exactly(mock_disk3)
-    end
-  end
-
-  describe '#find_disk_unit_number' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-    let(:controller) { mock_RbVmomi_VIM_VirtualSCSIController() }
-
-    it 'should return 0 when there are no devices' do
-      result = subject.find_disk_unit_number(vm_object,controller)
-
-      expect(result).to eq(0)
-    end
-
-    context 'with a single SCSI Controller' do
-      before(:each) do
-        vm_object.config.hardware.device << controller
-      end
-
-      it 'should return 1 when the host bus controller is at 0' do
-        controller.scsiCtlrUnitNumber = 0
-
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(1)
-      end
-
-      it 'should return the next lowest id when disks are attached' do
-        expected_id = 9
-        controller.scsiCtlrUnitNumber = 0
-
-        (1..expected_id-1).each do |disk_id|
-          mock_disk = mock_RbVmomi_VIM_VirtualDisk({
-            :controllerKey => controller.key,
-            :unitNumber => disk_id,
-          })
-          vm_object.config.hardware.device << mock_disk
-        end
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(expected_id)
-      end
-
-      it 'should return nil when there are no spare units' do
-        controller.scsiCtlrUnitNumber = 0
-
-        (1..15).each do |disk_id|
-          mock_disk = mock_RbVmomi_VIM_VirtualDisk({
-            :controllerKey => controller.key,
-            :unitNumber => disk_id,
-          })
-          vm_object.config.hardware.device << mock_disk
-        end
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(nil)
-      end
-    end
-  end
-
-  describe '#find_vm_folder' do
-    let(:foldername) { 'folder'}
-
-    context 'with no folder hierarchy' do
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy in many datacenters' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a multi layer folder hierarchy' do
-      let(:foldername) { 'folder2/folder4/folder' }
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-  end
-
-  describe '#get_host_utilization' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:limit) { 75 }
-    let(:default_limit) { 90 }
-
-    context "host with a different model" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem() }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,different_model,limit)).to be_nil
-      end
-    end
-
-    context "host in maintenance mode" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :maintenance_mode => true,
-        })
-      }
-      it 'should return nil' do
-        host.runtime.inMaintenanceMode = true
-
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host with status of not green" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_status => 'purple_alert',
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host with configuration issue" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :config_issue => 'No quickstats',
-      })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    # CPU utilization
-    context "host which exceeds limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 100,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host which exceeds default limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => default_limit + 1.0,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model)).to be_nil
-      end
-    end
-
-    context "host which does not exceed default limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => default_limit,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should not return nil' do
-        expect(subject.get_host_utilization(host,model)).to_not be_nil
-      end
-    end
-
-    # Memory utilization
-    context "host which exceeds limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => 100,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        # Set the Memory Usage to 100%
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host which exceeds default limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => default_limit + 1.0,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model)).to be_nil
-      end
-    end
-
-    context "host which does not exceed default limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => default_limit,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should not return nil' do
-        expect(subject.get_host_utilization(host,model)).to_not be_nil
-      end
-    end
-
-    context "host which does not exceed limits" do
-      # Set CPU to 10%
-      # Set Memory to 20%
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 10,
-        :overall_memory_usage => 20,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return the sum of CPU and Memory utilization' do
-        expect(subject.get_host_utilization(host,model,limit)[0]).to eq(10)
-      end
-
-      it 'should return the host' do
-        expect(subject.get_host_utilization(host,model,limit)[1]).to eq(host)
-      end
-    end
-
-    context 'host with no quickstats' do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :cpu_speed => 100,
-        :num_cores_per_cpu => 1,
-        :num_cpu => 1,
-        :memory_size => 100.0 * 1024 * 1024
-        })
-      }
-      before(:each) do
-        host.summary.quickStats.overallCpuUsage = nil
-      end
-
-      it 'should return nil' do
-        result = subject.get_host_utilization(host,model,limit)
-        expect(result).to be nil
-      end
-    end
-  end
-
-  describe '#host_has_cpu_model?' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:host) { mock_RbVmomi_VIM_HostSystem({
-      :cpu_model => cpu_model,
-      })
-    }
-
-    it 'should return true if the model matches' do
-      expect(subject.host_has_cpu_model?(host,model)).to eq(true)
-    end
-
-    it 'should return false if the model is different' do
-      expect(subject.host_has_cpu_model?(host,different_model)).to eq(false)
-    end
-  end
-
-  describe '#get_host_cpu_arch_version' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:host) { mock_RbVmomi_VIM_HostSystem({
-      :cpu_model => cpu_model,
-      :num_cpu => 2,
-      })
-    }
-
-    it 'should return the fifth element in the string delimited by spaces' do
-      expect(subject.get_host_cpu_arch_version(host)).to eq(model)
-    end
-
-    it 'should use the description of the first CPU' do
-      host.hardware.cpuPkg[0].description = 'vendor line type sku v6 speed'
-      expect(subject.get_host_cpu_arch_version(host)).to eq('v6')
-    end
-  end
-
-  describe '#cpu_utilization_for' do
-    [{ :cpu_usage => 10.0,
-      :core_speed => 10.0,
-      :num_cores => 2,
-      :expected_value => 50.0,
-    },
-    { :cpu_usage => 10.0,
-      :core_speed => 10.0,
-      :num_cores => 4,
-      :expected_value => 25.0,
-    },
-    { :cpu_usage => 14.0,
-      :core_speed => 12.0,
-      :num_cores => 5,
-      :expected_value => 23.0 + 1.0/3.0,
-    },
-    ].each do |testcase|
-      context "CPU Usage of #{testcase[:cpu_usage]}MHz with #{testcase[:num_cores]} x #{testcase[:core_speed]}MHz cores" do
-        it "should be #{testcase[:expected_value]}%" do
-          host = mock_RbVmomi_VIM_HostSystem({
-            :num_cores_per_cpu => testcase[:num_cores],
-            :cpu_speed         => testcase[:core_speed],
-            :overall_cpu_usage => testcase[:cpu_usage],
-          })
-
-          expect(subject.cpu_utilization_for(host)).to eq(testcase[:expected_value])
-        end
-      end
-    end
-  end
-
-  describe '#memory_utilization_for' do
-    [{ :memory_usage_gigbytes => 10.0,
-      :memory_size_bytes => 10.0 * 1024 * 1024,
-      :expected_value => 100.0,
-    },
-    { :memory_usage_gigbytes => 15.0,
-      :memory_size_bytes => 25.0 * 1024 * 1024,
-      :expected_value => 60.0,
-    },
-    { :memory_usage_gigbytes => 9.0,
-      :memory_size_bytes => 31.0 * 1024 * 1024,
-      :expected_value => 29.03225806451613,
-    },
-    ].each do |testcase|
-      context "Memory Usage of #{testcase[:memory_usage_gigbytes]}GBytes with #{testcase[:memory_size_bytes]}Bytes of total memory" do
-        it "should be #{testcase[:expected_value]}%" do
-          host = mock_RbVmomi_VIM_HostSystem({
-            :memory_size          => testcase[:memory_size_bytes],
-            :overall_memory_usage => testcase[:memory_usage_gigbytes],
-          })
-
-          expect(subject.memory_utilization_for(host)).to eq(testcase[:expected_value])
-        end
-      end
-    end
-  end
-
-  describe '#select_target_hosts' do
-    let(:target) { {} }
-    let(:cluster) { 'cluster1' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:hosts_hash) {
-      {
-        'hosts' => [host],
-        'architectures' => {
-          architecture => [host]
-        }
-      }
-    }
-
-    it 'returns a hash of the least used hosts by cluster and architecture' do
-      expect(subject).to receive(:find_least_used_hosts).and_return(hosts_hash)
-
-      subject.select_target_hosts(target, cluster, datacenter)
-      expect(target["#{datacenter}_#{cluster}"]).to eq(hosts_hash)
-    end
-
-    context 'with a cluster specified that does not exist' do
-      it 'raises an error' do
-        expect(subject).to receive(:find_least_used_hosts).with(missing_cluster_name, datacenter, 100).and_raise("Cluster #{cluster} cannot be found")
-        expect{subject.select_target_hosts(target, missing_cluster_name, datacenter)}.to raise_error(RuntimeError,/Cluster #{cluster} cannot be found/)
-      end
-    end
-  end
-
-  describe '#get_average_cluster_utilization' do
-    let(:hosts) {
-      [
-        [60, 'host1'],
-        [100, 'host2'],
-        [200, 'host3']
-      ]
-    }
-    it 'returns the average utilization for a given set of host utilizations assuming the first member of the list for each host is the utilization value' do
-      expect(subject.get_average_cluster_utilization(hosts)).to eq(120)
-    end
-  end
-
-  describe '#build_compatible_hosts_lists' do
-    let(:host1) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST1' })}
-    let(:host2) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST2' })}
-    let(:host3) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST3' })}
-    let(:architecture) { 'v4' }
-    let(:percentage) { 100 }
-    let(:hosts) {
-      [
-        [60, host1],
-        [100, host2],
-        [200, host3]
-      ]
-    }
-    let(:result) {
-      {
-          architecture => ['HOST1','HOST2']
-      }
-    }
-
-    it 'returns a hash of target host architecture versions containing lists of target hosts' do
-
-      expect(subject.build_compatible_hosts_lists(hosts, percentage)).to eq(result)
-    end
-  end
-
-  describe '#select_least_used_hosts' do
-    let(:percentage) { 100 }
-    let(:host1) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST1' })}
-    let(:host2) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST2' })}
-    let(:host3) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST3' })}
-    let(:hosts) {
-      [
-        [60, host1],
-        [100, host2],
-        [200, host3]
-      ]
-    }
-    let(:result) { ['HOST1','HOST2'] }
-    it 'returns the percentage specified of the least used hosts in the cluster determined by selecting from less than or equal to average cluster utilization' do
-      expect(subject.select_least_used_hosts(hosts, percentage)).to eq(result)
-    end
-
-    context 'when selecting 20 percent of hosts below average' do
-      let(:percentage) { 20 }
-      let(:result) { ['HOST1'] }
-
-      it 'should return the result' do
-        expect(subject.select_least_used_hosts(hosts, percentage)).to eq(result)
-      end
-    end
-
-    it 'should raise' do
-      expect{subject.select_least_used_hosts([], percentage)}.to raise_error(RuntimeError,/Provided hosts list to select_least_used_hosts is empty/)
-    end
-  end
-
-  describe '#run_select_hosts' do
-    let(:target) { {} }
-    let(:cluster) { 'cluster1' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:loop_delay) { 5 }
-    let(:max_age) { 60 }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:hosts_hash) {
-      {
-       dc => {
-          'hosts' => [host],
-          'architectures' => {
-            architecture => [host]
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_age: 60
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'target does not have key dc' do
-      let(:hosts_hash) { { } }
-      it 'should run select_target_hosts' do
-        expect(subject).to receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when check_time_finished is greater than max_age' do
-      before(:each) do
-        hosts_hash[dc]['check_time_finished'] = Time.now - (max_age + 10)
-      end
-
-      it 'should run select_target_hosts' do
-        expect(subject).to receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when check_time_finished is not greater than max_age' do
-      before(:each) do
-        hosts_hash[dc]['check_time_finished'] = Time.now - (max_age / 2)
-      end
-
-      it 'should not run select_target_hosts' do
-        expect(subject).to_not receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when hosts are being selected' do
-      before(:each) do
-        hosts_hash[dc]['checking'] = true
-      end
-
-      it 'should run wait_for_host_selection' do
-        expect(subject).to receive(:wait_for_host_selection).with(dc, hosts_hash, loop_delay, max_age)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-
-    end
-
-    context 'with no clone_target' do
-      before(:each) do
-        config[:pools][0].delete('clone_target')
-      end
-
-      it 'should raise an error' do
-        expect{subject.run_select_hosts(poolname, hosts_hash)}.to raise_error(/cluster for pool #{poolname} cannot be identified/)
-      end
-    end
-
-    context 'with no datacenter' do
-      before(:each) do
-        config[:pools][0].delete('datacenter')
-      end
-
-      it 'should raise an error' do
-        expect{subject.run_select_hosts(poolname, hosts_hash)}.to raise_error(/datacenter for pool #{poolname} cannot be identified/)
-      end
-    end
-  end
-
-  describe '#wait_for_host_selection' do
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:maxloop) { 1 }
-    let(:loop_delay) { 1 }
-    let(:max_age) { 60 }
-    let(:target) {
-      {
-        dc => { }
-      }
-    }
-
-    context 'when target does not have key check_time_finished' do
-      it 'should sleep for loop_delay maxloop times' do
-        expect(subject).to receive(:sleep).with(loop_delay).once
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-
-    context 'when target has dc and check_time_finished is greater than max_age' do
-      before(:each) do
-        target[dc]['check_time_finished'] = Time.now - (max_age + 10)
-      end
-
-      it 'should sleep for loop_delay maxloop times' do
-        expect(subject).to receive(:sleep).with(loop_delay).once
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-
-    context 'when target has dc and check_time_finished difference from now is less than max_age' do
-      before(:each) do
-        target[dc]['check_time_finished'] = Time.now - (max_age / 2)
-      end
-
-      it 'should not wait' do
-        expect(subject).to_not receive(:sleep).with(loop_delay)
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-  end
-
-  describe '#select_next_host' do
-    let(:cluster) { 'cluster1' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:loop_delay) { 5 }
-    let(:max_age) { 60 }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:hosts_hash) {
-      {
-       dc => {
-          'hosts' => [host, 'host2'],
-          'architectures' => {
-            architecture => ['host3', 'host4']
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_age: 60
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    context 'when host is requested' do
-      it 'should return the first host' do
-        result = subject.select_next_host(poolname, hosts_hash)
-        expect(result).to eq(host)
-      end
-
-      it 'should return the second host if called twice' do
-        result = subject.select_next_host(poolname, hosts_hash)
-        result2 = subject.select_next_host(poolname, hosts_hash)
-        expect(result2).to eq('host2')
-      end
-    end
-
-    context 'when host and architecture are requested' do
-      it 'should return the first host' do
-        result = subject.select_next_host(poolname, hosts_hash, architecture)
-        expect(result).to eq('host3')
-      end
-
-      it 'should return the second host if called twice' do
-        result = subject.select_next_host(poolname, hosts_hash, architecture)
-        result2 = subject.select_next_host(poolname, hosts_hash, architecture)
-        expect(result2).to eq('host4')
-      end
-    end
-
-    context 'with no hosts available' do
-      before(:each) do
-        hosts_hash[dc].delete('hosts')
-        hosts_hash[dc].delete('architectures')
-      end
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-
-      it 'should raise an error when selecting with architecture' do
-        expect{subject.select_next_host(poolname, hosts_hash, architecture)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-    end
-
-    context 'with no datacenter set for pool' do
-      before(:each) do
-        config[:pools][0].delete('datacenter')
-      end
-
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("datacenter for pool #{poolname} cannot be identified")
-      end
-    end
-
-    context 'with no clone_target set for pool' do
-      before(:each) do
-        config[:pools][0].delete('clone_target')
-      end
-
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("cluster for pool #{poolname} cannot be identified")
-      end
-    end
-  end
-
-  describe '#vm_in_target?' do
-    let(:parent_host) { 'host1' }
-    let(:architecture) { 'v3' }
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:maxloop) { 1 }
-    let(:loop_delay) { 1 }
-    let(:max_age) { 60 }
-    let(:target) {
-      {
-        dc => {
-          'hosts' => [parent_host],
-          'architectures' => {
-            architecture => [parent_host]
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    it 'returns true when parent_host is in hosts' do
-      result = subject.vm_in_target?(poolname, parent_host, architecture, target)
-      expect(result).to be true
-    end
-
-    context 'with parent_host only in architectures' do
-      before(:each) do
-        target[dc]['hosts'] = ['host2']
-      end
-
-      it 'returns true when parent_host is in architectures' do
-        result = subject.vm_in_target?(poolname, parent_host, architecture, target)
-        expect(result).to be true
-      end
-    end
-
-    it 'returns false when parent_host is not in hosts or architectures' do
-      result = subject.vm_in_target?(poolname, 'host3', architecture, target)
-      expect(result).to be false
-    end
-
-    context 'with no hosts key' do
-      before(:each) do
-        target[dc].delete('hosts')
-      end
-
-      it 'should raise an error' do
-        expect{subject.vm_in_target?(poolname, parent_host, architecture, target)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-    end
-  end
-
-  describe '#get_vm_details' do
-    let(:parent_host) { 'host1' }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:architecture) { 'v4' }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it 'returns nil when vm_object is not found' do
-      expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      result = subject.get_vm_details(poolname, vmname, connection)
-      expect(result).to be nil
-    end
-
-    it 'raises an error when unable to determine parent_host' do
-      expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-      expect{subject.get_vm_details(poolname, vmname, connection)}.to raise_error('Unable to determine which host the VM is running on')
-    end
-
-    context 'when it can find parent host' do
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'returns vm details hash' do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-        result = subject.get_vm_details(poolname, vmname, connection)
-        expect(result).to eq(vm_details)
-      end
-    end
-  end
-
-  describe '#migrate_vm' do
-    let(:parent_host) { 'host1' }
-    let(:new_host) { 'host2' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v4' }
-    let(:cluster) { 'cluster1' }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:new_host_object) { mock_RbVmomi_VIM_HostSystem({ :name => new_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  migration_limit: 5
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:provider_hosts) {
-      {
-        dc => {
-          'hosts' => [new_host],
-          'architectures' => {
-            architecture => [new_host]
-          },
-          'check_time_finished' => Time.now
-        }
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'when vm should be migrated' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          allow($redis).to receive(:hset)
-          expect(subject).to receive(:run_select_hosts).with(poolname, {})
-          expect(subject).to receive(:vm_in_target?).and_return false
-          expect(subject).to receive(:migration_enabled?).and_return true
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'logs a message' do
-        expect(subject).to receive(:select_next_host).and_return(new_host)
-        expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
-        expect(subject).to receive(:migrate_vm_host).with(vm_object, new_host_object)
-        expect($redis).to receive(:hget).with("vmpooler__vm__#{vmname}", 'checkout').and_return((Time.now - 1).to_s)
-        expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from #{parent_host} to #{new_host} in 0.00 seconds")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-
-      it 'migrates a vm' do
-        expect(subject).to receive(:migrate_vm_to_new_host).with(poolname, vmname, vm_details, connection)
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'current host is in the list of target hosts' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'host', parent_host)
-          expect(subject).to receive(:run_select_hosts).with(poolname, {})
-          expect(subject).to receive(:vm_in_target?).and_return true
-          expect(subject).to receive(:migration_enabled?).and_return true
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'logs a message that no migration is required' do
-        expect(logger).to receive(:log).with('s', "[ ] [#{poolname}] No migration required for '#{vmname}' running on #{parent_host}")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'with migration limit exceeded' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect($redis).to receive(:scard).with('vmpooler__migration').and_return(5)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to_not receive(:run_select_hosts)
-          expect(subject).to_not receive(:vm_in_target?)
-          expect(subject).to receive(:migration_enabled?).and_return true
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'should log the current host' do
-        expect(logger).to receive(:log).with('s', "[ ] [#{poolname}] '#{vmname}' is running on #{parent_host}. No migration will be evaluated since the migration_limit has been reached")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'when an error occurs' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to receive(:run_select_hosts)
-          expect(subject).to receive(:vm_in_target?).and_return false
-          expect(subject).to receive(:migration_enabled?).and_return true
-          expect(subject).to receive(:select_next_host).and_raise(RuntimeError,'Mock migration error')
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-      it 'should log the current host' do
-        expect(logger).to receive(:log).with('s', "[!] [#{poolname}] '#{vmname}' is running on #{parent_host}")
-
-        expect{subject.migrate_vm(poolname, vmname)}.to raise_error(RuntimeError, 'Mock migration error')
-      end
-    end
-  end
-
-  describe '#migrate_vm_to_new_host' do
-    let(:parent_host) { 'host1' }
-    let(:new_host) { 'host2' }
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:architecture) { 'v4' }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:new_host_object) { mock_RbVmomi_VIM_HostSystem({ :name => new_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  migration_limit: 5
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it' migrates a vm' do
-      expect($redis).to receive(:sadd).with('vmpooler__migration', vmname)
-      expect(subject).to receive(:select_next_host).and_return(new_host)
-      expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'host', new_host)
-      expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'migrated', true)
-      expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
-      expect(subject).to receive(:migrate_vm_and_record_timing).and_return(format('%.2f', (Time.now - (Time.now - 15))))
-      expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from host1 to host2 in 15.00 seconds")
-      expect($redis).to receive(:srem).with('vmpooler__migration', vmname)
-      subject.migrate_vm_to_new_host(poolname, vmname, vm_details, connection)
-    end
-  end
-
-  describe '#create_folder' do
-    let(:datacenter) { 'dc1' }
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-    let(:new_folder) { poolname }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it 'creates a folder' do
-      expect(connection.serviceInstance).to receive(:find_datacenter).with(datacenter).and_return(datacenter_object)
-      expect(datacenter_object.vmFolder).to receive(:traverse).with(new_folder, type=RbVmomi::VIM::Folder, create=true).and_return(folder_object)
-
-      result = subject.create_folder(connection, new_folder, datacenter)
-      expect(result).to eq(folder_object)
-    end
-
-    context 'with folder_object returning nil' do
-      it 'shoud raise an error' do
-        expect(connection.serviceInstance).to receive(:find_datacenter).with(datacenter).and_return(datacenter_object)
-        expect(datacenter_object.vmFolder).to receive(:traverse).with(new_folder, type=RbVmomi::VIM::Folder, create=true).and_return(nil)
-
-        expect{subject.create_folder(connection, new_folder, datacenter)}.to raise_error("Cannot create folder #{new_folder}")
-      end
-    end
-  end
-
-  describe '#migration_enabled?' do
-    let(:config) {
-      { :config => { 'migration_limit' => 5 } }
-    }
-    it 'returns true if migration is enabled' do
-      result = subject.migration_enabled?(config)
-      expect(result).to be true
-    end
-
-    context 'with migration disabled' do
-      let(:config) {
-        { :config => { } }
-      }
-      it 'should return false' do
-        result = subject.migration_enabled?(config)
-        expect(result).to be false
-      end
-    end
-
-    context 'with non-integer value for migration limit' do
-      let(:config) {
-        { :config => { 'migration_limit' => 1.0 } }
-      }
-      it 'should return false' do
-        result = subject.migration_enabled?(config)
-        expect(result).to be false
-      end
-    end
-  end
-
-  describe '#find_least_used_hosts' do
-    let(:cluster_name) { 'cluster' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
-    let(:percentage) { 100 }
-
-    before(:each) do
-      # This mocking is a little fragile but hard to do without a real vCenter instance
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-      datacenter_object.hostFolder.childEntity = [cluster_object]
-    end
-
-    context 'missing cluster' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-      }]})}
-      let(:expected_host) { cluster_object.host[0] }
-#,datacenter_name
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-      }]})}
-      let(:expected_host) { cluster_object.host[0][:name] }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'standalone host outside the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-          :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-      }]})}
-      let(:expected_host) { cluster_object.host[0] }
-
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'cluster of 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1].name }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1].name }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      it 'should return a host' do
-        pending('https://github.com/puppetlabs/vmpooler/issues/206')
-        result = subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)
-        expect(result).to_not be_nil
-      end
-    end
-  end
-
-  describe '#find_cluster' do
-    let(:cluster) {'cluster'}
-    let(:missing_cluster) {'missing_cluster'}
-
-    context 'no clusters in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name }
-          ]
-        }
-      }}
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'compute_resource'},
-                'cluster2' => {:object_type => 'compute_resource'},
-                cluster => {:object_type => 'compute_resource'},
-                'cluster3' => {:object_type => 'compute_resource'},
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy with multiple datacenters' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC',
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'compute_resource'},
-                'cluster2' => {:object_type => 'compute_resource'},
-              }
-            },
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                cluster => {:object_type => 'compute_resource'},
-                'cluster3' => {:object_type => 'compute_resource'},
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,'AnotherDC')).to be_nil
-      end
-    end
-
-    context 'with a multi layer folder hierarchy' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'compute_resource'},
-                'folder2' => {
-                  :children => {
-                    cluster => {:object_type => 'compute_resource'},
-                  }
-                },
-                'cluster3' => {:object_type => 'compute_resource'},
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        pending('https://github.com/puppetlabs/vmpooler/issues/205')
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-  end
-
-  describe '#get_cluster_host_utilization' do
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) }
-      
-      it 'should return array with one element' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(1)
-      end
-    end
-
-    context 'standalone host which is out the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 0 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(0)
-      end
-    end
-
-    context 'cluster with 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 3 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(3)
-      end
-    end
-
-    context 'cluster with 5 hosts of which 3 within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 3 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(3)
-      end
-    end
-
-    context 'cluster with 3 hosts of which none are within the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 0 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(0)
-      end
-    end
-  end
-
-  describe '#find_least_used_vpshere_compatible_host' do
-    let(:vm) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) }
-      let(:standalone_host) { cluster_object.host[0] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = standalone_host
-      end
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(standalone_host)
-        expect(result[1]).to eq(standalone_host.name)
-      end
-    end
-
-    context 'standalone host outside of limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:standalone_host) { cluster_object.host[0] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = standalone_host
-      end
-
-      it 'should raise error' do
-        expect{subject.find_least_used_vpshere_compatible_host(vm)}.to raise_error(/There is no host candidate in vcenter that meets all the required conditions/)
-      end
-    end
-
-    context 'cluster of 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return the least used host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(expected_host)
-        expect(result[1]).to eq(expected_host.name)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should raise error' do
-        expect{subject.find_least_used_vpshere_compatible_host(vm)}.to raise_error(/There is no host candidate in vcenter that meets all the required conditions/)
-      end
-    end
-
-    context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[2] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return the least used host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(expected_host)
-        expect(result[1]).to eq(expected_host.name)
-      end
-    end
-
-    context 'cluster of 3 hosts all with the same utilisation' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return a host' do
-        pending('https://github.com/puppetlabs/vmpooler/issues/206 is fixed')
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-      end
-    end
-  end
-
-  describe '#find_snapshot' do
-    let(:snapshot_name) {'snapshot'}
-    let(:missing_snapshot_name) {'missing_snapshot'}
-    let(:vm) { mock_RbVmomi_VIM_VirtualMachine(mock_options) }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-
-    context 'VM with no snapshots' do
-      let(:mock_options) {{ :snapshot_tree => nil }}
-      it 'should return nil' do
-        expect(subject.find_snapshot(vm,snapshot_name)).to be_nil
-      end
-    end
-
-    context 'VM with a single layer of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => nil,
-          'snapshot4' => nil,
-          snapshot_name => { :ref => snapshot_object},
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.find_snapshot(vm,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-
-      it 'should return nil which no matches are found' do
-        result = subject.find_snapshot(vm,missing_snapshot_name)
-        expect(result).to be_nil
-      end
-    end
-
-    context 'VM with a nested layers of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => { :children => {
-            'snapshot4' => nil,
-            'snapshot5' => { :children => {
-              snapshot_name => { :ref => snapshot_object},
-            }},
-          }},
-          'snapshot6' => nil,
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.find_snapshot(vm,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-
-      it 'should return nil which no matches are found' do
-        result = subject.find_snapshot(vm,missing_snapshot_name)
-        expect(result).to be_nil
-      end
-    end
-  end
-
-  describe '#find_vm' do
-    let(:missing_vm) { 'missing_vm' }
-    let(:folder) { 'Pooler/pool1' }
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    before(:each) do
-      allow(connection.searchIndex).to receive(:FindByInventoryPath)
-    end
-
-    it 'should call FindByInventoryPath with the correct parameters' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath)
-
-      subject.find_vm(poolname,vmname,connection)
-    end
-
-    it 'should return the VM object when found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(vm_object)
-
-      expect(subject.find_vm(poolname,vmname,connection)).to be(vm_object)
-    end
-
-    it 'should return nil if the VM is not found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-
-      expect(subject.find_vm(poolname,missing_vm,connection)).to be_nil
-    end
-  end
-
-  describe '#get_base_vm_container_from' do
-    it 'should return a recursive view of type VirtualMachine' do
-      result = subject.get_base_vm_container_from(connection)
-
-      expect(result.recursive).to be true
-      expect(result.type).to eq(['VirtualMachine'])
-    end
-  end
-
-  describe '#get_snapshot_list' do
-    let(:snapshot_name) {'snapshot'}
-    let(:snapshot_tree) { mock_RbVmomi_VIM_VirtualMachine(mock_options).snapshot.rootSnapshotList }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    it 'should raise if the snapshot tree is nil' do
-      expect{ subject.get_snapshot_list(nil,snapshot_name)}.to raise_error(NoMethodError)
-    end
-
-    context 'VM with a single layer of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => nil,
-          'snapshot4' => nil,
-          snapshot_name => { :ref => snapshot_object},
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.get_snapshot_list(snapshot_tree,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-    end
-
-    context 'VM with a nested layers of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => { :children => {
-            'snapshot4' => nil,
-            'snapshot5' => { :children => {
-              snapshot_name => { :ref => snapshot_object},
-            }},
-          }},
-          'snapshot6' => nil,
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.get_snapshot_list(snapshot_tree,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-    end
-  end
-
-  describe '#migrate_vm_host' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST' })}
-    let(:relocate_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task)
-      allow(relocate_task).to receive(:wait_for_completion)
-    end
-
-    it 'should call RelocateVM_Task' do
-      expect(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task)
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should use a Relocation Spec object with correct host' do
-      expect(vm_object).to receive(:RelocateVM_Task).with(relocation_spec_with_host(host_object))
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should wait for the relocation to complete' do
-      expect(relocate_task).to receive(:wait_for_completion)
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should return the result of the relocation' do
-      expect(relocate_task).to receive(:wait_for_completion).and_return('RELOCATE_RESULT')
-
-      expect(subject.migrate_vm_host(vm_object,host_object)).to eq('RELOCATE_RESULT')
-    end
-  end
-
-  describe 'find_template_vm' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    before(:each) do
-      allow(connection.searchIndex).to receive(:FindByInventoryPath)
-    end
-    it 'should raise an error when the datacenter cannot be found' do
-      config[:providers][:vsphere]['datacenter'] = nil
-
-      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error('cannot find datacenter')
-    end
-
-    it 'should raise an error when the template specified cannot be found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-
-      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error("Pool #{poolname} specifies a template VM of #{config[:pools][0]['template']} which does not exist for the provider vsphere")
-    end
-
-    it 'should return the vm object' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(vm_object)
-
-      subject.find_template_vm(config[:pools][0],connection)
-    end
-  end
-
-  describe 'valid_template_path?' do
-
-    it 'should return true with a valid template path' do
-      expect(subject.valid_template_path?('test/template')).to eq(true)
-    end
-
-    it 'should return false when no / is found' do
-      expect(subject.valid_template_path?('testtemplate')).to eq(false)
-    end
-
-    it 'should return false when template path begins with /' do
-      expect(subject.valid_template_path?('/testtemplate')).to eq(false)
-    end
-
-    it 'should return false when template path ends with /' do
-      expect(subject.valid_template_path?('testtemplate/')).to eq(false)
-    end
-  end
-
-  describe 'create_template_delta_disks' do
-    let(:template_object) { mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-      })
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'with a template VM found' do
-
-      before(:each) do
-        expect(subject).to receive(:find_template_vm).and_return(template_object)
-      end
-
-      it 'should reconfigure the VM creating delta disks' do
-        expect(template_object).to receive(:add_delta_disk_layer_on_all_disks)
-
-        subject.create_template_delta_disks(config[:pools][0])
-      end
-    end
-  end
-
-  describe 'get_disk_backing' do
-
-    it 'should return moveChildMostDiskBacking when linked clone enabled' do
-      expect( subject.get_disk_backing({'create_linked_clone' => true}) ).to eq(:moveChildMostDiskBacking)
-    end
-
-    it 'should return moveAllDiskBackingsAndConsolidate when no preference is specified' do
-      expect( subject.get_disk_backing({})).to eq(:moveAllDiskBackingsAndConsolidate)
-    end
-
-    it 'should return moveAllDiskBackingsAndConsolidate when linked clone is false' do
-      expect( subject.get_disk_backing({create_linked_clone: false})).to eq(:moveAllDiskBackingsAndConsolidate)
-    end
-  end
-
-  describe 'linked_clone?' do
-    it 'should return true when linked clone is enabled on the pool' do
-      expect( subject.linked_clone?({'create_linked_clone' => true}) ).to be true
-    end
-
-    it 'should return nil when linked clone is not enabled on the pool' do
-      expect( subject.linked_clone?({}) ).to be nil
-    end
-  end
-end
diff --git a/spec/unit/providers_spec.rb b/spec/unit/providers_spec.rb
index 0ccc7f0..91045f2 100644
--- a/spec/unit/providers_spec.rb
+++ b/spec/unit/providers_spec.rb
@@ -12,12 +12,8 @@ describe 'providers' do
   end
 
   it '#load_all_providers' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(Vmpooler::Providers.load_all_providers).to match_array(p)
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
   it '#installed_providers' do
@@ -30,21 +26,18 @@ describe 'providers' do
   end
 
   it '#load_by_name' do
-    expect(Vmpooler::Providers.load_by_name('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to_not match(%r{,})
   end
 
-  it '#load only vpshere' do
-    expect(providers.load_from_gems('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+  it '#load only dummy' do
+    expect(providers.load_from_gems('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(providers.load_from_gems('dummy').join(', ')).to_not match(%r{,})
   end
 
   it '#load all providers from gems' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(providers.load_from_gems).to match_array(p)
-
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
 
diff --git a/spec/unit/queue_reliability_spec.rb b/spec/unit/queue_reliability_spec.rb
new file mode 100644
index 0000000..fe95548
--- /dev/null
+++ b/spec/unit/queue_reliability_spec.rb
@@ -0,0 +1,497 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'vmpooler/pool_manager'
+
+describe 'Vmpooler::PoolManager - Queue Reliability Features' do
+  let(:logger) { MockLogger.new }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { redis } }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { YAML.load(<<~EOT
+    ---
+    :config:
+      task_limit: 10
+      vm_checktime: 1
+      vm_lifetime: 12
+      prefix: 'pooler-'
+      dlq_enabled: true
+      dlq_ttl: 168
+      dlq_max_entries: 100
+      purge_enabled: true
+      purge_dry_run: false
+      max_pending_age: 7200
+      max_ready_age: 86400
+      max_completed_age: 3600
+      health_check_enabled: true
+      health_check_interval: 300
+      health_thresholds:
+        pending_queue_max: 100
+        ready_queue_max: 500
+        dlq_max_warning: 100
+        dlq_max_critical: 1000
+        stuck_vm_age_threshold: 7200
+    :providers:
+      :dummy: {}
+    :pools:
+      - name: 'test-pool'
+        size: 5
+        provider: 'dummy'
+    EOT
+    )
+  }
+
+  subject { Vmpooler::PoolManager.new(config, logger, redis_connection_pool, metrics) }
+
+  describe 'Dead-Letter Queue (DLQ)' do
+    let(:vm) { 'vm-abc123' }
+    let(:pool) { 'test-pool' }
+    let(:error_class) { 'StandardError' }
+    let(:error_message) { 'template does not exist' }
+    let(:request_id) { 'req-123' }
+    let(:pool_alias) { 'test-alias' }
+
+    before(:each) do
+      redis_connection_pool.with do |redis_connection|
+        allow(redis_connection).to receive(:zadd)
+        allow(redis_connection).to receive(:zcard).and_return(0)
+        allow(redis_connection).to receive(:expire)
+      end
+    end
+
+    describe '#dlq_enabled?' do
+      it 'returns true when dlq_enabled is true in config' do
+        expect(subject.dlq_enabled?).to be true
+      end
+
+      it 'returns false when dlq_enabled is false in config' do
+        config[:config]['dlq_enabled'] = false
+        expect(subject.dlq_enabled?).to be false
+      end
+    end
+
+    describe '#dlq_ttl' do
+      it 'returns configured TTL' do
+        expect(subject.dlq_ttl).to eq(168)
+      end
+
+      it 'returns default TTL when not configured' do
+        config[:config].delete('dlq_ttl')
+        expect(subject.dlq_ttl).to eq(168)
+      end
+    end
+
+    describe '#dlq_max_entries' do
+      it 'returns configured max entries' do
+        expect(subject.dlq_max_entries).to eq(100)
+      end
+
+      it 'returns default max entries when not configured' do
+        config[:config].delete('dlq_max_entries')
+        expect(subject.dlq_max_entries).to eq(10000)
+      end
+    end
+
+    describe '#move_to_dlq' do
+      context 'when DLQ is enabled' do
+        it 'adds entry to DLQ sorted set' do
+          redis_connection_pool.with do |redis_connection|
+            dlq_key = 'vmpooler__dlq__pending'
+            
+            expect(redis_connection).to receive(:zadd).with(dlq_key, anything, anything)
+            expect(redis_connection).to receive(:expire).with(dlq_key, anything)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, 
+                               redis_connection, request_id: request_id, pool_alias: pool_alias)
+          end
+        end
+
+        it 'includes error details in DLQ entry' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).to receive(:zadd) do |_key, _score, entry|
+              expect(entry).to include(vm)
+              expect(entry).to include(error_message)
+              expect(entry).to include(error_class)
+            end
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'increments DLQ metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with('vmpooler_dlq.pending.count')
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'enforces max entries limit' do
+          redis_connection_pool.with do |redis_connection|
+            allow(redis_connection).to receive(:zcard).and_return(150)
+            expect(redis_connection).to receive(:zremrangebyrank).with(anything, 0, 49)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+
+      context 'when DLQ is disabled' do
+        before { config[:config]['dlq_enabled'] = false }
+
+        it 'does not add entry to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).not_to receive(:zadd)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+    end
+  end
+
+  describe 'Auto-Purge' do
+    describe '#purge_enabled?' do
+      it 'returns true when purge_enabled is true in config' do
+        expect(subject.purge_enabled?).to be true
+      end
+
+      it 'returns false when purge_enabled is false in config' do
+        config[:config]['purge_enabled'] = false
+        expect(subject.purge_enabled?).to be false
+      end
+    end
+
+    describe '#purge_dry_run?' do
+      it 'returns false when purge_dry_run is false in config' do
+        expect(subject.purge_dry_run?).to be false
+      end
+
+      it 'returns true when purge_dry_run is true in config' do
+        config[:config]['purge_dry_run'] = true
+        expect(subject.purge_dry_run?).to be true
+      end
+    end
+
+    describe '#max_pending_age' do
+      it 'returns configured max age' do
+        expect(subject.max_pending_age).to eq(7200)
+      end
+
+      it 'returns default max age when not configured' do
+        config[:config].delete('max_pending_age')
+        expect(subject.max_pending_age).to eq(7200)
+      end
+    end
+
+    describe '#purge_pending_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old' }
+      let(:new_vm) { 'vm-new' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (3 hours old, exceeds 2 hour threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'clone', (Time.now - 10800).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      context 'when not in dry-run mode' do
+        it 'purges stale pending VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be false
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", new_vm)).to be true
+          end
+        end
+
+        it 'moves purged VMs to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).to receive(:move_to_dlq).with(
+              old_vm, pool, 'pending', 'Purge', anything, redis_connection, anything
+            )
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+
+        it 'increments purge metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with("vmpooler_purge.pending.#{pool}.count")
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+
+      context 'when in dry-run mode' do
+        before { config[:config]['purge_dry_run'] = true }
+
+        it 'detects but does not purge stale VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be true
+          end
+        end
+
+        it 'does not move to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).not_to receive(:move_to_dlq)
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+    end
+
+    describe '#purge_ready_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-ready' }
+      let(:new_vm) { 'vm-new-ready' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (25 hours old, exceeds 24 hour threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'ready', (Time.now - 90000).to_s)
+          
+          # New VM (2 hours old, within threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'ready', (Time.now - 7200).to_s)
+        end
+      end
+
+      it 'moves stale ready VMs to completed queue' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_ready_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be true
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+
+    describe '#purge_completed_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-completed' }
+      let(:new_vm) { 'vm-new-completed' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (2 hours old, exceeds 1 hour threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'destroy', (Time.now - 7200).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'destroy', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'removes stale completed VMs' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_completed_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+  end
+
+  describe 'Health Checks' do
+    describe '#health_check_enabled?' do
+      it 'returns true when health_check_enabled is true in config' do
+        expect(subject.health_check_enabled?).to be true
+      end
+
+      it 'returns false when health_check_enabled is false in config' do
+        config[:config]['health_check_enabled'] = false
+        expect(subject.health_check_enabled?).to be false
+      end
+    end
+
+    describe '#health_thresholds' do
+      it 'returns configured thresholds' do
+        thresholds = subject.health_thresholds
+        expect(thresholds['pending_queue_max']).to eq(100)
+        expect(thresholds['stuck_vm_age_threshold']).to eq(7200)
+      end
+
+      it 'merges with defaults when partially configured' do
+        config[:config]['health_thresholds'] = { 'pending_queue_max' => 200 }
+        thresholds = subject.health_thresholds
+        
+        expect(thresholds['pending_queue_max']).to eq(200)
+        expect(thresholds['ready_queue_max']).to eq(500) # default
+      end
+    end
+
+    describe '#calculate_queue_ages' do
+      let(:pool) { 'test-pool' }
+      let(:vm1) { 'vm-1' }
+      let(:vm2) { 'vm-2' }
+      let(:vm3) { 'vm-3' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          redis_connection.hset("vmpooler__vm__#{vm1}", 'clone', (Time.now - 3600).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm2}", 'clone', (Time.now - 7200).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm3}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'calculates ages for all VMs' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, vm2, vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(3)
+          expect(ages[0]).to be_within(5).of(3600)
+          expect(ages[1]).to be_within(5).of(7200)
+          expect(ages[2]).to be_within(5).of(1800)
+        end
+      end
+
+      it 'skips VMs with missing timestamps' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, 'vm-nonexistent', vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(2)
+        end
+      end
+    end
+
+    describe '#determine_health_status' do
+      let(:base_metrics) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50 }
+            }
+          },
+          'errors' => {
+            'dlq_total_size' => 50,
+            'stuck_vm_count' => 2
+          }
+        }
+      end
+
+      it 'returns healthy when all metrics are within thresholds' do
+        status = subject.determine_health_status(base_metrics)
+        expect(status).to eq('healthy')
+      end
+
+      it 'returns degraded when DLQ size exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 150
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when DLQ size exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 1500
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns degraded when pending queue exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 120
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when pending queue exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 250
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns unhealthy when stuck VM count exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['stuck_vm_count'] = 60
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+    end
+
+    describe '#push_health_metrics' do
+      let(:metrics_data) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'oldest_age' => 3600, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50, 'oldest_age' => 7200 },
+              'completed' => { 'size' => 5 }
+            }
+          },
+          'tasks' => {
+            'clone' => { 'active' => 3 },
+            'ondemand' => { 'active' => 2, 'pending' => 5 }
+          },
+          'errors' => {
+            'dlq_total_size' => 25,
+            'stuck_vm_count' => 2,
+            'orphaned_metadata_count' => 3
+          }
+        }
+      end
+
+      it 'pushes status metric' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.status', 0)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes error metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.dlq.total_size', 25)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.stuck_vms.count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.orphaned_metadata.count', 3)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes per-pool queue metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.size', 10)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.oldest_age', 3600)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.stuck_count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.ready.size', 50)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes task metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.clone.active', 3)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.active', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.pending', 5)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+    end
+  end
+end
diff --git a/spec/unit/util/parsing_spec.rb b/spec/unit/util/parsing_spec.rb
new file mode 100644
index 0000000..c0277f9
--- /dev/null
+++ b/spec/unit/util/parsing_spec.rb
@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+# The only method previously tested here was '#get_domain_for_pool'
+# which was moved to Vmpooler::Dns as the more appropriate class
+#
+# TODO: Add tests for last remaining method, or move to more appropriate class
+describe 'Vmpooler::Parsing' do
+  let(:pool) { 'pool1' }
+  subject { Vmpooler::Parsing }
+
+end
diff --git a/spec/unit/vmpooler_spec.rb b/spec/unit/vmpooler_spec.rb
index ac9f620..8e6d39a 100644
--- a/spec/unit/vmpooler_spec.rb
+++ b/spec/unit/vmpooler_spec.rb
@@ -20,6 +20,21 @@ describe 'Vmpooler' do
           expect(Vmpooler.config[:pools]).to eq(default_config[:pools])
         end
       end
+
+      it 'keeps a copy of the original pools at startup' do
+        Dir.chdir(fixtures_dir) do
+          configuration = Vmpooler.config
+          expect(configuration[:pools]).to eq(configuration[:pools_at_startup])
+        end
+      end
+
+      it 'the copy is a separate object and not a reference' do
+        Dir.chdir(fixtures_dir) do
+          configuration = Vmpooler.config
+          configuration[:pools][0]['template'] = 'sam'
+          expect(configuration[:pools]).not_to eq(configuration[:pools_at_startup])
+        end
+      end
     end
 
     context 'when config variable is set' do
@@ -30,10 +45,55 @@ describe 'Vmpooler' do
     end
 
     context 'when config file is set' do
-      it 'should use the file' do
+      before(:each) do
         ENV['VMPOOLER_CONFIG_FILE'] = config_file
+      end
+      it 'should use the file' do
         expect(Vmpooler.config[:pools]).to eq(config[:pools])
       end
+      it 'merges one extra file, results in two providers' do
+        ENV['EXTRA_CONFIG'] = File.join(fixtures_dir, 'extra_config1.yaml')
+        expect(Vmpooler.config[:providers].keys).to include(:dummy)
+        expect(Vmpooler.config[:providers].keys).to include(:alice)
+      end
+      it 'merges two extra file, results in three providers and an extra pool' do
+        extra1 = File.join(fixtures_dir, 'extra_config1.yaml')
+        extra2 = File.join(fixtures_dir, 'extra_config2.yaml')
+        ENV['EXTRA_CONFIG'] = "#{extra1},#{extra2}"
+        expect(Vmpooler.config[:providers].keys).to include(:dummy)
+        expect(Vmpooler.config[:providers].keys).to include(:alice)
+        expect(Vmpooler.config[:providers].keys).to include(:bob)
+        merged_pools = [{"name"=>"pool03", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5},
+                        {"name"=>"pool04", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5},
+                        {"name"=>"pool05", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5}]
+        expect(Vmpooler.config[:pools]).to eq(merged_pools)
+        expect(Vmpooler.config[:config]).not_to be_nil #merge does not deleted existing keys
+      end
+    end
+
+    context 'when domain' do
+      context 'is set as a variable' do
+        before(:each) do
+          ENV['VMPOOLER_CONFIG_FILE'] = config_file
+          ENV['DOMAIN'] = 'example.com'
+        end
+
+        it 'should exit' do
+          expect { Vmpooler.config }.to raise_error(SystemExit)
+        end
+      end
+
+      context 'is set in a config file' do
+        let(:config_file) { File.join(fixtures_dir, 'vmpooler_domain.yaml') }
+
+        before(:each) do
+          ENV['VMPOOLER_CONFIG_FILE'] = config_file
+        end
+
+        it 'should exit' do
+          expect { Vmpooler.config }.to raise_error(SystemExit)
+        end
+      end
     end
   end
 end
diff --git a/update-gemfile-lock b/update-gemfile-lock
new file mode 100755
index 0000000..ec95ac1
--- /dev/null
+++ b/update-gemfile-lock
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+docker run -it --rm \
+  -v $(pwd):/app \
+  jruby:9.4.12.1-jdk11 \
+  /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase build-essential && cd /app && gem install bundler && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"'
diff --git a/utils/vmp_utils b/utils/vmp_utils
new file mode 100755
index 0000000..613abd7
--- /dev/null
+++ b/utils/vmp_utils
@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'redis'
+require 'thor'
+
+class VMPUtils < Thor
+  desc 'export_tokens', 'Export VMPooler tokens'
+  long_desc <<-LONGDESC
+    vmp_utils export_tokens will export the tokens stored in redis as JSON
+    to the specified file. By default it will export all tokens used within
+    the last year. This can be overridden by setting the "since-date" option.
+  LONGDESC
+  option :since_date, default: (DateTime.now - 365).iso8601
+  option :redis_host, required: true
+  option :redis_password
+  option :redis_port, type: :numeric, default: 6379
+  option :output_file, required: true
+  def export_tokens
+    redis = Redis.new(host: options[:redis_host], port: options[:redis_port], password: options[:redis_password])
+    begin
+      redis.ping
+      puts "connection successful to #{options[:redis_host]}"
+      tokens_to_export = {}
+      date_filter = DateTime.parse(options[:since_date])
+      redis.scan_each(match: 'vmpooler__token__*') do |key|
+        print('.')
+        key_data = redis.hgetall(key)
+        export = false
+        if key_data.key?('last') && date_filter < DateTime.parse(key_data['last'])
+          export = true
+        elsif date_filter < DateTime.parse(key_data['created'])
+          export = true
+        end
+        tokens_to_export[key] = key_data if export
+      end
+      puts ''
+      File.open(options[:output_file], 'w') do |output|
+        output.puts JSON.pretty_generate(tokens_to_export)
+      end
+    rescue Redis::CannotConnectError => e
+      raise Thor::Error, e
+    rescue Redis::CommandError => e
+      raise Thor::Error, e
+    end
+  end
+
+  desc 'import_tokens', 'Import previously exported VMPooler tokens'
+  long_desc <<-LONGDESC
+    vmp_utils import_tokens will load tokens from a previous export into the
+    specified redis instance. It expects a JSON file like the one generated by
+    export_tokens to be provided as input.
+  LONGDESC
+  option :redis_host, required: true
+  option :redis_password
+  option :redis_port, type: :numeric, default: 6379
+  option :input_file, required: true
+  def import_tokens
+    raise Thor::Error, "input-file '#{options[:input_file]}' doesn't exist" unless File.exist?(options[:input_file])
+
+    begin
+      data_file = File.read(options[:input_file])
+      tokens_to_import = JSON.parse(data_file)
+    rescue JSON::ParserError
+      raise Thor::Error, "Unable to parse please verify that #{options[:input_file]}. Please make sure it is a valid JSON file"
+    end
+
+    redis = Redis.new(host: options[:redis_host], port: options[:redis_port], password: options[:redis_password])
+    begin
+      redis.ping
+      puts "connection successful to #{options[:redis_host]}"
+      tokens_to_import.each do |key, value|
+        puts "importing token for #{value['user']}"
+        redis.hset(key, value)
+      end
+    rescue Redis::CannotConnectError => e
+      raise Thor::Error, e
+    rescue Redis::CommandError => e
+      raise Thor::Error, e
+    end
+  end
+end
+
+VMPUtils.start(ARGV)
diff --git a/vmpooler.gemspec b/vmpooler.gemspec
index c8b0433..fe53085 100644
--- a/vmpooler.gemspec
+++ b/vmpooler.gemspec
@@ -11,21 +11,41 @@ Gem::Specification.new do |s|
   s.summary       = 'vmpooler provides configurable pools of instantly-available (running) virtual machines'
   s.homepage      = 'https://github.com/puppetlabs/vmpooler'
   s.license       = 'Apache-2.0'
-  s.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
   s.files         = Dir[ "bin/*", "lib/**/*" ]
   s.bindir        = 'bin'
   s.executables   = 'vmpooler'
   s.require_paths = ["lib"]
-  s.add_dependency 'pickup', '~> 0.0.11'
-  s.add_dependency 'puma', '~> 3.11'
-  s.add_dependency 'rack', '~> 2.0'
-  s.add_dependency 'rake', '~> 12.3'
-  s.add_dependency 'redis', '~> 4.0'
-  s.add_dependency 'rbvmomi', '~> 1.13'
-  s.add_dependency 'sinatra', '~> 2.0'
+  s.add_dependency 'concurrent-ruby', '~> 1.1'
+  s.add_dependency 'connection_pool', '~> 2.4'
+  s.add_dependency 'deep_merge', '~> 1.2'
   s.add_dependency 'net-ldap', '~> 0.16'
+  s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.23.0'
+  s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.21.1'
+  s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.22.2'
+  s.add_dependency 'opentelemetry-instrumentation-rack', '= 0.23.4'
+  s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.25.3'
+  s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.23.2'
+  s.add_dependency 'opentelemetry-resource_detectors', '= 0.24.2'
+  s.add_dependency 'opentelemetry-sdk', '~> 1.8'
+  s.add_dependency 'pickup', '~> 0.0.11'
+  s.add_dependency 'prometheus-client', '>= 2', '< 5'
+  s.add_dependency 'puma', '>= 5.0.4', '< 7'
+  s.add_dependency 'rack', '>= 2.2', '< 4.0'
+  s.add_dependency 'rake', '~> 13.0'
+  s.add_dependency 'redis', '~> 5.0'
+  s.add_dependency 'sinatra', '>= 2', '< 4'
+  s.add_dependency 'spicy-proton', '~> 2.1'
   s.add_dependency 'statsd-ruby', '~> 1.4'
-  s.add_dependency 'connection_pool', '~> 2.2'
-  s.add_dependency 'nokogiri', '~> 1.8'
-end
+
+  # Testing dependencies
+  s.add_development_dependency 'climate_control', '>= 0.2.0'
+  s.add_development_dependency 'mock_redis', '= 0.37.0'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rack-test', '>= 0.6'
+  s.add_development_dependency 'rspec', '>= 3.2'
+  s.add_development_dependency 'rubocop', '~> 1.56.0'
+  s.add_development_dependency 'simplecov', '>= 0.11.2'
+  s.add_development_dependency 'thor', '~> 1.0', '>= 1.0.1'
+  s.add_development_dependency 'yarjuf', '>= 2.0'
+end
\ No newline at end of file
diff --git a/vmpooler.yaml.dummy-example b/vmpooler.yaml.dummy-example
index 3a15630..e27efd6 100644
--- a/vmpooler.yaml.dummy-example
+++ b/vmpooler.yaml.dummy-example
@@ -3,6 +3,12 @@
   :dummy:
     filename: '/tmp/dummy-backing.yaml'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    #domain: 'localhost' # Flip these out for local requests
+    domain: 'example.com'
+
 :redis:
   server: 'localhost'
 
@@ -17,14 +23,13 @@
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
-  #domain: 'localhost' # Flip these out for local requests
   prefix: 'poolvm-'
 
 :pools:
@@ -35,8 +40,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64'
     alias: [ 'debian-7-64', 'debian-7-amd64' ]
     template: 'Templates/debian-7-x86_64'
@@ -44,6 +51,7 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
-
+    dns_plugin: 'example'
diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example
index f6781b6..f05ded2 100644
--- a/vmpooler.yaml.example
+++ b/vmpooler.yaml.example
@@ -11,22 +11,29 @@
 #     For multiple providers, specify one of the supported backing services (vsphere or dummy)
 #     (optional: will default to it's parent :key: name eg. 'vsphere')
 #
-#   - purge_unconfigured_folders
-#     Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-#     Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-#     Nested child folders will not be destroyed. An optional whitelist can be provided to exclude folders
-#     A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
-#     Setting this on the provider will enable folder purging for the provider
+#   - purge_unconfigured_folders DEPRECATED, use purge_unconfigured_resources
+#   - purge_unconfigured_resources
+#     Enable purging of resources (typically VMs) or other items based on the provider. Provides a high-level cleanup
+#     mechanism for things that are live but not found in the vmpooler config ie in a pool config. See the provider's
+#     implementation for more details.
+#     Setting this on the provider will enable purging for the provider
 #     Expects a boolean value
 #     (optional; default: false)
 #
-#   - folder_whitelist
-#     Specify folders that are within the base folder path, not in the configuration, and should not be destroyed
-#     To exclude 'vmpooler/myfolder' from being destroyed when the base path is 'vmpooler' you would specify 'myfolder' in the whitelist
-#     This option is only evaluated when 'purge_unconfigured_folders' is enabled
-#     Expects an array of strings specifying the whitelisted folders by name
+#   - folder_whitelist DEPRECATED, use resources_allowlist
+#   - resources_allowlist
+#     Specify items names that should be ignored when purging. See the provider's
+#     implementation for more details.
+#     This option is only evaluated when 'purge_unconfigured_resources' is enabled
+#     Expects an array of strings specifying the allowed items by name
 #     (optional; default: nil)
 #
+#   - skip_dns_check_before_creating_vm
+#     Setting this configuration parameter in a provider will make vmpooler skip the check it normally does for a DNS
+#     record. The normal behavior is to check DNS and if a record already exists (conflict) to re-generate a new hostname.
+#     By using this configuration parameter, it will skip the check and continue with the same conflicting name. This is
+#     useful for providers that can handle that case, for instance by replacing the existing DNS record with a new one.
+#
 # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
 # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
 # the provider to use.
@@ -195,14 +202,27 @@
 #
 #   - data_ttl
 #     How long (in hours) to retain metadata in Redis after VM destruction.
-#     (optional; default: '168')
+#     (default: 168)
+#
+#   - redis_connection_pool_size
+#     Maximum number of connections to utilize for the redis connection pool.
+#     (default: 10)
+#
+#   - redis_connection_pool_timeout
+#     How long a task should wait (in seconds) for a redis connection when all connections are in use.
+#     (default: 5)
+#
+#   - reconnect_attempts
+#     How many times to retry one redis connection, for example if the host:port is not available
+#     The time between attempts starts at 1.5s and increases up to 10s, in such a way that 10 attempts
+#     takes about 80s, at which point an error is returned.
+#     (default: 10)
 
 # Example:
 
 :redis:
   server: 'redis.example.com'
 
-
 # :graphs:
 #
 # This section contains the server and prefix information for a graphite-
@@ -233,7 +253,7 @@
 #
 # This section contains the connection information required to store
 # historical data via statsd.  This is mutually exclusive with graphite
-# and takes precedence.
+# and prometheus and takes precedence.
 #
 # Available configuration parameters:
 #
@@ -260,7 +280,7 @@
 #
 # This section contains the connection information required to store
 # historical data in an external Graphite database.  This is mutually exclusive
-# with statsd.
+# with statsd and prometheus - i.e. only one can be selected.
 #
 # Available configuration parameters:
 #
@@ -281,6 +301,34 @@
 :graphite:
   server: 'graphite.example.com'
 
+# :prometheus
+#
+# This section contains the connection information required to store
+# historical data in an external Graphite database.  This is mutually exclusive
+# with statsd and graphite - i.e. only one can be selected.
+#
+# Available configuration parameters:
+#
+#   - prefix
+#     The prefix for this vmpooler instance.
+#     (optional; default: 'vmpooler')
+#
+#   - prometheus_prefix
+#     The prefix to use while storing prometheus data.
+#     (optional; default: 'vmpooler')
+#
+#   - prometheus_endpoint
+#     The metrics endpoint on the vmpooler server
+#     (optional; default: '/prometheus')
+
+# Example:
+
+:prometheus:
+  prefix: 'staging'
+  prometheus_prefix: 'vmpooler'
+  prometheus_endpoint: '/prometheus'
+
+
 # :auth:
 #
 # This section contains information related to authenticating users
@@ -319,6 +367,15 @@
 #   - user_object
 #     The LDAP object-type used to designate a user object.
 #
+#   - service_account_hash
+#     A hash containing the following parameters for a service account to perform the
+#     initial bind. After the initial bind, then a search query is performed using the
+#     'base' and 'user_object', then re-binds as the returned user.
+#     - :user_dn
+#     The full distinguished name (DN) of the service account used to bind.
+#     - :password
+#     The password for the service account used to bind.
+#
 # Example:
 # :auth:
 #   provider: 'ldap'
@@ -327,12 +384,33 @@
 #     port: 389
 #     base: 'ou=users,dc=company,dc=com'
 #     user_object: 'uid'
+#
+# :auth:
+#   provider: 'ldap'
+#   :ldap:
+#     host: 'ldap.example.com'
+#     port: 636
+#     service_account_hash:
+#       :user_dn: 'cn=Service Account,ou=Accounts,dc=ldap,dc=example,dc=com'
+#       :password: 'service-account-password'
+#     encryption:
+#       :method: :simple_tls
+#       :tls_options:
+#         :ssl_version: 'TLSv1_2'
+#     base:
+#       - 'ou=Accounts,dc=company,dc=com'
+#     user_object:
+#       - 'samAccountName'
 
 :auth:
   provider: 'ldap'
   :ldap:
     host: 'ldap.example.com'
-    port: 389
+    port: 636
+    encryption:
+      :method: :simple_tls
+      :tls_options:
+        :ssl_version: 'TLSv1_2'
     base: 'ou=users,dc=company,dc=com'
     user_object: 'uid'
 
@@ -368,15 +446,25 @@
 #
 #   - task_limit
 #     The number of concurrent VM creation tasks to perform.
-#     (optional; default: '10')
+#     (default: 10)
+#
+#   - ondemand_clone_limit
+#     The number of concurrent VM creation tasks to perform for ondemand VM requests.
+#     (default: 10)
 #
 #   - timeout
 #     How long (in minutes) before marking a clone in 'pending' queues as 'failed' and retrying.
-#     (optional; default: '15')
+#     (default: 15)
+#
+#   - max_vm_retries
+#     Maximum number of times to retry VM creation for a failed request before marking it as permanently failed.
+#     This helps prevent infinite retry loops when there are configuration issues like invalid template paths.
+#     Permanent errors (like invalid template paths) are detected and will not be retried.
+#     (default: 3)
 #
 #   - vm_checktime
 #     How often (in minutes) to check the sanity of VMs in 'ready' queues.
-#     (optional; default: '1')
+#     (default: 1)
 #
 #   - vm_lifetime
 #     How long (in hours) to keep VMs in 'running' queues before destroying.
@@ -389,9 +477,6 @@
 #   - allowed_tags
 #     If set, restricts tags to those specified in this array.
 #
-#   - domain
-#     If set, returns a top-level 'domain' JSON key in POST requests
-#
 #   - prefix
 #     If set, prefixes all created VMs with this string.  This should include
 #     a separator.
@@ -469,10 +554,11 @@
 #     Expects a boolean value
 #     (optional; default: false)
 #
-#   - purge_unconfigured_folders (vSphere Provider only)
-#     Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-#     Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-#     A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+#   - purge_unconfigured_folders DEPRECATED, use purge_unconfigured_resources
+#   - purge_unconfigured_resources
+#     Enable purging of resources (typically VMs) or other items based on the provider. Provides a high-level cleanup
+#     mechanism for things that are live but not found in the vmpooler config ie in a pool config. See the provider's
+#     implementation for more details.
 #     When enabled in the global configuration then purging is enabled for all providers
 #     Expects a boolean value
 #     (optional; default: false)
@@ -487,17 +573,47 @@
 #
 #   - usage_stats
 #     Enable shipping of VM usage stats
-#     When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize
+#     When enabled a metric is emitted when a user requested to allocate and destroy a VM. Tags are inspected and used to organize
 #     shipped metrics if there is a jenkins_build_url tag set for the VM.
-#     Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$pool_name".
+#     Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$operation.$pool_name".
 #     When the jenkins_build_url tag is set the metric will be sent with additional data. Here is an example
 #     based off of the following URL, and requested by the user ABS;
 #     https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/
-#     "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$pool_name", which translates to
+#     "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$operation.$pool_name", which translates to
 #     "usage.$user.jenkins_example_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool_name"
 #     Expects a boolean value
 #     (optional; default: false)
 #
+#   - request_logger
+#     Enable API Request logging to the logger
+#     When enabled all requests to the API are written to the standard logger.
+#     Expects a boolean value
+#     (optional; default: false)
+#
+#   - max_lifetime_upper_limit
+#     Sets a lifetime upper limit (in hours) for how long the vm lifetime can be set via the API. Lifetime can be set and extended
+#     so this configuration is used to enforce an upper limit to both the initial lifetime request and/or the extended
+#     lifetime (by checking how long it has already been running).
+#     (optional; default: unlimited)
+#
+#   - extra_config
+#     Specify additional application configuration files
+#     The argument can accept a full path to a file, or multiple files comma separated.
+#     Expects a string value
+#     (optional)
+#
+#   - max_ondemand_instances_per_request
+#     The maximum number of instances any individual ondemand request may contain per pool.
+#     (default: 10)
+#
+#   - ondemand_request_ttl
+#     The amount of time (in minutes) to give for a ondemand request to be fulfilled before considering it to have failed.
+#     (default: 5)
+#
+#   - ready_ttl
+#     How long (in minutes) a ready VM should stay in the ready queue.
+#     (default: 60)
+#
 # Example:
 
 :config:
@@ -505,19 +621,43 @@
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
+  max_vm_retries: 3
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
   experimental_features: true
   backend_weight:
     'backend1': 60
     'backend2': 40
 
+# :dns_configs:
+#
+# This section a list of dns configurations to be referenced by one or more pools.
+#
+# The currently supported backing services are:
+# - dynamic-dns (This assumes that dynamic dns is handling record management and VMPooler does not require interaction)
+# - gcp (Google Cloud DNS https://github.com/puppetlabs/vmpooler-dns-gcp)
+#
+#   - dns_class
+#     Specify one of the supported backing services.
+#
+#   - domain
+#     The domain expected to make up the FQDN when attempting to resolve VM instances.
+#
+# See DNS plugin docs for additional options specific to that class.
+#
+# Example
+
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # :pools:
 #
 # This section contains a list of virtual machine 'pools' for vmpooler to
@@ -548,6 +688,12 @@
 #     If you have more than one provider, this is where you would choose which
 #     one to use for this pool
 #
+#   - dns_plugin
+#     The name of the DNS plugin to use with this pool in order to determine the
+#     domain and settings specific to a DNS service. This should match
+#     a name in the :dns_configs: section above. e.g. example
+#     (required)
+#
 #   - clone_target
 #     Per-pool option to override the global 'clone_target' cluster.
 #     (optional)
@@ -580,6 +726,15 @@
 #       The datacenter within vCenter to manage VMs.
 #       (optional: default is the first datacenter in vSphere)
 #
+#     - snapshot_mainMem_ioBlockPages
+#       Provisions VMs with the option "mainMem.ioBlockPages". This setting can be useful
+#       (paired with mainMem.iowait below) for tuning the performance of snapshot management actions.
+#       See: https://kb.vmware.com/s/article/76687
+#
+#     - snapshot_mainMem_iowait
+#       Provisions VMs with the option "mainMem.iowait". This setting can be useful
+#       for tuning the performance of snapshot management actions
+#
 # Example:
 
 :pools:
@@ -590,6 +745,7 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: vsphere
     create_linked_clone: true
@@ -600,6 +756,9 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: vsphere
     create_linked_clone: false
+    snapshot_mainMem_ioBlockPages: '2048'
+    snapshot_mainMem_iowait: '0'
diff --git a/vmpooler.yml.example b/vmpooler.yml.example
new file mode 100644
index 0000000..31060c2
--- /dev/null
+++ b/vmpooler.yml.example
@@ -0,0 +1,92 @@
+---
+# VMPooler Configuration Example with Dead-Letter Queue, Auto-Purge, and Health Checks
+
+# Redis Configuration
+:redis:
+  server: 'localhost'
+  port: 6379
+  data_ttl: 168  # hours - how long to keep VM metadata in Redis
+  
+  # Dead-Letter Queue (DLQ) Configuration
+  dlq_enabled: true
+  dlq_ttl: 168  # hours (7 days) - how long to keep DLQ entries
+  dlq_max_entries: 10000  # maximum entries per DLQ queue before trimming
+
+# Application Configuration
+:config:
+  # ... other existing config ...
+  
+  # Dead-Letter Queue (DLQ) - Optional, defaults shown
+  dlq_enabled: false  # Set to true to enable DLQ
+  dlq_ttl: 168  # hours (7 days)
+  dlq_max_entries: 10000  # per DLQ queue
+  
+  # Auto-Purge Stale Queue Entries
+  purge_enabled: false  # Set to true to enable auto-purge
+  purge_interval: 3600  # seconds (1 hour) - how often to run purge cycle
+  purge_dry_run: false  # Set to true to log what would be purged without actually purging
+  
+  # Auto-Purge Age Thresholds (in seconds)
+  max_pending_age: 7200  # 2 hours - VMs stuck in pending
+  max_ready_age: 86400  # 24 hours - VMs idle in ready queue
+  max_completed_age: 3600  # 1 hour - VMs in completed queue
+  max_orphaned_age: 86400  # 24 hours - orphaned VM metadata
+  max_request_age: 86400  # 24 hours - stale on-demand requests
+  
+  # Health Checks
+  health_check_enabled: false  # Set to true to enable health checks
+  health_check_interval: 300  # seconds (5 minutes) - how often to run health checks
+  
+  # Health Check Thresholds
+  health_thresholds:
+    pending_queue_max: 100  # Warning threshold for pending queue size
+    ready_queue_max: 500  # Warning threshold for ready queue size
+    dlq_max_warning: 100  # Warning threshold for DLQ size
+    dlq_max_critical: 1000  # Critical threshold for DLQ size
+    stuck_vm_age_threshold: 7200  # 2 hours - age at which VM is considered "stuck"
+    stuck_vm_max_warning: 10  # Warning threshold for stuck VM count
+    stuck_vm_max_critical: 50  # Critical threshold for stuck VM count
+
+# Pool Configuration
+:pools:
+  - name: 'centos-7-x86_64'
+    size: 5
+    provider: 'vsphere'
+    # ... other pool settings ...
+
+# Provider Configuration
+:providers:
+  :vsphere:
+    server: 'vcenter.example.com'
+    username: 'vmpooler'
+    password: 'secret'
+    # ... other provider settings ...
+
+# Example: Production Configuration
+# For production use, you might want:
+# :config:
+#   dlq_enabled: true
+#   dlq_ttl: 168  # Keep failed VMs for a week
+#   
+#   purge_enabled: true
+#   purge_interval: 1800  # Run every 30 minutes
+#   purge_dry_run: false
+#   max_pending_age: 3600  # Purge pending VMs after 1 hour
+#   max_ready_age: 172800  # Purge ready VMs after 2 days
+#   
+#   health_check_enabled: true
+#   health_check_interval: 300  # Check every 5 minutes
+
+# Example: Development Configuration
+# For development/testing, you might want:
+# :config:
+#   dlq_enabled: true
+#   dlq_ttl: 24  # Keep failed VMs for a day
+#   
+#   purge_enabled: true
+#   purge_interval: 600  # Run every 10 minutes
+#   purge_dry_run: true  # Test mode - log but don't actually purge
+#   max_pending_age: 1800  # More aggressive - 30 minutes
+#   
+#   health_check_enabled: true
+#   health_check_interval: 60  # Check every minute