diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index 1b242ef..0000000
--- a/.dockerignore
+++ /dev/null
@@ -1,13 +0,0 @@
-**/*.yml
-**/*.yaml
-**/*.md
-**/*example
-**/Dockerfile*
-Gemfile.lock
-Rakefile
-Vagrantfile
-coverage
-docs
-examples
-scripts
-vendor
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index c8f8016..4ae585a 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -3,6 +3,11 @@ updates:
 - package-ecosystem: bundler
   directory: "/"
   schedule:
-    interval: daily
-    time: "13:00"
+    interval: weekly
+  open-pull-requests-limit: 10
+
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly
   open-pull-requests-limit: 10
diff --git a/.github/workflows/auto_release_prep.yml b/.github/workflows/auto_release_prep.yml
new file mode 100644
index 0000000..57a12de
--- /dev/null
+++ b/.github/workflows/auto_release_prep.yml
@@ -0,0 +1,12 @@
+name: Automated release prep
+
+on:
+  workflow_dispatch:
+
+jobs:
+  auto_release_prep:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/auto_release_prep.yml@v1
+    secrets: inherit
+    with:
+      project-type: ruby
+      version-file-path: lib/vmpooler/version.rb
diff --git a/.github/workflows/dependabot_merge.yml b/.github/workflows/dependabot_merge.yml
new file mode 100644
index 0000000..75b9cea
--- /dev/null
+++ b/.github/workflows/dependabot_merge.yml
@@ -0,0 +1,8 @@
+name: Dependabot auto-merge
+
+on: pull_request
+
+jobs:
+  dependabot_merge:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/dependabot_merge.yml@v1
+    secrets: inherit
diff --git a/.github/workflows/ensure_label.yml b/.github/workflows/ensure_label.yml
new file mode 100644
index 0000000..50a5fa8
--- /dev/null
+++ b/.github/workflows/ensure_label.yml
@@ -0,0 +1,8 @@
+name: Ensure label
+
+on: pull_request
+
+jobs:
+  ensure_label:
+    uses: puppetlabs/release-engineering-repo-standards/.github/workflows/ensure_label.yml@v1
+    secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..d020d40
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,58 @@
+name: Release Gem
+
+on: workflow_dispatch
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.repository == 'puppetlabs/vmpooler'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Current Version
+        uses: actions/github-script@v7
+        id: cv
+        with:
+          script: |
+            const { data: response } = await github.rest.repos.getLatestRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            console.log(`The latest release is ${response.tag_name}`)
+            return response.tag_name
+          result-encoding: string
+
+      - name: Get Next Version
+        id: nv
+        run: |
+          version=$(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev)
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "Found version $version from lib/vmpooler/version.rb"
+
+      - name: Tag Release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.nv.outputs.version }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          bodyfile: release-notes.md
+          draft: false
+          prerelease: false
+
+      # This step should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+      - name: Install Ruby jruby-9.4.12.1
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 'jruby-9.4.12.1'
+
+      - name: Build gem
+        run: gem build *.gemspec
+
+      - name: Publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: '${{ secrets.RUBYGEMS_AUTH_TOKEN }}'
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
new file mode 100644
index 0000000..ba273f5
--- /dev/null
+++ b/.github/workflows/security.yml
@@ -0,0 +1,39 @@
+name: Security
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  scan:
+    name: Mend Scanning
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout repo content
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - name: setup ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    # setup a package lock if one doesn't exist, otherwise do nothing
+    - name: check lock
+      run: '[ -f "Gemfile.lock" ] && echo "package lock file exists, skipping" || bundle lock'
+    # install java
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+    # download mend
+    - name: download_mend
+      run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
+    - name: run mend
+      run: java -jar wss-unified-agent.jar
+      env:
+        WS_APIKEY: ${{ secrets.MEND_API_KEY }}
+        WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
+        WS_USERKEY: ${{ secrets.MEND_TOKEN }}
+        WS_PRODUCTNAME: RE
+        WS_PROJECTNAME: ${{ github.event.repository.name }}
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
new file mode 100644
index 0000000..d93859a
--- /dev/null
+++ b/.github/workflows/testing.yml
@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run Rubocop
+      run: bundle exec rake rubocop
+
+  spec_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 'jruby-9.4.12.1'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run spec tests
+      run: bundle exec rake test
diff --git a/.github_changelog_generator b/.github_changelog_generator
new file mode 100644
index 0000000..ebeb260
--- /dev/null
+++ b/.github_changelog_generator
@@ -0,0 +1,5 @@
+project=vmpooler
+user=puppetlabs
+exclude_labels=maintenance
+github-api=https://api.github.com
+release-branch=main
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0e49c05..700a888 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
+.bundle/
 .vagrant/
-results.xml
+coverage/
+vendor/
+.dccache
 .ruby-version
-Gemfile.lock
 Gemfile.local
-vendor
+results.xml
 /vmpooler.yaml
-.bundle
-coverage
diff --git a/.rubocop.yml b/.rubocop.yml
index 2d50ca0..0fe1eff 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -66,6 +66,7 @@ Style/Next:
   Enabled: false
 Metrics/ParameterLists:
   Max: 10
+  MaxOptionalParameters: 10
 Style/GuardClause:
   Enabled: false
 
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
index 6105e46..7046eb6 100644
--- a/.rubocop_todo.yml
+++ b/.rubocop_todo.yml
@@ -12,7 +12,7 @@
 # SupportedStyles: with_first_parameter, with_fixed_indentation
 Layout/ParameterAlignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 9
 # Cop supports --auto-correct.
@@ -21,13 +21,13 @@ Layout/ParameterAlignment:
 Layout/CaseIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 Layout/ClosingParenthesisIndentation:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -58,14 +58,14 @@ Layout/EmptyLinesAroundModuleBody:
 Layout/FirstHashElementIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: Width, IgnoredPatterns.
 Layout/IndentationWidth:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -73,7 +73,7 @@ Layout/IndentationWidth:
 # SupportedStyles: symmetrical, new_line, same_line
 Layout/MultilineMethodCallBraceLayout:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -87,14 +87,14 @@ Layout/SpaceAroundEqualsInParameterDefault:
 # Cop supports --auto-correct.
 Layout/SpaceAroundKeyword:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowForAlignment.
 Layout/SpaceAroundOperators:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 8
 # Cop supports --auto-correct.
@@ -109,14 +109,14 @@ Layout/SpaceInsideHashLiteralBraces:
 # Cop supports --auto-correct.
 Layout/SpaceInsideParens:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Configuration parameters: AllowSafeAssignment.
 Lint/AssignmentInCondition:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 Lint/SuppressedException:
@@ -148,7 +148,7 @@ Lint/UselessAssignment:
 Style/AndOr:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 Style/CaseEquality:
@@ -169,7 +169,7 @@ Style/For:
 Style/HashSyntax:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
@@ -177,7 +177,7 @@ Style/HashSyntax:
 Style/IfUnlessModifier:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
@@ -185,13 +185,13 @@ Style/IfUnlessModifier:
 # SupportedStyles: both, prefix, postfix
 Style/NegatedIf:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
 Style/Not:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -200,26 +200,26 @@ Style/Not:
 Style/NumericPredicate:
   Exclude:
     - 'spec/**/*'
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/ParallelAssignment:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 # Configuration parameters: AllowSafeAssignment.
 Style/ParenthesesAroundCondition:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/PerlBackrefs:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist.
@@ -235,7 +235,7 @@ Naming/PredicateName:
 # Cop supports --auto-correct.
 Style/RedundantParentheses:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -256,7 +256,7 @@ Style/RedundantSelf:
 # SupportedStyles: single_quotes, double_quotes
 Style/StringLiterals:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -271,7 +271,7 @@ Style/TernaryParentheses:
 # SupportedStyles: snake_case, camelCase
 Naming/VariableName:
   Exclude:
-    - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v3.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 4469cd6..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-cache: bundler
-language: ruby
-dist: xenial
-os: linux
-
-jobs:
-  include:
-    - rvm: 2.5.8
-      env: "CHECK=rubocop"
-
-    - rvm: 2.5.8
-      env: "CHECK=test"
-
-    - rvm: jruby-9.2.12.0
-      env: "CHECK=test"
-
-script:
-  - "bundle exec rake $CHECK"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 28ec616..af092e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,218 +1,927 @@
-# Change Log
+# Changelog
 
-All notable changes to this project will be documented in this file.
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
 
-The format is based on
-[Keep a Changelog](http://keepachangelog.com)
-& makes a strong effort to adhere to
-[Semantic Versioning](http://semver.org).
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
 
-Tracking in this Changelog began for this project with the tagging of version 0.1.0.
-If you're looking for changes from before this, refer to the project's
-git logs & PR history.
+**Implemented enhancements:**
 
-## [Unreleased](https://github.com/puppetlabs/vmpooler/compare/0.11.0...master)
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-## [0.11.1](https://github.com/puppetlabs/vmpooler/compare/0.11.0...0.11.1)
+**Fixed bugs:**
 
-### Added
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
 
-- Reconnect to Redis if it fails to connect (POOLER-156)
+## [3.7.0](https://github.com/puppetlabs/vmpooler/tree/3.7.0) (2025-06-04)
 
-### Fixed/Security
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.6.0...3.7.0)
 
-- Update Gem dependencies
+**Implemented enhancements:**
 
-## [0.11.0](https://github.com/puppetlabs/vmpooler/compare/0.10.3...0.11.0)
+- \(P4DEVOPS-6096\) Include VMs that have been requested but not moved to pending when getting queue metrics [\#681](https://github.com/puppetlabs/vmpooler/pull/681) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump redis from 5.1.0 to 5.2.0 [\#675](https://github.com/puppetlabs/vmpooler/pull/675) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.1.0 to 13.2.1 [\#673](https://github.com/puppetlabs/vmpooler/pull/673) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.8 to 5.1.0 [\#665](https://github.com/puppetlabs/vmpooler/pull/665) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rspec from 3.12.0 to 3.13.0 [\#664](https://github.com/puppetlabs/vmpooler/pull/664) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.1 to 1.4.0 [\#663](https://github.com/puppetlabs/vmpooler/pull/663) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.43.0 to 0.44.0 [\#662](https://github.com/puppetlabs/vmpooler/pull/662) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.41.0 to 0.43.0 [\#658](https://github.com/puppetlabs/vmpooler/pull/658) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.18.0 to 0.19.0 [\#653](https://github.com/puppetlabs/vmpooler/pull/653) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump sinatra from 3.1.0 to 3.2.0 [\#652](https://github.com/puppetlabs/vmpooler/pull/652) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.40.0 to 0.41.0 [\#650](https://github.com/puppetlabs/vmpooler/pull/650) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.37.0 to 0.40.0 [\#643](https://github.com/puppetlabs/vmpooler/pull/643) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rake from 13.0.6 to 13.1.0 [\#638](https://github.com/puppetlabs/vmpooler/pull/638) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.2 to 1.3.0 [\#635](https://github.com/puppetlabs/vmpooler/pull/635) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Added
+**Fixed bugs:**
 
-- Option to provide path to additional configuration files (POOLER-157)
+- Bump opentelemetry-sdk from 1.4.0 to 1.4.1 [\#672](https://github.com/puppetlabs/vmpooler/pull/672) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8.1 to 2.2.9 [\#671](https://github.com/puppetlabs/vmpooler/pull/671) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.3.0 to 1.3.1 [\#668](https://github.com/puppetlabs/vmpooler/pull/668) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.8 to 2.2.8.1 [\#666](https://github.com/puppetlabs/vmpooler/pull/666) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump concurrent-ruby from 1.2.2 to 1.2.3 [\#660](https://github.com/puppetlabs/vmpooler/pull/660) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.1 to 6.4.2 [\#655](https://github.com/puppetlabs/vmpooler/pull/655) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.4.0 to 6.4.1 [\#654](https://github.com/puppetlabs/vmpooler/pull/654) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.22.2 to = 0.22.3 [\#646](https://github.com/puppetlabs/vmpooler/pull/646) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-concurrent\_ruby requirement from = 0.21.1 to = 0.21.2 [\#645](https://github.com/puppetlabs/vmpooler/pull/645) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump opentelemetry-sdk from 1.3.0 to 1.3.1 [\#642](https://github.com/puppetlabs/vmpooler/pull/642) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump prometheus-client from 4.2.1 to 4.2.2 [\#641](https://github.com/puppetlabs/vmpooler/pull/641) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump redis from 5.0.7 to 5.0.8 [\#637](https://github.com/puppetlabs/vmpooler/pull/637) ([dependabot[bot]](https://github.com/apps/dependabot))
+- \(RE-15817\) Reword fail warning and get error from redis before generating message [\#633](https://github.com/puppetlabs/vmpooler/pull/633) ([isaac-hammes](https://github.com/isaac-hammes))
 
-### Fixed
+**Merged pull requests:**
 
-- Pin Dockerfiles to use JRuby 9.2.9 instead of 9.2.11 to avoid a StackOverlow that was occuring.
-- Fixes or adds exceptions for all outstanding Rubocop issues and enables mandatory Rubocop checks on PRs
+- \(P4DEVOPS-6096\) Fix gems to prevent warnings in logs [\#685](https://github.com/puppetlabs/vmpooler/pull/685) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(maint\) Revert gems to last release [\#683](https://github.com/puppetlabs/vmpooler/pull/683) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump actions/setup-java from 3 to 4 [\#648](https://github.com/puppetlabs/vmpooler/pull/648) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/github-script from 6 to 7 [\#644](https://github.com/puppetlabs/vmpooler/pull/644) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-## [0.10.3](https://github.com/puppetlabs/vmpooler/compare/0.10.2...0.10.3)
+## [3.6.0](https://github.com/puppetlabs/vmpooler/tree/3.6.0) (2023-10-05)
 
-- Fix Redis update during migration to better reflect actual state (POOLER-154)
-- Update Rubocop config, now we can actually see all the failing cops
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.1...3.6.0)
 
-## [0.10.2](https://github.com/puppetlabs/vmpooler/compare/0.9.1...0.10.2)
+**Fixed bugs:**
 
-### Added
+- \(maint\) Fix message for timeout notification. [\#624](https://github.com/puppetlabs/vmpooler/pull/624) ([isaac-hammes](https://github.com/isaac-hammes))
 
-- API endpoint for resetting a pool clearing ready and pending instances (POOLER-153)
+**Merged pull requests:**
 
-## [0.9.1](https://github.com/puppetlabs/vmpooler/compare/0.9.0...0.9.1)
+- Bump rubocop from 1.56.3 to 1.56.4 [\#631](https://github.com/puppetlabs/vmpooler/pull/631) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.3.1 to 6.4.0 [\#630](https://github.com/puppetlabs/vmpooler/pull/630) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.2 to 1.56.3 [\#628](https://github.com/puppetlabs/vmpooler/pull/628) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/checkout from 3 to 4 [\#627](https://github.com/puppetlabs/vmpooler/pull/627) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.24.1 to = 0.24.2 [\#626](https://github.com/puppetlabs/vmpooler/pull/626) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.1 to 1.56.2 [\#625](https://github.com/puppetlabs/vmpooler/pull/625) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rubocop from 1.56.0 to 1.56.1 [\#623](https://github.com/puppetlabs/vmpooler/pull/623) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Added
+## [3.5.1](https://github.com/puppetlabs/vmpooler/tree/3.5.1) (2023-08-24)
 
-- Generate a wider set of human readable machine names
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.5.0...3.5.1)
 
-## [0.9.0](https://github.com/puppetlabs/vmpooler/compare/0.8.2...0.9.0)
+**Fixed bugs:**
 
-### Added
+- \(maint\) Fix bugs from redis and timeout notification updates. [\#621](https://github.com/puppetlabs/vmpooler/pull/621) ([isaac-hammes](https://github.com/isaac-hammes))
 
-- Support nested folders in find_cluster()
-- Add capability to set a max machine TTL (POOLER-123)
-- Add marked as failed stat
+## [3.5.0](https://github.com/puppetlabs/vmpooler/tree/3.5.0) (2023-08-23)
 
-## [0.8.2](https://github.com/puppetlabs/vmpooler/compare/0.8.1...0.8.2)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.4.0...3.5.0)
 
-### Added
+**Implemented enhancements:**
 
-- Uniqueness check on generated hostnames to prevent collisions
+- Improve LDAP auth [\#616](https://github.com/puppetlabs/vmpooler/issues/616)
+- \(maint\) Raise error when ip address is not given to vm after clone. [\#619](https://github.com/puppetlabs/vmpooler/pull/619) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(POD-8\) Add timeout\_notification config to log warning before vm is destroyed. [\#618](https://github.com/puppetlabs/vmpooler/pull/618) ([isaac-hammes](https://github.com/isaac-hammes))
+- \(RE-15565\) Add ability to use bind\_as with a service account [\#617](https://github.com/puppetlabs/vmpooler/pull/617) ([yachub](https://github.com/yachub))
 
-### Fixed
+**Merged pull requests:**
 
-- Update hostname_shorten regex that was causing problems with returning VMs
-- Rubocop linter fixes
+- Bump puma from 6.3.0 to 6.3.1 [\#615](https://github.com/puppetlabs/vmpooler/pull/615) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-## [0.8.1](https://github.com/puppetlabs/vmpooler/compare/0.7.2...0.8.1)
+## [3.4.0](https://github.com/puppetlabs/vmpooler/tree/3.4.0) (2023-08-18)
 
-### Added
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.3.0...3.4.0)
 
-- Make VM names human readable
+**Implemented enhancements:**
 
-## [0.7.2](https://github.com/puppetlabs/vmpooler/compare/0.7.1...0.7.2)
+- \(POD-10\) Log reason for failed VM checks. [\#611](https://github.com/puppetlabs/vmpooler/pull/611) ([isaac-hammes](https://github.com/isaac-hammes))
 
-### Fixed
+**Closed issues:**
 
-- Synchronize checkout operations across API threads (POOLER-150)
+- Log reason connection on port 22 of a failed VM [\#609](https://github.com/puppetlabs/vmpooler/issues/609)
 
-## [0.7.1](https://github.com/puppetlabs/vmpooler/compare/0.7.0...0.7.1)
+## [3.3.0](https://github.com/puppetlabs/vmpooler/tree/3.3.0) (2023-08-16)
 
-### Fixed
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.2.0...3.3.0)
 
-- Correctly detect create\_linked\_clone on a pool level (POOLER-147)
+**Closed issues:**
 
-## [0.7.0](https://github.com/puppetlabs/vmpooler/compare/0.6.3...0.7.0)
+- Redis 5.x Deprecations [\#603](https://github.com/puppetlabs/vmpooler/issues/603)
 
-### Added
+**Merged pull requests:**
 
-- Add capability to disable linked clones for vsphere provider (POOLER-147)
-- Add running host to VM data returned from /vm/hostname (POOLER-142)
+- Update rubocop requirement from ~\> 1.55.1 to ~\> 1.56.0 [\#608](https://github.com/puppetlabs/vmpooler/pull/608) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-## [0.6.3](https://github.com/puppetlabs/vmpooler/compare/0.6.2...0.6.3)
+## [3.2.0](https://github.com/puppetlabs/vmpooler/tree/3.2.0) (2023-08-10)
 
-### Added
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.1.0...3.2.0)
 
-- Add capability to configure pool cluster via config api (POOLER-143)
+**Implemented enhancements:**
 
-## [0.6.2](https://github.com/puppetlabs/vmpooler/compare/0.6.1...0.6.2)
+- \(maint\) Update opentelemetry gems. [\#606](https://github.com/puppetlabs/vmpooler/pull/606) ([isaac-hammes](https://github.com/isaac-hammes))
+- Bump jruby to 9.4.3.0 and bundle update [\#604](https://github.com/puppetlabs/vmpooler/pull/604) ([yachub](https://github.com/yachub))
 
-### Added
+**Fixed bugs:**
 
-- Validate a machine responds to vm\_ready? at checkout (POOLER-140)
+- \(RE-15692\) Do not attempt loading DNS classes if none are defined [\#602](https://github.com/puppetlabs/vmpooler/pull/602) ([yachub](https://github.com/yachub))
 
-## [0.6.1](https://github.com/puppetlabs/vmpooler/compare/0.6.0...0.6.1)
+**Closed issues:**
 
-### Added
+- Fix startup error when not using any dns plugins [\#601](https://github.com/puppetlabs/vmpooler/issues/601)
 
-- Vmpooler /status legacy api optimization
+**Merged pull requests:**
 
-## [0.6.0](https://github.com/puppetlabs/vmpooler/compare/0.5.1...0.6.0)
+- Bump prometheus-client from 4.1.0 to 4.2.1 [\#599](https://github.com/puppetlabs/vmpooler/pull/599) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.54.2 to ~\> 1.55.1 [\#597](https://github.com/puppetlabs/vmpooler/pull/597) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.7 to 2.2.8 [\#594](https://github.com/puppetlabs/vmpooler/pull/594) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.51.0 to ~\> 1.54.2 [\#593](https://github.com/puppetlabs/vmpooler/pull/593) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 6.2.2 to 6.3.0 [\#586](https://github.com/puppetlabs/vmpooler/pull/586) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump connection\_pool from 2.4.0 to 2.4.1 [\#583](https://github.com/puppetlabs/vmpooler/pull/583) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rubocop requirement from ~\> 1.50.1 to ~\> 1.51.0 [\#582](https://github.com/puppetlabs/vmpooler/pull/582) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump thor from 1.2.1 to 1.2.2 [\#581](https://github.com/puppetlabs/vmpooler/pull/581) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack from 2.2.6.4 to 2.2.7 [\#579](https://github.com/puppetlabs/vmpooler/pull/579) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Fixed
+## [3.1.0](https://github.com/puppetlabs/vmpooler/tree/3.1.0) (2023-05-01)
 
-- Ensure migrations and pending evaluations are processed FIFO (POOLER-141)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.0.0...3.1.0)
 
-### Added
+**Merged pull requests:**
 
-- Vmpooler pool statistic endpoint optimization
+- Bump rubocop from 1.50.1 to 1.50.2 [\#578](https://github.com/puppetlabs/vmpooler/pull/578) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update puma requirement from ~\> 5.0, \>= 5.0.4 to \>= 5.0.4, \< 7 [\#577](https://github.com/puppetlabs/vmpooler/pull/577) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-resource\_detectors requirement from = 0.19.1 to = 0.23.0 [\#576](https://github.com/puppetlabs/vmpooler/pull/576) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Migrate issue management to Jira [\#575](https://github.com/puppetlabs/vmpooler/pull/575) ([yachub](https://github.com/yachub))
+- Bump jruby to 9.4.2.0 [\#574](https://github.com/puppetlabs/vmpooler/pull/574) ([yachub](https://github.com/yachub))
+- Update rubocop requirement from ~\> 1.28.2 to ~\> 1.50.1 [\#573](https://github.com/puppetlabs/vmpooler/pull/573) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update sinatra requirement from ~\> 2.0 to \>= 2, \< 4 [\#572](https://github.com/puppetlabs/vmpooler/pull/572) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump net-ldap from 0.17.1 to 0.18.0 [\#571](https://github.com/puppetlabs/vmpooler/pull/571) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update prometheus-client requirement from ~\> 2.0 to \>= 2, \< 5 [\#566](https://github.com/puppetlabs/vmpooler/pull/566) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump rack-test from 2.0.2 to 2.1.0 [\#564](https://github.com/puppetlabs/vmpooler/pull/564) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update rack requirement from ~\> 2.2 to \>= 2.2, \< 4.0 [\#562](https://github.com/puppetlabs/vmpooler/pull/562) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-exporter-jaeger requirement from = 0.20.1 to = 0.22.0 [\#524](https://github.com/puppetlabs/vmpooler/pull/524) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-### Fixed
+## [3.0.0](https://github.com/puppetlabs/vmpooler/tree/3.0.0) (2023-03-28)
 
-- Ensure a checked out VM stays in a queue during checkout (POOLER-140)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.5.0...3.0.0)
 
-## [0.5.1](https://github.com/puppetlabs/vmpooler/compare/0.5.0...0.5.1)
+**Breaking changes:**
 
-## [0.5.0](https://github.com/puppetlabs/vmpooler/compare/0.4.0...0.5.0)
+- Direct Users to vmpooler-deployment [\#568](https://github.com/puppetlabs/vmpooler/pull/568) ([yachub](https://github.com/yachub))
+- \(RE-15124\) Implement DNS Plugins and Remove api v1 and v2 [\#551](https://github.com/puppetlabs/vmpooler/pull/551) ([yachub](https://github.com/yachub))
 
-### Fixed
+## [2.5.0](https://github.com/puppetlabs/vmpooler/tree/2.5.0) (2023-03-06)
 
-- Eliminate window for checked out VM to be discovered (POOLER-139)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.4.0...2.5.0)
 
-## [0.4.0](https://github.com/puppetlabs/vmpooler/compare/0.3.0...0.4.0)
+**Implemented enhancements:**
 
-### Fixed
+- \(RE-15161\) Use timeout builtin to TCPSocket when opening sockets. [\#555](https://github.com/puppetlabs/vmpooler/pull/555) ([isaac-hammes](https://github.com/isaac-hammes))
 
-- Improve support for configuration via environment variables (POOLER-137)
-- Support multiple pool backends per alias (POOLER-138)
-- Remove redis server testing requirement
+**Merged pull requests:**
 
-## [0.3.0](https://github.com/puppetlabs/vmpooler/compare/0.2.2...0.3.0)
+- Add docs and update actions [\#550](https://github.com/puppetlabs/vmpooler/pull/550) ([yachub](https://github.com/yachub))
+- \(RE-15111\) Migrate Snyk to Mend Scanning [\#546](https://github.com/puppetlabs/vmpooler/pull/546) ([yachub](https://github.com/yachub))
+- \(RE-14811\) Remove DIO as codeowners [\#517](https://github.com/puppetlabs/vmpooler/pull/517) ([yachub](https://github.com/yachub))
+- Add Snyk action and Move to RE org [\#511](https://github.com/puppetlabs/vmpooler/pull/511) ([yachub](https://github.com/yachub))
+- Add release-engineering to codeowners [\#508](https://github.com/puppetlabs/vmpooler/pull/508) ([yachub](https://github.com/yachub))
+- Update docker/Gemfile.lock [\#503](https://github.com/puppetlabs/vmpooler/pull/503) ([yachub](https://github.com/yachub))
 
-### Fixed
+## [2.4.0](https://github.com/puppetlabs/vmpooler/tree/2.4.0) (2022-07-25)
 
-- Sync pool size before dashboard is displayed (POOLER-132)
-- Remove a failed VM from the ready queue (POOLER-133)
-- Begin checking ready VMs to ensure alive after 1 minute by default
-- Ensure that metric nodes for vm usage stats are consistent
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.3.0...2.4.0)
 
-### Added
+**Merged pull requests:**
 
-- Add capability to ship VM usage metrics (POOLER-134)
+- \(maint\) Bump version to 2.4.0 [\#502](https://github.com/puppetlabs/vmpooler/pull/502) ([sbeaulie](https://github.com/sbeaulie))
+- \(bug\) Prevent failing VMs to be retried infinitely \(ondemand\) [\#501](https://github.com/puppetlabs/vmpooler/pull/501) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-3138\) vmpooler v2 api missing vm/hostname [\#500](https://github.com/puppetlabs/vmpooler/pull/500) ([sbeaulie](https://github.com/sbeaulie))
+- Update rubocop requirement from ~\> 1.1.0 to ~\> 1.28.2 [\#499](https://github.com/puppetlabs/vmpooler/pull/499) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump mock\_redis from 0.30.0 to 0.31.0 [\#496](https://github.com/puppetlabs/vmpooler/pull/496) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-redis requirement from = 0.21.2 to = 0.21.3 [\#494](https://github.com/puppetlabs/vmpooler/pull/494) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump puma from 5.5.2 to 5.6.4 [\#490](https://github.com/puppetlabs/vmpooler/pull/490) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update opentelemetry-instrumentation-http\_client requirement from = 0.19.3 to = 0.19.4 [\#478](https://github.com/puppetlabs/vmpooler/pull/478) ([dependabot[bot]](https://github.com/apps/dependabot))
 
-## [0.2.2](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
+## [2.3.0](https://github.com/puppetlabs/vmpooler/tree/2.3.0) (2022-04-07)
 
-### Fixed
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.2.0...2.3.0)
 
-- Return label used to request VMs when fulfilling VM requests (POOLER-131)
+**Merged pull requests:**
 
-## [0.2.1](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+- \(maint\) Fix deprecation warning for redis ruby library [\#489](https://github.com/puppetlabs/vmpooler/pull/489) ([sbeaulie](https://github.com/sbeaulie))
+- Add OTel HttpClient Instrumentation [\#477](https://github.com/puppetlabs/vmpooler/pull/477) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Update dev tooling and related docs [\#476](https://github.com/puppetlabs/vmpooler/pull/476) ([genebean](https://github.com/genebean))
+- \(DIO-2833\) Connect domain settings to pools, create v2 API [\#475](https://github.com/puppetlabs/vmpooler/pull/475) ([genebean](https://github.com/genebean))
 
-### Fixed
+## [2.2.0](https://github.com/puppetlabs/vmpooler/tree/2.2.0) (2021-12-30)
 
-- Better handle delta disk creation errors (POOLER-130)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.1.0...2.2.0)
 
-### Added
+**Merged pull requests:**
 
-- Re-write check\_pool in pool\_manager to improve readability
-- Add a docker-compose file for testing vmpooler
-- Add capability to weight backends when an alias spans multiple backends (POOLER-129)
+- Bump version to 2.2.0 [\#473](https://github.com/puppetlabs/vmpooler/pull/473) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Fix EXTRA\_CONFIG merge behavior [\#472](https://github.com/puppetlabs/vmpooler/pull/472) ([sbeaulie](https://github.com/sbeaulie))
+- Update to latest OTel gems [\#471](https://github.com/puppetlabs/vmpooler/pull/471) ([genebean](https://github.com/genebean))
+- Add additional data to spans in api/v1.rb [\#400](https://github.com/puppetlabs/vmpooler/pull/400) ([genebean](https://github.com/genebean))
 
-## [0.2.0](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+## [2.1.0](https://github.com/puppetlabs/vmpooler/tree/2.1.0) (2021-12-13)
 
-### Fixed
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/2.0.0...2.1.0)
 
-- (POOLER-128) VM specific mutex objects are not dereferenced when a VM is destroyed
-- A VM that is being destroyed is reported as discovered
+**Merged pull requests:**
 
-### Added
+- Ensure all configured providers are loaded [\#470](https://github.com/puppetlabs/vmpooler/pull/470) ([genebean](https://github.com/genebean))
+- \(maint\) Adding a provider method tag\_vm\_user [\#469](https://github.com/puppetlabs/vmpooler/pull/469) ([sbeaulie](https://github.com/sbeaulie))
+- Update testing.yml [\#468](https://github.com/puppetlabs/vmpooler/pull/468) ([sbeaulie](https://github.com/sbeaulie))
+- Move vsphere specific methods out of vmpooler [\#467](https://github.com/puppetlabs/vmpooler/pull/467) ([sbeaulie](https://github.com/sbeaulie))
+- Release prep for v2.0.0 [\#465](https://github.com/puppetlabs/vmpooler/pull/465) ([genebean](https://github.com/genebean))
 
-- Adds a new mechanism to load providers from any gem or file path
+## [2.0.0](https://github.com/puppetlabs/vmpooler/tree/2.0.0) (2021-12-08)
 
-## [0.1.0](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...master)
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.3.0...2.0.0)
 
-### Fixed
+**Merged pull requests:**
 
-- Remove unused method `find_pool` and related pending tests
-- Setting `max_tries` results in an infinite loop (POOLER-124)
-- Do not evaluate folders as VMs in `get_pool_vms` (POOLER-40)
-- Expire redis VM key when clone fails (POOLER-31)
-- Remove all usage of propertyCollector
-- Replace `find_vm` search mechanism (POOLER-68)
-- Fix configuration file loading (POOLER-103)
-- Update vulnerable dependencies (POOLER-101)
+- Use credentials file for Rubygems auth [\#466](https://github.com/puppetlabs/vmpooler/pull/466) ([genebean](https://github.com/genebean))
+- Add Gem release workflow [\#464](https://github.com/puppetlabs/vmpooler/pull/464) ([genebean](https://github.com/genebean))
+- Update icon in the readme to reference this repo [\#463](https://github.com/puppetlabs/vmpooler/pull/463) ([genebean](https://github.com/genebean))
+- \(DIO-2769\) Move vsphere provider to its own gem [\#462](https://github.com/puppetlabs/vmpooler/pull/462) ([genebean](https://github.com/genebean))
 
-### Added
+## [1.3.0](https://github.com/puppetlabs/vmpooler/tree/1.3.0) (2021-11-15)
 
-- Allow API and manager to run separately (POOLER-109)
-- Add configuration API endpoint (POOLER-107)
-- Add option to disable VM hostname mismatch checks
-- Add a gemspec file
-- Add time remaining information (POOLER-81)
-- Ship metrics for clone to ready time (POOLER-34)
-- Reduce duplicate checking of VMs
-- Reduce object lookups when retrieving VMs and folders
-- Optionally create delta disks for pool templates
-- Drop support for any ruby before 2.3
-- Add support for multiple LDAP search base DNs (POOLER-113)
-- Ensure a VM is only destroyed once (POOLER-112)
-- Add support for setting redis server port and password
-- Greatly reduce time it takes to add disks
-- Add Dockerfile that does not bundle redis
-- Add vmpooler.service to support systemd managing the service
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.2.0...1.3.0)
+
+**Merged pull requests:**
+
+- \(DIO-2675\) Undo pool size & template overrides [\#461](https://github.com/puppetlabs/vmpooler/pull/461) ([genebean](https://github.com/genebean))
+- \(DIO-2186\) Token migration [\#460](https://github.com/puppetlabs/vmpooler/pull/460) ([genebean](https://github.com/genebean))
+
+## [1.2.0](https://github.com/puppetlabs/vmpooler/tree/1.2.0) (2021-09-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.2...1.2.0)
+
+**Merged pull requests:**
+
+- \(DIO-2621\) Make LDAP encryption configurable [\#459](https://github.com/puppetlabs/vmpooler/pull/459) ([genebean](https://github.com/genebean))
+
+## [1.1.2](https://github.com/puppetlabs/vmpooler/tree/1.1.2) (2021-08-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.1...1.1.2)
+
+**Merged pull requests:**
+
+- \(DIO-541\) Fix jenkins and user usage metrics [\#458](https://github.com/puppetlabs/vmpooler/pull/458) ([yachub](https://github.com/yachub))
+
+## [1.1.1](https://github.com/puppetlabs/vmpooler/tree/1.1.1) (2021-08-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0...1.1.1)
+
+**Merged pull requests:**
+
+- \(POOLER-198\) Fix otel warning: Bump otel gems to 0.17.0 [\#457](https://github.com/puppetlabs/vmpooler/pull/457) ([yachub](https://github.com/yachub))
+
+## [1.1.0](https://github.com/puppetlabs/vmpooler/tree/1.1.0) (2021-08-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.1.0-rc.1...1.1.0)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#455](https://github.com/puppetlabs/vmpooler/pull/455) ([yachub](https://github.com/yachub))
+
+## [1.1.0-rc.1](https://github.com/puppetlabs/vmpooler/tree/1.1.0-rc.1) (2021-08-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/1.0.0...1.1.0-rc.1)
+
+**Merged pull requests:**
+
+- \(POOLER-176\) Add Operation Label to User Metric [\#454](https://github.com/puppetlabs/vmpooler/pull/454) ([yachub](https://github.com/yachub))
+- Update OTel gems to 0.15.0 [\#450](https://github.com/puppetlabs/vmpooler/pull/450) ([genebean](https://github.com/genebean))
+- Migrate testing to GH Actions from Travis [\#446](https://github.com/puppetlabs/vmpooler/pull/446) ([genebean](https://github.com/genebean))
+
+## [1.0.0](https://github.com/puppetlabs/vmpooler/tree/1.0.0) (2021-02-02)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.2...1.0.0)
+
+**Merged pull requests:**
+
+- Update OTel gems to 0.13.z [\#447](https://github.com/puppetlabs/vmpooler/pull/447) ([genebean](https://github.com/genebean))
+- \(DIO-1503\) Fix regex for ondemand instances [\#445](https://github.com/puppetlabs/vmpooler/pull/445) ([genebean](https://github.com/genebean))
+- \(maint\) Update lightstep pre-deploy ghaction to v0.2.6 [\#440](https://github.com/puppetlabs/vmpooler/pull/440) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.2](https://github.com/puppetlabs/vmpooler/tree/0.18.2) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.1...0.18.2)
+
+**Merged pull requests:**
+
+- Remove usage of redis multi from api [\#438](https://github.com/puppetlabs/vmpooler/pull/438) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Fix checkout counter allocation [\#437](https://github.com/puppetlabs/vmpooler/pull/437) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.18.1](https://github.com/puppetlabs/vmpooler/tree/0.18.1) (2020-11-10)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.18.0...0.18.1)
+
+**Merged pull requests:**
+
+- Update Puma to 5.0.4 from ~4.3 [\#436](https://github.com/puppetlabs/vmpooler/pull/436) ([genebean](https://github.com/genebean))
+- \(MAINT\) Fix checkout counter allocation [\#435](https://github.com/puppetlabs/vmpooler/pull/435) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-193\) Mark checked out VM as active [\#434](https://github.com/puppetlabs/vmpooler/pull/434) ([mattkirby](https://github.com/mattkirby))
+- Update to OTel 0.8.0 [\#432](https://github.com/puppetlabs/vmpooler/pull/432) ([genebean](https://github.com/genebean))
+- \(POOLER-192\) Use Rubocop 1.0 [\#423](https://github.com/puppetlabs/vmpooler/pull/423) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.18.0](https://github.com/puppetlabs/vmpooler/tree/0.18.0) (2020-10-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.17.0...0.18.0)
+
+**Merged pull requests:**
+
+- \(maint\) Speedup the tagging method [\#422](https://github.com/puppetlabs/vmpooler/pull/422) ([sbeaulie](https://github.com/sbeaulie))
+- \(DIO-1065\) Add lightstep gh action [\#421](https://github.com/puppetlabs/vmpooler/pull/421) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.17.0](https://github.com/puppetlabs/vmpooler/tree/0.17.0) (2020-10-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.3...0.17.0)
+
+**Merged pull requests:**
+
+- \(DIO-1059\) Optionally add snapshot tuning params at clone time [\#419](https://github.com/puppetlabs/vmpooler/pull/419) ([suckatrash](https://github.com/suckatrash))
+
+## [0.16.3](https://github.com/puppetlabs/vmpooler/tree/0.16.3) (2020-10-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.2...0.16.3)
+
+**Merged pull requests:**
+
+- \(POOLER-191\) Add checking for running instances that are not in active [\#418](https://github.com/puppetlabs/vmpooler/pull/418) ([mattkirby](https://github.com/mattkirby))
+
+## [0.16.2](https://github.com/puppetlabs/vmpooler/tree/0.16.2) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.1...0.16.2)
+
+**Merged pull requests:**
+
+- Bump OTel Sinatra to 0.7.1 [\#417](https://github.com/puppetlabs/vmpooler/pull/417) ([genebean](https://github.com/genebean))
+
+## [0.16.1](https://github.com/puppetlabs/vmpooler/tree/0.16.1) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.16.0...0.16.1)
+
+## [0.16.0](https://github.com/puppetlabs/vmpooler/tree/0.16.0) (2020-10-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.15.0...0.16.0)
+
+**Merged pull requests:**
+
+- Update to OTel 0.7.0 [\#416](https://github.com/puppetlabs/vmpooler/pull/416) ([genebean](https://github.com/genebean))
+
+## [0.15.0](https://github.com/puppetlabs/vmpooler/tree/0.15.0) (2020-09-30)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.9...0.15.0)
+
+**Merged pull requests:**
+
+- \(maint\) Centralize dependency management in the gemspec [\#407](https://github.com/puppetlabs/vmpooler/pull/407) ([sbeaulie](https://github.com/sbeaulie))
+- \(pooler-180\) Add healthcheck endpoint, spec testing [\#406](https://github.com/puppetlabs/vmpooler/pull/406) ([suckatrash](https://github.com/suckatrash))
+
+## [0.14.9](https://github.com/puppetlabs/vmpooler/tree/0.14.9) (2020-09-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.8...0.14.9)
+
+**Merged pull requests:**
+
+- Adding make to the other two Dockerfiles [\#405](https://github.com/puppetlabs/vmpooler/pull/405) ([genebean](https://github.com/genebean))
+
+## [0.14.8](https://github.com/puppetlabs/vmpooler/tree/0.14.8) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.7...0.14.8)
+
+**Merged pull requests:**
+
+- Fix mixup of gem placement. [\#404](https://github.com/puppetlabs/vmpooler/pull/404) ([genebean](https://github.com/genebean))
+
+## [0.14.7](https://github.com/puppetlabs/vmpooler/tree/0.14.7) (2020-09-18)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.6...0.14.7)
+
+**Merged pull requests:**
+
+- Add OTel resource detectors [\#401](https://github.com/puppetlabs/vmpooler/pull/401) ([genebean](https://github.com/genebean))
+- Add distributed tracing [\#399](https://github.com/puppetlabs/vmpooler/pull/399) ([genebean](https://github.com/genebean))
+
+## [0.14.6](https://github.com/puppetlabs/vmpooler/tree/0.14.6) (2020-09-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.5...0.14.6)
+
+**Merged pull requests:**
+
+- \(POOLER-184\) Pool manager retry and exit on failure [\#398](https://github.com/puppetlabs/vmpooler/pull/398) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Add promstats component check [\#397](https://github.com/puppetlabs/vmpooler/pull/397) ([rooneyshuman](https://github.com/rooneyshuman))
+- Test vmpooler on latest 2.5 [\#393](https://github.com/puppetlabs/vmpooler/pull/393) ([mattkirby](https://github.com/mattkirby))
+- Update rbvmomi requirement from ~\> 2.1 to \>= 2.1, \< 4.0 [\#391](https://github.com/puppetlabs/vmpooler/pull/391) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [0.14.5](https://github.com/puppetlabs/vmpooler/tree/0.14.5) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.4...0.14.5)
+
+**Merged pull requests:**
+
+- \(MAINT\) Fix Staledns error counter [\#396](https://github.com/puppetlabs/vmpooler/pull/396) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.4](https://github.com/puppetlabs/vmpooler/tree/0.14.4) (2020-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.3...0.14.4)
+
+**Merged pull requests:**
+
+- \(MAINT\) Normalise all tokens for stats [\#395](https://github.com/puppetlabs/vmpooler/pull/395) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.3](https://github.com/puppetlabs/vmpooler/tree/0.14.3) (2020-08-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.2...0.14.3)
+
+**Merged pull requests:**
+
+- \(POOLER-186\) Fix template alias evaluation with backend weight of 0 [\#394](https://github.com/puppetlabs/vmpooler/pull/394) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Clarity refactor of Prom Stats code [\#390](https://github.com/puppetlabs/vmpooler/pull/390) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.14.2](https://github.com/puppetlabs/vmpooler/tree/0.14.2) (2020-08-03)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.1...0.14.2)
+
+**Merged pull requests:**
+
+- Ensure lifetime is set when creating ondemand instances [\#392](https://github.com/puppetlabs/vmpooler/pull/392) ([mattkirby](https://github.com/mattkirby))
+- Fix vmpooler folder purging [\#389](https://github.com/puppetlabs/vmpooler/pull/389) ([mattkirby](https://github.com/mattkirby))
+
+## [0.14.1](https://github.com/puppetlabs/vmpooler/tree/0.14.1) (2020-07-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.14.0...0.14.1)
+
+**Merged pull requests:**
+
+- Correctly handle multiple pools of same alias in ondemand checkout [\#388](https://github.com/puppetlabs/vmpooler/pull/388) ([mattkirby](https://github.com/mattkirby))
+- Update travis config to remove deprecated style [\#387](https://github.com/puppetlabs/vmpooler/pull/387) ([rooneyshuman](https://github.com/rooneyshuman))
+- Update Dependabot config file [\#386](https://github.com/puppetlabs/vmpooler/pull/386) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.14.0](https://github.com/puppetlabs/vmpooler/tree/0.14.0) (2020-07-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.3...0.14.0)
+
+**Merged pull requests:**
+
+- Add a note on jruby 9.2.11 and redis connection pooling changes [\#384](https://github.com/puppetlabs/vmpooler/pull/384) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-167\) Allow for network configuration at vm clone time [\#382](https://github.com/puppetlabs/vmpooler/pull/382) ([rooneyshuman](https://github.com/rooneyshuman))
+- \(POOLER-160\) Add Prometheus Metrics to vmpooler [\#372](https://github.com/puppetlabs/vmpooler/pull/372) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.13.3](https://github.com/puppetlabs/vmpooler/tree/0.13.3) (2020-06-15)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.2...0.13.3)
+
+**Merged pull requests:**
+
+- \(POOLER-174\) Reduce duplicate of on demand code introduced in POOLER-158 [\#383](https://github.com/puppetlabs/vmpooler/pull/383) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.2](https://github.com/puppetlabs/vmpooler/tree/0.13.2) (2020-06-05)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.1...0.13.2)
+
+**Merged pull requests:**
+
+- Rescue and warn when graphite connection cannot be opened [\#379](https://github.com/puppetlabs/vmpooler/pull/379) ([mattkirby](https://github.com/mattkirby))
+
+## [0.13.1](https://github.com/puppetlabs/vmpooler/tree/0.13.1) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.13.0...0.13.1)
+
+**Merged pull requests:**
+
+- \(maint\) Fix merge issue [\#378](https://github.com/puppetlabs/vmpooler/pull/378) ([sbeaulie](https://github.com/sbeaulie))
+
+## [0.13.0](https://github.com/puppetlabs/vmpooler/tree/0.13.0) (2020-06-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.12.0...0.13.0)
+
+**Merged pull requests:**
+
+- \(POOLER-166\) Check for stale dns records [\#377](https://github.com/puppetlabs/vmpooler/pull/377) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-158\) Add support for ondemand provisioning [\#375](https://github.com/puppetlabs/vmpooler/pull/375) ([mattkirby](https://github.com/mattkirby))
+
+## [0.12.0](https://github.com/puppetlabs/vmpooler/tree/0.12.0) (2020-05-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.3...0.12.0)
+
+**Merged pull requests:**
+
+- \(POOLER-171\) Enable support for multiple user objects [\#376](https://github.com/puppetlabs/vmpooler/pull/376) ([rooneyshuman](https://github.com/rooneyshuman))
+
+## [0.11.3](https://github.com/puppetlabs/vmpooler/tree/0.11.3) (2020-04-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.2...0.11.3)
+
+**Merged pull requests:**
+
+- \(DIO-608\) vmpooler SUT handed out multiple times [\#374](https://github.com/puppetlabs/vmpooler/pull/374) ([sbeaulie](https://github.com/sbeaulie))
+- \(MAINT\) Update CODEOWNERS [\#373](https://github.com/puppetlabs/vmpooler/pull/373) ([jcoconnor](https://github.com/jcoconnor))
+
+## [0.11.2](https://github.com/puppetlabs/vmpooler/tree/0.11.2) (2020-04-16)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.1...0.11.2)
+
+**Merged pull requests:**
+
+- \(POOLER-161\) Fix extending vm lifetime when max lifetime is set [\#371](https://github.com/puppetlabs/vmpooler/pull/371) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-165\) Fix purge\_unconfigured\_folders [\#370](https://github.com/puppetlabs/vmpooler/pull/370) ([mattkirby](https://github.com/mattkirby))
+- Update rake requirement from ~\> 12.3 to \>= 12.3, \< 14.0 [\#369](https://github.com/puppetlabs/vmpooler/pull/369) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview))
+
+## [0.11.1](https://github.com/puppetlabs/vmpooler/tree/0.11.1) (2020-03-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.11.0...0.11.1)
+
+**Merged pull requests:**
+
+- Remove providers addition to docker-compose.yml [\#368](https://github.com/puppetlabs/vmpooler/pull/368) ([mattkirby](https://github.com/mattkirby))
+- Add Dependabot to keep gems updated [\#367](https://github.com/puppetlabs/vmpooler/pull/367) ([genebean](https://github.com/genebean))
+- Update gem dependencies to latest versions [\#366](https://github.com/puppetlabs/vmpooler/pull/366) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-156\) Detect redis connection failures [\#365](https://github.com/puppetlabs/vmpooler/pull/365) ([mattkirby](https://github.com/mattkirby))
+- Add a .dockerignore file [\#363](https://github.com/puppetlabs/vmpooler/pull/363) ([mattkirby](https://github.com/mattkirby))
+
+## [0.11.0](https://github.com/puppetlabs/vmpooler/tree/0.11.0) (2020-03-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.3...0.11.0)
+
+**Merged pull requests:**
+
+- Pin to JRuby 9.2.9 in Dockerfiles [\#362](https://github.com/puppetlabs/vmpooler/pull/362) ([highb](https://github.com/highb))
+- Manual Rubocop Fixes [\#361](https://github.com/puppetlabs/vmpooler/pull/361) ([highb](https://github.com/highb))
+- "Unsafe" rubocop fixes [\#360](https://github.com/puppetlabs/vmpooler/pull/360) ([highb](https://github.com/highb))
+- Fix Rubocop "safe" auto-corrections [\#359](https://github.com/puppetlabs/vmpooler/pull/359) ([highb](https://github.com/highb))
+- Remove duplicate of 0.10.2 from CHANGELOG [\#358](https://github.com/puppetlabs/vmpooler/pull/358) ([highb](https://github.com/highb))
+- \(POOLER-157\) Add extra\_config option to vmpooler [\#357](https://github.com/puppetlabs/vmpooler/pull/357) ([mattkirby](https://github.com/mattkirby))
+
+## [0.10.3](https://github.com/puppetlabs/vmpooler/tree/0.10.3) (2020-03-04)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.2...0.10.3)
+
+**Merged pull requests:**
+
+- Release 0.10.3 [\#356](https://github.com/puppetlabs/vmpooler/pull/356) ([highb](https://github.com/highb))
+- \(POOLER-154\) Delay vm host update until after migration completes [\#355](https://github.com/puppetlabs/vmpooler/pull/355) ([highb](https://github.com/highb))
+
+## [0.10.2](https://github.com/puppetlabs/vmpooler/tree/0.10.2) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.1...0.10.2)
+
+## [0.10.1](https://github.com/puppetlabs/vmpooler/tree/0.10.1) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.10.0...0.10.1)
+
+## [0.10.0](https://github.com/puppetlabs/vmpooler/tree/0.10.0) (2020-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.1...0.10.0)
+
+**Merged pull requests:**
+
+- Update changelog for 0.10.0 release [\#354](https://github.com/puppetlabs/vmpooler/pull/354) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-153\) Add endpoint for resetting a pool [\#353](https://github.com/puppetlabs/vmpooler/pull/353) ([mattkirby](https://github.com/mattkirby))
+
+## [0.9.1](https://github.com/puppetlabs/vmpooler/tree/0.9.1) (2020-01-28)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.9.0...0.9.1)
+
+**Merged pull requests:**
+
+- Generate a wider set of legal names [\#351](https://github.com/puppetlabs/vmpooler/pull/351) ([nicklewis](https://github.com/nicklewis))
+
+## [0.9.0](https://github.com/puppetlabs/vmpooler/tree/0.9.0) (2019-12-12)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.2...0.9.0)
+
+**Closed issues:**
+
+- find\_cluster in vsphere\_helper doesn't support host folders [\#205](https://github.com/puppetlabs/vmpooler/issues/205)
+
+**Merged pull requests:**
+
+- \(QENG-7531\) Add Marked as Failed Stat [\#350](https://github.com/puppetlabs/vmpooler/pull/350) ([jcoconnor](https://github.com/jcoconnor))
+- \(POOLER-123\) Implement a max TTL [\#349](https://github.com/puppetlabs/vmpooler/pull/349) ([sbeaulie](https://github.com/sbeaulie))
+- Support nested host folders in find\_cluster\(\) [\#348](https://github.com/puppetlabs/vmpooler/pull/348) ([seanmil](https://github.com/seanmil))
+- Update CHANGELOG for 0.8.2 [\#347](https://github.com/puppetlabs/vmpooler/pull/347) ([highb](https://github.com/highb))
+
+## [0.8.2](https://github.com/puppetlabs/vmpooler/tree/0.8.2) (2019-11-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.1...0.8.2)
+
+**Merged pull requests:**
+
+- Update rubocop configs [\#346](https://github.com/puppetlabs/vmpooler/pull/346) ([highb](https://github.com/highb))
+- \(QENG-7530\) Add check for unique hostnames [\#345](https://github.com/puppetlabs/vmpooler/pull/345) ([highb](https://github.com/highb))
+- \(QENG-7530\) Fix hostname\_shorten regex [\#344](https://github.com/puppetlabs/vmpooler/pull/344) ([highb](https://github.com/highb))
+- Update changelog for 0.8.1 release [\#343](https://github.com/puppetlabs/vmpooler/pull/343) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.1](https://github.com/puppetlabs/vmpooler/tree/0.8.1) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.8.0...0.8.1)
+
+**Merged pull requests:**
+
+- Add spicy-proton to vmpooler.gemspec [\#342](https://github.com/puppetlabs/vmpooler/pull/342) ([mattkirby](https://github.com/mattkirby))
+
+## [0.8.0](https://github.com/puppetlabs/vmpooler/tree/0.8.0) (2019-10-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.2...0.8.0)
+
+**Merged pull requests:**
+
+- \(QENG-7530\) Make VM names more human readable [\#341](https://github.com/puppetlabs/vmpooler/pull/341) ([highb](https://github.com/highb))
+
+## [0.7.2](https://github.com/puppetlabs/vmpooler/tree/0.7.2) (2019-10-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.1...0.7.2)
+
+**Merged pull requests:**
+
+- Simplify declaration of checkoutlock mutex [\#340](https://github.com/puppetlabs/vmpooler/pull/340) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-150\) Synchronize checkout operations for API [\#339](https://github.com/puppetlabs/vmpooler/pull/339) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-148\) Fix undefined variable bug in \_check\_ready\_vm. [\#338](https://github.com/puppetlabs/vmpooler/pull/338) ([quorten](https://github.com/quorten))
+- Add CODEOWNERS file to vmpooler [\#337](https://github.com/puppetlabs/vmpooler/pull/337) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.1](https://github.com/puppetlabs/vmpooler/tree/0.7.1) (2019-08-26)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.7.0...0.7.1)
+
+**Merged pull requests:**
+
+- \(POOLER-147\) Fix create\_linked\_clone pool option [\#336](https://github.com/puppetlabs/vmpooler/pull/336) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update changelog for 0.7.0 release [\#335](https://github.com/puppetlabs/vmpooler/pull/335) ([mattkirby](https://github.com/mattkirby))
+
+## [0.7.0](https://github.com/puppetlabs/vmpooler/tree/0.7.0) (2019-08-21)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.3...0.7.0)
+
+**Merged pull requests:**
+
+- \(POOLER-142\) Add running host to vm API data [\#334](https://github.com/puppetlabs/vmpooler/pull/334) ([mattkirby](https://github.com/mattkirby))
+- Make it possible to disable linked clones [\#333](https://github.com/puppetlabs/vmpooler/pull/333) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.3](https://github.com/puppetlabs/vmpooler/tree/0.6.3) (2019-07-29)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.2...0.6.3)
+
+**Closed issues:**
+
+- Named snapshots? [\#140](https://github.com/puppetlabs/vmpooler/issues/140)
+
+**Merged pull requests:**
+
+- \(POOLER-143\) Add clone\_target config change to API [\#332](https://github.com/puppetlabs/vmpooler/pull/332) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Update changelog for 0.6.2 [\#331](https://github.com/puppetlabs/vmpooler/pull/331) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.2](https://github.com/puppetlabs/vmpooler/tree/0.6.2) (2019-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.1...0.6.2)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Fix typo in domain [\#330](https://github.com/puppetlabs/vmpooler/pull/330) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-140\) Ensure a VM is alive at checkout [\#329](https://github.com/puppetlabs/vmpooler/pull/329) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.1](https://github.com/puppetlabs/vmpooler/tree/0.6.1) (2019-05-08)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.6.0...0.6.1)
+
+**Merged pull requests:**
+
+- Update Changelog ahead of building 0.6.1 [\#328](https://github.com/puppetlabs/vmpooler/pull/328) ([sbeaulie](https://github.com/sbeaulie))
+- Update API.md \[skip ci\] [\#327](https://github.com/puppetlabs/vmpooler/pull/327) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Optimize the status api using redis pipeline [\#326](https://github.com/puppetlabs/vmpooler/pull/326) ([sbeaulie](https://github.com/sbeaulie))
+- Update changelog ahead of 0.6.0 release. [\#325](https://github.com/puppetlabs/vmpooler/pull/325) ([mattkirby](https://github.com/mattkirby))
+
+## [0.6.0](https://github.com/puppetlabs/vmpooler/tree/0.6.0) (2019-04-24)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.1...0.6.0)
+
+**Merged pull requests:**
+
+- \(QENG-7201\) Vmpooler pool statistic endpoint optimization [\#324](https://github.com/puppetlabs/vmpooler/pull/324) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-141\) Fix order of processing migrating and pending queues [\#323](https://github.com/puppetlabs/vmpooler/pull/323) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Add bundler to dockerfile\_local [\#322](https://github.com/puppetlabs/vmpooler/pull/322) ([mattkirby](https://github.com/mattkirby))
+- Update changelog to 0.5.1 [\#321](https://github.com/puppetlabs/vmpooler/pull/321) ([mattkirby](https://github.com/mattkirby))
+
+## [0.5.1](https://github.com/puppetlabs/vmpooler/tree/0.5.1) (2019-04-11)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.5.0...0.5.1)
+
+**Merged pull requests:**
+
+- \(POOLER-140\) Ensure a running VM stays in a queue [\#320](https://github.com/puppetlabs/vmpooler/pull/320) ([mattkirby](https://github.com/mattkirby))
+- Fix Dockerfile link in readme and add note about http requests for dev [\#316](https://github.com/puppetlabs/vmpooler/pull/316) ([briancain](https://github.com/briancain))
+
+## [0.5.0](https://github.com/puppetlabs/vmpooler/tree/0.5.0) (2019-02-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.4.0...0.5.0)
+
+**Merged pull requests:**
+
+- \(POOLER-139\) Fix discovering checked out VM [\#318](https://github.com/puppetlabs/vmpooler/pull/318) ([mattkirby](https://github.com/mattkirby))
+
+## [0.4.0](https://github.com/puppetlabs/vmpooler/tree/0.4.0) (2019-02-06)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.3.0...0.4.0)
+
+**Merged pull requests:**
+
+- \(MAINT\) Update changelog for 0.4.0 release [\#315](https://github.com/puppetlabs/vmpooler/pull/315) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-138\) Support multiple pools per alias [\#314](https://github.com/puppetlabs/vmpooler/pull/314) ([mattkirby](https://github.com/mattkirby))
+- Update dockerfile jruby to 9.2 [\#313](https://github.com/puppetlabs/vmpooler/pull/313) ([mattkirby](https://github.com/mattkirby))
+- Stop testing ruby 2.3.x [\#312](https://github.com/puppetlabs/vmpooler/pull/312) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-137\) Support integer environment variables [\#311](https://github.com/puppetlabs/vmpooler/pull/311) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Update travis to test latest ruby [\#309](https://github.com/puppetlabs/vmpooler/pull/309) ([mattkirby](https://github.com/mattkirby))
+
+## [0.3.0](https://github.com/puppetlabs/vmpooler/tree/0.3.0) (2018-12-20)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.2...0.3.0)
+
+**Merged pull requests:**
+
+- Change version 0.2.2 to 0.3.0 [\#310](https://github.com/puppetlabs/vmpooler/pull/310) ([mattkirby](https://github.com/mattkirby))
+- Ensure nodes are consistent for usage stats [\#308](https://github.com/puppetlabs/vmpooler/pull/308) ([mattkirby](https://github.com/mattkirby))
+- Update changelog for 0.2.3 [\#307](https://github.com/puppetlabs/vmpooler/pull/307) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-134\) Ship VM usage stats [\#306](https://github.com/puppetlabs/vmpooler/pull/306) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-133\) Identify when a ready VM has failed [\#305](https://github.com/puppetlabs/vmpooler/pull/305) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-37\) Document HTTP responses [\#304](https://github.com/puppetlabs/vmpooler/pull/304) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-132\) Sync pool size on dashboard start [\#303](https://github.com/puppetlabs/vmpooler/pull/303) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.2](https://github.com/puppetlabs/vmpooler/tree/0.2.2) (2018-10-01)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
+
+**Merged pull requests:**
+
+- Update changelog version in preparation for release [\#302](https://github.com/puppetlabs/vmpooler/pull/302) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-131\) Return requested name when getting VMs [\#301](https://github.com/puppetlabs/vmpooler/pull/301) ([mattkirby](https://github.com/mattkirby))
+- Add docker-compose and dockerfile to support it [\#300](https://github.com/puppetlabs/vmpooler/pull/300) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.1](https://github.com/puppetlabs/vmpooler/tree/0.2.1) (2018-09-19)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.2.0...0.2.1)
+
+**Merged pull requests:**
+
+- Bump version for vmpooler in changelog [\#299](https://github.com/puppetlabs/vmpooler/pull/299) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-129\) Allow setting weights for backends [\#298](https://github.com/puppetlabs/vmpooler/pull/298) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-130\) Improve delta disk creation handling [\#297](https://github.com/puppetlabs/vmpooler/pull/297) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-114\) Refactor check\_pool in pool\_manager [\#296](https://github.com/puppetlabs/vmpooler/pull/296) ([mattkirby](https://github.com/mattkirby))
+
+## [0.2.0](https://github.com/puppetlabs/vmpooler/tree/0.2.0) (2018-07-25)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
+
+**Closed issues:**
+
+- create release [\#262](https://github.com/puppetlabs/vmpooler/issues/262)
+- Add API to delete a snapshot [\#163](https://github.com/puppetlabs/vmpooler/issues/163)
+
+**Merged pull requests:**
+
+- \(MAINT\) release 0.2.0 [\#294](https://github.com/puppetlabs/vmpooler/pull/294) ([mattkirby](https://github.com/mattkirby))
+- Remove VM from completed only after destroy [\#293](https://github.com/puppetlabs/vmpooler/pull/293) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-128\) Remove references to VM mutex when destroying [\#292](https://github.com/puppetlabs/vmpooler/pull/292) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Document config via environment [\#291](https://github.com/puppetlabs/vmpooler/pull/291) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) change domain to example.com [\#290](https://github.com/puppetlabs/vmpooler/pull/290) ([steveax](https://github.com/steveax))
+- Update entrypoint in dockerfile for vmpooler gem [\#289](https://github.com/puppetlabs/vmpooler/pull/289) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) release 0.1.0 [\#288](https://github.com/puppetlabs/vmpooler/pull/288) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-66\) Purge vms and folders no longer configured [\#274](https://github.com/puppetlabs/vmpooler/pull/274) ([mattkirby](https://github.com/mattkirby))
+- Adds a new mechanism to load providers from any gem or file path automatically [\#263](https://github.com/puppetlabs/vmpooler/pull/263) ([logicminds](https://github.com/logicminds))
+
+## [0.1.0](https://github.com/puppetlabs/vmpooler/tree/0.1.0) (2018-07-17)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...0.1.0)
+
+**Closed issues:**
+
+- jruby 1.7.8 does not support safe\_load [\#243](https://github.com/puppetlabs/vmpooler/issues/243)
+- YAML.safe\_load does not work with symbols in config file [\#240](https://github.com/puppetlabs/vmpooler/issues/240)
+- vmpooler fails to fetch vm with dummy provider [\#238](https://github.com/puppetlabs/vmpooler/issues/238)
+- Any interest in VRA7 support? [\#235](https://github.com/puppetlabs/vmpooler/issues/235)
+- Do not have a hardcoded list of VM providers [\#230](https://github.com/puppetlabs/vmpooler/issues/230)
+- Use a dynamic check\_pool period [\#226](https://github.com/puppetlabs/vmpooler/issues/226)
+- vmpooler doesn't seem to recognize ready VMs [\#218](https://github.com/puppetlabs/vmpooler/issues/218)
+- `find\_vmdks` in `vsphere\_helper` should not use `vmdk\_datastore.\_connection` [\#213](https://github.com/puppetlabs/vmpooler/issues/213)
+- `get\_base\_vm\_container\_from` in `vsphere\_helper` ensures the wrong connection [\#212](https://github.com/puppetlabs/vmpooler/issues/212)
+- `close` in vsphere\_helper throws an error if a connection was never made [\#211](https://github.com/puppetlabs/vmpooler/issues/211)
+- `find\_pool` in vsphere\_helper.rb has subtle errors [\#210](https://github.com/puppetlabs/vmpooler/issues/210)
+- `find\_pool` in vsphere\_helper tends to throw instead of returning nil for missing pools [\#209](https://github.com/puppetlabs/vmpooler/issues/209)
+- Vsphere connections are always insecure \(Ignore cert errors\) [\#207](https://github.com/puppetlabs/vmpooler/issues/207)
+- `find\_folder` in vsphere\_helper.rb has subtle errors [\#204](https://github.com/puppetlabs/vmpooler/issues/204)
+- Should not use `abort` in vsphere\_helper [\#203](https://github.com/puppetlabs/vmpooler/issues/203)
+- No reason why get\_snapshot\_list is defined in vsphere\_helper [\#202](https://github.com/puppetlabs/vmpooler/issues/202)
+- Setting max\_tries in configuration results in vSphereHelper going into infinite loop [\#199](https://github.com/puppetlabs/vmpooler/issues/199)
+- "connect.open" metric is doubled up if a connection is broken [\#195](https://github.com/puppetlabs/vmpooler/issues/195)
+- Remove the use of global variables in the vSphere helper [\#194](https://github.com/puppetlabs/vmpooler/issues/194)
+- Should exit Threads cleanly [\#193](https://github.com/puppetlabs/vmpooler/issues/193)
+- check\_ready\_vm unnecessarily calls open\_socket [\#185](https://github.com/puppetlabs/vmpooler/issues/185)
+- Feature Request: Add provider support [\#181](https://github.com/puppetlabs/vmpooler/issues/181)
+- Document all possible HTTP response codes for endpoints [\#166](https://github.com/puppetlabs/vmpooler/issues/166)
+- Add API to clone new VM from existing VM snapshot [\#165](https://github.com/puppetlabs/vmpooler/issues/165)
+- vsphere\_helper.rb: find\_least\_used\_host should warn if no suitable hosts are found [\#164](https://github.com/puppetlabs/vmpooler/issues/164)
+- find\_vm uses just hostname delta, vSphere searchIndex matches on FQDN [\#141](https://github.com/puppetlabs/vmpooler/issues/141)
+- Tagging does not support boolean values [\#135](https://github.com/puppetlabs/vmpooler/issues/135)
+- POST to /api/v1/token returns WEBrick::HTTPStatus::LengthRequired error [\#132](https://github.com/puppetlabs/vmpooler/issues/132)
+- vmpooler throwing exceptions [\#129](https://github.com/puppetlabs/vmpooler/issues/129)
+- NilClass error when running API without Graphite configured [\#81](https://github.com/puppetlabs/vmpooler/issues/81)
+- Manually removing VM's result in state mis-match [\#80](https://github.com/puppetlabs/vmpooler/issues/80)
+- Add support for customization specs [\#79](https://github.com/puppetlabs/vmpooler/issues/79)
+
+**Merged pull requests:**
+
+- \(maint\) Fix vmpooler require in bin/vmpooler [\#287](https://github.com/puppetlabs/vmpooler/pull/287) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Remove ruby 2.2.10 from travis config [\#286](https://github.com/puppetlabs/vmpooler/pull/286) ([mattkirby](https://github.com/mattkirby))
+- \(doc\) Add changelog and contributing guidlines [\#285](https://github.com/puppetlabs/vmpooler/pull/285) ([mattkirby](https://github.com/mattkirby))
+- \(MAINT\) Remove find\_pool and update pending tests [\#283](https://github.com/puppetlabs/vmpooler/pull/283) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-109\) Allow API to run independently [\#281](https://github.com/puppetlabs/vmpooler/pull/281) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-81\) Add time remaining information [\#280](https://github.com/puppetlabs/vmpooler/pull/280) ([smcelmurry](https://github.com/smcelmurry))
+- Revert "\(POOLER-81\) Add time\_remaining information" [\#279](https://github.com/puppetlabs/vmpooler/pull/279) ([smcelmurry](https://github.com/smcelmurry))
+- \(MAINT\) Fix test reference to find\_vm [\#278](https://github.com/puppetlabs/vmpooler/pull/278) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-34\) Ship clone request to ready time to metrics [\#277](https://github.com/puppetlabs/vmpooler/pull/277) ([smcelmurry](https://github.com/smcelmurry))
+- \(POOLER-81\) Add time\_remaining information [\#276](https://github.com/puppetlabs/vmpooler/pull/276) ([smcelmurry](https://github.com/smcelmurry))
+- Add jruby 9.2 to travis testing [\#275](https://github.com/puppetlabs/vmpooler/pull/275) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-124\) Fix evaluation of max\_tries [\#273](https://github.com/puppetlabs/vmpooler/pull/273) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-40\) Do not return folders with get\_pool\_vms [\#272](https://github.com/puppetlabs/vmpooler/pull/272) ([mattkirby](https://github.com/mattkirby))
+- Ensure template deltas are created once [\#271](https://github.com/puppetlabs/vmpooler/pull/271) ([mattkirby](https://github.com/mattkirby))
+- Do not run duplicate instances of inventory check for a pool [\#270](https://github.com/puppetlabs/vmpooler/pull/270) ([mattkirby](https://github.com/mattkirby))
+- Eliminate duplicate VM object lookups where possible [\#269](https://github.com/puppetlabs/vmpooler/pull/269) ([mattkirby](https://github.com/mattkirby))
+- Reduce object lookups for finding folders [\#268](https://github.com/puppetlabs/vmpooler/pull/268) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-113\) Add support for multiple LDAP search bases [\#267](https://github.com/puppetlabs/vmpooler/pull/267) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-31\) Expire redis vm key when clone fails [\#266](https://github.com/puppetlabs/vmpooler/pull/266) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-112\) Ensure a VM is only destroyed once [\#265](https://github.com/puppetlabs/vmpooler/pull/265) ([mattkirby](https://github.com/mattkirby))
+- Adds a gemspec file [\#264](https://github.com/puppetlabs/vmpooler/pull/264) ([logicminds](https://github.com/logicminds))
+- Change default vsphere connection behavior [\#261](https://github.com/puppetlabs/vmpooler/pull/261) ([mattkirby](https://github.com/mattkirby))
+- Remove propertyCollector from add\_disk [\#260](https://github.com/puppetlabs/vmpooler/pull/260) ([mattkirby](https://github.com/mattkirby))
+- Update ruby versions for travis [\#259](https://github.com/puppetlabs/vmpooler/pull/259) ([mattkirby](https://github.com/mattkirby))
+- Update to generic launcher [\#258](https://github.com/puppetlabs/vmpooler/pull/258) ([frozenfoxx](https://github.com/frozenfoxx))
+- Add support for setting redis port and password [\#257](https://github.com/puppetlabs/vmpooler/pull/257) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-107\) Add configuration API endpoint [\#256](https://github.com/puppetlabs/vmpooler/pull/256) ([mattkirby](https://github.com/mattkirby))
+- Create vmpooler.service [\#255](https://github.com/puppetlabs/vmpooler/pull/255) ([frozenfoxx](https://github.com/frozenfoxx))
+- \(POOLER-101\) Update nokogiri and net-ldap [\#254](https://github.com/puppetlabs/vmpooler/pull/254) ([mattkirby](https://github.com/mattkirby))
+- Add dockerfile without redis [\#253](https://github.com/puppetlabs/vmpooler/pull/253) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-103\) Fix configuration file loading [\#252](https://github.com/puppetlabs/vmpooler/pull/252) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-68\) Replace find\_vm search mechanism [\#251](https://github.com/puppetlabs/vmpooler/pull/251) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Add the last boot time for each pool [\#250](https://github.com/puppetlabs/vmpooler/pull/250) ([sbeaulie](https://github.com/sbeaulie))
+- Fix typo in error message [\#249](https://github.com/puppetlabs/vmpooler/pull/249) ([teancom](https://github.com/teancom))
+- Identify when ESXi host quickstats do not return [\#248](https://github.com/puppetlabs/vmpooler/pull/248) ([mattkirby](https://github.com/mattkirby))
+- Update jruby version for travis to 9.1.13.0 [\#247](https://github.com/puppetlabs/vmpooler/pull/247) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-96\) Setting the Rubygems version [\#246](https://github.com/puppetlabs/vmpooler/pull/246) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-93\) Extend API endpoint to provide just what is needed [\#245](https://github.com/puppetlabs/vmpooler/pull/245) ([sbeaulie](https://github.com/sbeaulie))
+- \(POOLER-92\) Add the alias information in the API status page for each… [\#244](https://github.com/puppetlabs/vmpooler/pull/244) ([sbeaulie](https://github.com/sbeaulie))
+- \(QENG-5305\) Improve vmpooler host selection [\#242](https://github.com/puppetlabs/vmpooler/pull/242) ([mattkirby](https://github.com/mattkirby))
+- Allow user to specify a configuration file in VMPOOLER\_CONFIG\_FILE variable [\#241](https://github.com/puppetlabs/vmpooler/pull/241) ([amcdson](https://github.com/amcdson))
+- Fix no implicit conversion to rational from nil [\#239](https://github.com/puppetlabs/vmpooler/pull/239) ([sbeaulie](https://github.com/sbeaulie))
+- Updated Vagrant box and associated docs [\#237](https://github.com/puppetlabs/vmpooler/pull/237) ([genebean](https://github.com/genebean))
+- \(GH-226\) Respond quickly to VMs being consumed [\#236](https://github.com/puppetlabs/vmpooler/pull/236) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-89\) Identify when config issue is present [\#234](https://github.com/puppetlabs/vmpooler/pull/234) ([mattkirby](https://github.com/mattkirby))
+- \(maint\) Update template delta script for moved vsphere credentials [\#233](https://github.com/puppetlabs/vmpooler/pull/233) ([ScottGarman](https://github.com/ScottGarman))
+- Fix rubocop [\#232](https://github.com/puppetlabs/vmpooler/pull/232) ([glennsarti](https://github.com/glennsarti))
+- \(GH-230\) Dynamically load VM Providers [\#231](https://github.com/puppetlabs/vmpooler/pull/231) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Remove phantom VMs that are in Redis but don't exist in provider [\#229](https://github.com/puppetlabs/vmpooler/pull/229) ([glennsarti](https://github.com/glennsarti))
+- Update find\_least\_used\_compatible\_host to specify pool [\#228](https://github.com/puppetlabs/vmpooler/pull/228) ([mattkirby](https://github.com/mattkirby))
+- \(GH-226\) Use a dynamic pool\_check loop period [\#227](https://github.com/puppetlabs/vmpooler/pull/227) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Update development documentation [\#225](https://github.com/puppetlabs/vmpooler/pull/225) ([glennsarti](https://github.com/glennsarti))
+- \(GH-213\) Remove use of private \_connection method [\#224](https://github.com/puppetlabs/vmpooler/pull/224) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-83\) Add ability to specify a datacenter for vsphere [\#223](https://github.com/puppetlabs/vmpooler/pull/223) ([glennsarti](https://github.com/glennsarti))
+- Added Vagrant setup and fixed the Dockerfile so it actually works [\#222](https://github.com/puppetlabs/vmpooler/pull/222) ([genebean](https://github.com/genebean))
+- Adding support for multiple vsphere providers [\#221](https://github.com/puppetlabs/vmpooler/pull/221) ([sbeaulie](https://github.com/sbeaulie))
+- Refactor get\_cluster\_host\_utilization method [\#220](https://github.com/puppetlabs/vmpooler/pull/220) ([sbeaulie](https://github.com/sbeaulie))
+- \(maint\) Pin rack to 1.x [\#219](https://github.com/puppetlabs/vmpooler/pull/219) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-72\)\(POOLER-70\)\(POOLER-52\) Move Pool Manager to use the VM Provider [\#216](https://github.com/puppetlabs/vmpooler/pull/216) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Emit console messages when debugging is enabled [\#215](https://github.com/puppetlabs/vmpooler/pull/215) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\)\(POOLER-52\) Create a functional vSphere Provider  [\#214](https://github.com/puppetlabs/vmpooler/pull/214) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop violations  [\#208](https://github.com/puppetlabs/vmpooler/pull/208) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix credentials in vsphere\_helper [\#200](https://github.com/puppetlabs/vmpooler/pull/200) ([glennsarti](https://github.com/glennsarti))
+- Update usage of global variablesin vsphere\_helper [\#198](https://github.com/puppetlabs/vmpooler/pull/198) ([mattkirby](https://github.com/mattkirby))
+- Remove duplicate of metrics.connect.open [\#197](https://github.com/puppetlabs/vmpooler/pull/197) ([mattkirby](https://github.com/mattkirby))
+- \(POOLER-73\) Add spec tests for vsphere\_helper [\#196](https://github.com/puppetlabs/vmpooler/pull/196) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Fix rubocop offenses [\#191](https://github.com/puppetlabs/vmpooler/pull/191) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Prepare to refactor VSphere code into a VM Provider [\#190](https://github.com/puppetlabs/vmpooler/pull/190) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-70\) Refactor clone\_vm to take pool configuration object [\#189](https://github.com/puppetlabs/vmpooler/pull/189) ([glennsarti](https://github.com/glennsarti))
+- \(GH-185\) Remove unnecessary checks in check\_ready\_vm  [\#188](https://github.com/puppetlabs/vmpooler/pull/188) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Only load rubocop rake tasks if gem is available [\#187](https://github.com/puppetlabs/vmpooler/pull/187) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Add rubocop and allow failures in Travis CI [\#183](https://github.com/puppetlabs/vmpooler/pull/183) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-73\) Update unit tests prior to refactoring [\#182](https://github.com/puppetlabs/vmpooler/pull/182) ([glennsarti](https://github.com/glennsarti))
+- \(POOLER-71\) Add dummy authentication provider [\#180](https://github.com/puppetlabs/vmpooler/pull/180) ([glennsarti](https://github.com/glennsarti))
+- \(maint\) Enhance VM Pooler developer experience [\#177](https://github.com/puppetlabs/vmpooler/pull/177) ([glennsarti](https://github.com/glennsarti))
+
+
+
+\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
diff --git a/CODEOWNERS b/CODEOWNERS
index 9c4ecfc..b47017c 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -1,10 +1,10 @@
 
-# This will cause DIO to be assigned review of any opened PRs against
+# This will cause RE to be assigned review of any opened PRs against
 # the branches containing this file.
 # See https://help.github.com/en/articles/about-code-owners for info on how to
 # take ownership of parts of the code base that should be reviewed by another
 # team.
 
-# DIO will be the default owners for everything in the repo.
-*                                     @puppetlabs/dio
+# RE will be the default owners for everything in the repo.
+*                                     @puppetlabs/release-engineering
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5d350eb..edf18f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,9 +1,6 @@
 # How to contribute
 
-Third-party patches are essential for keeping vmpooler great. We want to keep it as easy as possible to contribute changes that
-get things working in your environment. There are a few guidelines that we
-need contributors to follow so that we can have a chance of keeping on
-top of things.
+Third-party patches are essential for keeping VMPooler great. We want to keep it as easy as possible to contribute changes that get things working in your environment. There are a few guidelines that we need contributors to follow so that we can have a chance of keeping on top of things.
 
 ## Getting Started
 
@@ -18,28 +15,25 @@ top of things.
 
 * Create a topic branch from where you want to base your work.
   * This is usually the master branch.
-  * Only target release branches if you are certain your fix must be on that
-    branch.
-  * To quickly create a topic branch based on master; `git checkout -b
-    fix/master/my_contribution master`. Please avoid working directly on the
-    `master` branch.
+  * Only target release branches if you are certain your fix must be on that branch.
+  * To quickly create a topic branch based on master: `git checkout -b fix/master/my_contribution master`. Please avoid working directly on the `master` branch.
 * Make commits of logical units.
 * Check for unnecessary whitespace with `git diff --check` before committing.
 * Make sure your commit messages are in the proper format.
 
-````
-    (POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
+```plain
+(POOLER-1234) Make the example in CONTRIBUTING imperative and concrete
 
-    Without this patch applied the example commit message in the CONTRIBUTING
-    document is not a concrete example.  This is a problem because the
-    contributor is left to imagine what the commit message should look like
-    based on a description rather than an example.  This patch fixes the
-    problem by making the example concrete and imperative.
+Without this patch applied the example commit message in the CONTRIBUTING
+document is not a concrete example.  This is a problem because the
+contributor is left to imagine what the commit message should look like
+based on a description rather than an example.  This patch fixes the
+problem by making the example concrete and imperative.
 
-    The first line is a real life imperative statement with a ticket number
-    from our issue tracker.  The body describes the behavior without the patch,
-    why this is a problem, and how the patch fixes the problem when applied.
-````
+The first line is a real life imperative statement with a ticket number
+from our issue tracker.  The body describes the behavior without the patch,
+why this is a problem, and how the patch fixes the problem when applied.
+```
 
 * Make sure you have added the necessary tests for your changes.
 * Run _all_ the tests to assure nothing else was accidentally broken.
@@ -48,36 +42,32 @@ top of things.
 
 ### Documentation
 
-For changes of a trivial nature to comments and documentation, it is not
-always necessary to create a new ticket in Jira. In this case, it is
-appropriate to start the first line of a commit with '(doc)' instead of
-a ticket number.
+For changes of a trivial nature to comments and documentation, it is not always necessary to create a new ticket in Jira. In this case, it is appropriate to start the first line of a commit with '(doc)' instead of a ticket number.
 
-````
-    (doc) Add documentation commit example to CONTRIBUTING
+```plain
+(doc) Add documentation commit example to CONTRIBUTING
 
-    There is no example for contributing a documentation commit
-    to the Puppet repository. This is a problem because the contributor
-    is left to assume how a commit of this nature may appear.
+There is no example for contributing a documentation commit
+to the Puppet repository. This is a problem because the contributor
+is left to assume how a commit of this nature may appear.
 
-    The first line is a real life imperative statement with '(doc)' in
-    place of what would have been the ticket number in a
-    non-documentation related commit. The body describes the nature of
-    the new documentation or comments added.
-````
+The first line is a real life imperative statement with '(doc)' in
+place of what would have been the ticket number in a
+non-documentation related commit. The body describes the nature of
+the new documentation or comments added.
+```
 
 ## Submitting Changes
 
-* Sign the [Contributor License Agreement](http://links.puppetlabs.com/cla).
+* Sign the Contributor License Agreement.
 * Push your changes to a topic branch in your fork of the repository.
 * Submit a pull request to the repository in the puppetlabs organization.
 * Update your Jira ticket to mark that you have submitted code and are ready for it to be reviewed (Status: Ready for Merge).
   * Include a link to the pull request in the ticket.
-* The Puppet SRE team looks at Pull Requests on a regular basis.
-* After feedback has been given we expect responses within two weeks. After two
-  weeks we may close the pull request if it isn't showing any activity.
+* The Puppet Release Engineering team looks at Pull Requests on a regular basis.
+* After feedback has been given we expect responses within two weeks. After two weeks we may close the pull request if it isn't showing any activity.
 
-# Additional Resources
+## Additional Resources
 
 * [Puppet Labs community guildelines](http://docs.puppetlabs.com/community/community_guidelines.html)
 * [Bug tracker (Jira)](http://tickets.puppetlabs.com)
diff --git a/Gemfile b/Gemfile
index 122d6b5..0313b80 100644
--- a/Gemfile
+++ b/Gemfile
@@ -3,11 +3,11 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 gemspec
 
 # Evaluate Gemfile.local if it exists
-if File.exists? "#{__FILE__}.local"
+if File.exist? "#{__FILE__}.local"
   instance_eval(File.read("#{__FILE__}.local"))
 end
 
 # Evaluate ~/.gemfile if it exists
-if File.exists?(File.join(Dir.home, '.gemfile'))
+if File.exist?(File.join(Dir.home, '.gemfile'))
   instance_eval(File.read(File.join(Dir.home, '.gemfile')))
 end
diff --git a/Gemfile.lock b/Gemfile.lock
new file mode 100644
index 0000000..a63b584
--- /dev/null
+++ b/Gemfile.lock
@@ -0,0 +1,219 @@
+PATH
+  remote: .
+  specs:
+    vmpooler (3.8.1)
+      concurrent-ruby (~> 1.1)
+      connection_pool (~> 2.4)
+      deep_merge (~> 1.2)
+      net-ldap (~> 0.16)
+      opentelemetry-exporter-jaeger (= 0.23.0)
+      opentelemetry-instrumentation-concurrent_ruby (= 0.21.1)
+      opentelemetry-instrumentation-http_client (= 0.22.2)
+      opentelemetry-instrumentation-rack (= 0.23.4)
+      opentelemetry-instrumentation-redis (= 0.25.3)
+      opentelemetry-instrumentation-sinatra (= 0.23.2)
+      opentelemetry-resource_detectors (= 0.24.2)
+      opentelemetry-sdk (~> 1.8)
+      pickup (~> 0.0.11)
+      prometheus-client (>= 2, < 5)
+      puma (>= 5.0.4, < 7)
+      rack (>= 2.2, < 4.0)
+      rake (~> 13.0)
+      redis (~> 5.0)
+      sinatra (>= 2, < 4)
+      spicy-proton (~> 2.1)
+      statsd-ruby (~> 1.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    base64 (0.1.2)
+    bindata (2.5.1)
+    builder (3.3.0)
+    climate_control (1.2.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    deep_merge (1.2.2)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    faraday (2.13.1)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
+    ffi (1.17.2-java)
+    google-cloud-env (2.2.1)
+      faraday (>= 1.0, < 3.a)
+    json (2.12.2)
+    json (2.12.2-java)
+    language_server-protocol (3.17.0.5)
+    logger (1.7.0)
+    method_source (1.1.0)
+    mock_redis (0.37.0)
+    mustermann (3.0.3)
+      ruby2_keywords (~> 0.0.1)
+    net-http (0.6.0)
+      uri
+    net-ldap (0.19.0)
+    nio4r (2.7.4)
+    nio4r (2.7.4-java)
+    opentelemetry-api (1.5.0)
+    opentelemetry-common (0.20.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-jaeger (0.23.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+      thrift
+    opentelemetry-instrumentation-base (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-concurrent_ruby (0.21.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http_client (0.22.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rack (0.23.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-redis (0.25.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-sinatra (0.23.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-resource_detectors (0.24.2)
+      google-cloud-env
+      opentelemetry-sdk (~> 1.0)
+    opentelemetry-sdk (1.8.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.11.0)
+      opentelemetry-api (~> 1.0)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
+    pickup (0.0.11)
+    prism (1.4.0)
+    prometheus-client (4.2.4)
+      base64
+    pry (0.15.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry (0.15.2-java)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+      spoon (~> 0.0)
+    puma (6.6.0)
+      nio4r (~> 2.0)
+    puma (6.6.0-java)
+      nio4r (~> 2.0)
+    racc (1.8.1)
+    racc (1.8.1-java)
+    rack (2.2.17)
+    rack-protection (3.2.0)
+      base64 (>= 0.1.0)
+      rack (~> 2.2, >= 2.2.4)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    redis (5.4.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.24.0)
+      connection_pool
+    regexp_parser (2.10.0)
+    rexml (3.4.1)
+    rspec (3.13.1)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.4)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.4)
+    rubocop (1.56.4)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.1)
+    simplecov_json_formatter (0.1.4)
+    sinatra (3.2.0)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.2.0)
+      tilt (~> 2.0)
+    spicy-proton (2.1.15)
+      bindata (~> 2.3)
+    spoon (0.0.6)
+      ffi
+    statsd-ruby (1.5.0)
+    thor (1.3.2)
+    thrift (0.22.0)
+    tilt (2.6.0)
+    unicode-display_width (2.6.0)
+    uri (1.0.3)
+    yarjuf (2.0.0)
+      builder
+      rspec (~> 3)
+
+PLATFORMS
+  arm64-darwin-22
+  arm64-darwin-23
+  arm64-darwin-25
+  universal-java-11
+  universal-java-17
+  x86_64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  climate_control (>= 0.2.0)
+  mock_redis (= 0.37.0)
+  pry
+  rack-test (>= 0.6)
+  rspec (>= 3.2)
+  rubocop (~> 1.56.0)
+  simplecov (>= 0.11.2)
+  thor (~> 1.0, >= 1.0.1)
+  vmpooler!
+  yarjuf (>= 2.0)
+
+BUNDLED WITH
+   2.4.18
diff --git a/PROVIDER_API.md b/PROVIDER_API.md
deleted file mode 100644
index 0210c3b..0000000
--- a/PROVIDER_API.md
+++ /dev/null
@@ -1,83 +0,0 @@
-# Provider API
-
-## Create a new provider gem from scratch
-
-### Requirements
-1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
-2. the main provider code file should be named the same at the name of the provider. ie. (vpshere == lib/vmpooler/providers/vsphere.rb)
-3. The gem must be installed on the same machine as vmpooler
-4. The provider name must be referenced in the vmpooler config file in order for it to be loaded.
-5. Your gem name or repository name should contain vmpooler-<name>-provider so the community can easily search provider plugins
-   for vmpooler.
-### 1. Use bundler to create the provider scaffolding
-
-```
-bundler gem --test=rspec --no-exe --no-ext vmpooler-spoof-provider
-cd vmpooler-providers-spoof/
-mkdir -p ./lib/vmpooler/providers
-cd ./lib/vmpooler/providers
-touch spoof.rb
-
-```
-
-There may be some boilerplate files there were generated, just delete those.
-
-### 2. Create the main provider file
-Ensure the main provider file uses the following code.
-
-
-```ruby
-# lib/vmpooler/providers/spoof.rb
-require 'yaml'
-require 'vmpooler/providers/base'
-
-module Vmpooler
-  class PoolManager
-    class Provider
-      class Spoof < Vmpooler::PoolManager::Provider::Base
-      
-      # at this time it is not documented which methods should be implemented
-      # have a look at the vmpooler/providers/vpshere provider for examples
-
-      end
-
-    end
-  end
-end
-  
-
-```
-
-### 3. Fill out your gemspec
-Ensure you fill out your gemspec file to your specifications.  If you need a dependency please make sure you require them.
-
-`spec.add_dependency "vmware", "~> 1.15"`.
-
-At a minimum you may want to add the vmpooler gem as a dev dependency so you can use it during testing.
-
-`spec.add_dev_dependency "vmpooler", "~> 1.15"`
-
-or in your Gemfile
-
-```ruby
-
-gem 'vmpooler', github: 'puppetlabs/vmpooler'
-```
-
-Also make sure this dependency can be loaded by jruby.  If the dependency cannot be used by jruby don't use it.
-
-### 4. Create some tests
-Your provider code should be tested before releasing.  Copy and refactor some tests from the vmpooler gem under 
-`spec/unit/providers/dummy_spec.rb`
-
-### 5. Publish
-Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
-
-
-## Available Third Party Providers
-Be the first to update this list.  Create a provider today!
-
-
-## Example provider
-You can use the following [repo as an example](https://github.com/logicminds/vmpooler-vsphere-provider) of how to setup your provider gem.
-
diff --git a/README.md b/README.md
index 66b9bd9..a05bcec 100644
--- a/README.md
+++ b/README.md
@@ -1,41 +1,101 @@
-![vmpooler](https://raw.github.com/sschneid/vmpooler/master/lib/vmpooler/public/img/logo.gif)
+![VMPooler](lib/vmpooler/public/img/logo.png)
 
-# vmpooler
+# VMPooler
 
-vmpooler provides configurable 'pools' of instantly-available (running) virtual machines.
+- [VMPooler](#vmpooler)
+  - [Usage](#usage)
+    - [Migrating to v3](#migrating-to-v3)
+    - [v2.0.0 note](#v200-note)
+  - [Installation](#installation)
+    - [Dependencies](#dependencies)
+      - [Redis](#redis)
+      - [Other gems](#other-gems)
+  - [Configuration](#configuration)
+  - [Components](#components)
+    - [API](#api)
+    - [Dashboard](#dashboard)
+  - [Related tools and resources](#related-tools-and-resources)
+    - [Command-line Utility](#command-line-utility)
+    - [Vagrant plugin](#vagrant-plugin)
+  - [Development](#development)
+    - [docker-compose](#docker-compose)
+    - [Running docker-compose inside Vagrant](#running-docker-compose-inside-vagrant)
+    - [URLs when using docker-compose](#urls-when-using-docker-compose)
+  - [Update the Gemfile Lock](#update-the-gemfile-lock)
+  - [Releasing](#releasing)
+  - [License](#license)
 
+VMPooler provides configurable 'pools' of instantly-available (pre-provisioned) and/or on-demand (provisioned on request) virtual machines.
 
 ## Usage
 
-At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. Vmpooler manages the lifecycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand.
+At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. VMPooler manages the life cycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand.
+
+The recommended method for deploying VMPooler is via [https://github.com/puppetlabs/vmpooler-deployment](vmpooler-deployment).
+
+### Migrating to v3
+
+Starting with the v3.x release, management of DNS records is implemented as DNS plugins, similar to compute providers. This means each pool configuration should be pointing to a configuration object in `:dns_config` to determine it's method of record management.
+
+For those using the global `DOMAIN` environment variable or global `:config.domain` key, this means records were not previously being managed by VMPooler (presumably managed via dynamic dns), so it's value should be moved to `:dns_configs:<INSERT_YOUR_OWN_SYMBOL>:domain` with the value for `dns_class` for the config set to `dynamic-dns`.
+
+For example, the following < v3.x configuration:
+
+```yaml
+:config:
+  domain: 'example.com'
+```
+
+becomes:
+
+```yaml
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+```
+
+Then any pools that should have records created via the dns config above should now reference the named dns config in the `dns_plugin` key:
+
+```yaml
+:pools:
+  - name: 'debian-8-x86_64'
+    dns_plugin: 'example'
+```
+
+For those using the GCE provider, [vmpooler-provider-gce](https://github.com/puppetlabs/vmpooler-provider-gce), as of version 1.x the DNS management has been decoupled. See <https://github.com/puppetlabs/vmpooler-provider-gce#migrating-to-v1>
+
+### v2.0.0 note
+
+As of version 2.0.0, all providers other than the dummy one are now separate gems. Historically the vSphere provider was included within VMPooler itself. That code has been moved to the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository and the `vmpooler-provider-vsphere` gem. To migrate from VMPooler 1.x to 2.0 you will need to ensure that `vmpooler-provider-vsphere` is installed along side the `vmpooler` gem. See the  [Provider API](docs/PROVIDER_API.md) docs for more information.
 
 ## Installation
 
-### Prerequisites
-
-vmpooler is available as a gem
-
-To use the gem `gem install vmpooler`
+The recommended method of installation is via the Helm chart located in [puppetlabs/vmpooler-deployment](https://github.com/puppetlabs/vmpooler-deployment). That repository also provides Docker images of VMPooler.
 
 ### Dependencies
 
-Vmpooler requires a [Redis](http://redis.io/) server. This is the datastore used for vmpooler's inventory and queueing services.
+#### Redis
 
-### Configuration
+VMPooler requires a [Redis](http://redis.io/) server. This is the data store used for VMPooler's inventory and queuing services.
 
-Configuration for vmpooler may be provided via environment variables, or a configuration file.
+#### Other gems
 
-Note on jruby 9.2.11.x: We have found when running vmpooler on jruby 9.2.11.x we occasionally encounter a stack overflow error that causes the pool\_manager application component to fail and stop doing work. To address this issue on jruby 9.2.11.x we recommend setting the jruby option 'invokedynamic.yield=false'. To set this with jruby 9.2.11.1 you can specify the environment variable 'JRUBY\_OPTS' with the value '-Xinvokedynamic.yield=false'.
+VMPooler itself and the dev environment talked about below require additional Ruby gems to function. You can update the currently required ones for VMPooler by running `./update-gemfile-lock.sh`. The gems for the dev environment can be updated by running `./docker/update-gemfile-lock.sh`. These scripts will utilize the container on the FROM line of the Dockerfile to update the Gemfile.lock in the root of this repo and in the docker folder, respectively.
 
-The provided configuration defaults are reasonable for  small vmpooler instances with a few pools. If you plan to run a large vmpooler instance it is important to consider configuration values appropriate for the instance of your size in order to avoid starving the provider, or redis, of connections.
+## Configuration
 
-As of vmpooler 0.13.x redis uses a connection pool to improve efficiency and ensure thread safe usage. At Puppet, we run an instance with about 100 pools at any given time. We have to provide it with 200 redis connections to the redis connection pool, and a timeout for connections of 40 seconds, to avoid timeouts. Because metrics are generated for connection available and waited your metrics provider will need to be able to cope with this volume. Statsd is recommended to ensure metrics get delivered reliably.
+Configuration for VMPooler may be provided via environment variables, or a configuration file.
 
-Please see this [configuration](docs/configuration.md) document for more details about configuring vmpooler via environment variables.
+The provided configuration defaults are reasonable for  small VMPooler instances with a few pools. If you plan to run a large VMPooler instance it is important to consider configuration values appropriate for the instance of your size in order to avoid starving the provider, or Redis, of connections.
+
+VMPooler uses a connection pool for Redis to improve efficiency and ensure thread safe usage. At Puppet, we run an instance with about 100 pools at any given time. We have to provide it with 200 Redis connections to the Redis connection pool, and a timeout for connections of 40 seconds, to avoid timeouts. Because metrics are generated for connection available and waited, your metrics provider will need to be able to cope with this volume. Prometheus or StatsD is recommended to ensure metrics get delivered reliably.
+
+Please see this [configuration](docs/configuration.md) document for more details about configuring VMPooler via environment variables.
 
 The following YAML configuration sets up two pools, `debian-7-i386` and `debian-7-x86_64`, which contain 5 running VMs each:
 
-```
+```yaml
 ---
 :providers:
   :vsphere:
@@ -68,45 +128,13 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian-
 
 See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for additional configuration options and parameters or for supporting multiple providers.
 
-### Running via Docker
+## Components
 
-A [Dockerfile](/docker/Dockerfile) is included in this repository to allow running vmpooler inside a Docker container. A configuration file can be used via volume mapping, and specifying the destination as the configuration file via environment variables, or the application can be configured with environment variables alone. The Dockerfile provides an entrypoint so you may choose whether to run API, or manager services. The default behavior will run both. To build and run:
-
-```
-docker build -t vmpooler . && docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-```
-
-To run only the API and dashboard
-
-```
-docker run -p 80:4567 -it vmpooler api
-```
-
-To run only the manager component
-
-```
-docker run -it vmpooler manager
-```
-
-### docker-compose
-
-A docker-compose file is provided to support running vmpooler easily via docker-compose. This is useful for development because your local code is used to build the gem used in the docker-compose environment.
-
-```
-docker-compose -f docker/docker-compose.yml up
-```
-
-### Running Docker inside Vagrant
-
-A vagrantfile is included in this repository. Please see [vagrant instructions](docs/vagrant.md) for details.
-
-## API and Dashboard
-
-vmpooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
+VMPooler provides an API and web front-end (dashboard) on port `:4567`.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on.
 
 ### API
 
-vmpooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
+VMPooler provides a REST API for VM management.  See the [API documentation](docs/API.md) for more information.
 
 ### Dashboard
 
@@ -116,23 +144,71 @@ A dashboard is provided to offer real-time statistics and historical graphs.  It
 
 [Graphite](http://graphite.wikidot.com/) is required for historical data retrieval.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for details.
 
-## Command-line Utility
+## Related tools and resources
 
-- [vmfloaty](https://github.com/briancain/vmfloaty) is a ruby based CLI tool and scripting library written in ruby.
+### Command-line Utility
 
-## Vagrant plugin
+- [vmfloaty](https://github.com/puppetlabs/vmfloaty) is a ruby based CLI tool and scripting library. We consider it the primary way for users to interact with VMPooler.
 
-- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler) Use Vagrant to create and manage your vmpooler instances.
+### Vagrant plugin
 
-## Development and further documentation
+- [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler): Use Vagrant to create and manage your VMPooler instances.
 
-For more information about setting up a development instance of vmpooler or other subjects, see the [docs/](docs) directory.
+## Development
 
-## Build status
+### docker-compose
 
-[![Build Status](https://travis-ci.org/puppetlabs/vmpooler.png?branch=master)](https://travis-ci.org/puppetlabs/vmpooler)
+A docker-compose file is provided to support running VMPooler and associated tools locally. This is useful for development because your local code is used to build the gem used in the docker-compose environment. The compose environment also pulls in the latest providers via git. Details of this setup are stored in the `docker/` folder.
 
+```bash
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+### Running docker-compose inside Vagrant
+
+A Vagrantfile is included in this repository so as to provide a reproducible development environment.
+
+```bash
+vagrant up
+vagrant ssh
+cd /vagrant
+docker-compose -f docker/docker-compose.yml build && \
+docker-compose -f docker/docker-compose.yml up
+```
+
+The Vagrant environment also contains multiple rubies you can utilize for spec test and the like. You can see a list of the pre-installed ones when you log in as part of the message of the day.
+
+For more information about setting up a development instance of VMPooler or other subjects, see the [docs/](docs) directory.
+
+### URLs when using docker-compose
+
+| Endpoint          | URL                                                                   |
+|-------------------|-----------------------------------------------------------------------|
+| Redis Commander   | [http://localhost:8079](http://localhost:8079)                        |
+| API               | [http://localhost:8080/api/v1]([http://localhost:8080/api/v1)         |
+| Dashboard         | [http://localhost:8080/dashboard/](http://localhost:8080/dashboard/)  |
+| Metrics (API)     | [http://localhost:8080/prometheus]([http://localhost:8080/prometheus) |
+| Metrics (Manager) | [http://localhost:8081/prometheus]([http://localhost:8081/prometheus) |
+| Jaeger            | [http://localhost:8082](http://localhost:8082)                        |
+
+Additionally, the Redis instance can be accessed at `localhost:6379`.
+
+## Update the Gemfile Lock
+
+To update the `Gemfile.lock` run `./update-gemfile-lock`.
+
+Verify, and update if needed, that the docker tag in the script and GitHub action workflows matches what is used in the [vmpooler-deployment Dockerfile](https://github.com/puppetlabs/vmpooler-deployment/blob/main/docker/Dockerfile).
+
+## Releasing
+
+Follow these steps to publish a new GitHub release, and build and push the gem to <https://rubygems.org>.
+
+1. Bump the "VERSION" in `lib/vmpooler/version.rb` appropriately based on changes in `CHANGELOG.md` since the last release.
+2. Run `./release-prep` to update `Gemfile.lock` and `CHANGELOG.md`.
+3. Commit and push changes to a new branch, then open a pull request against `main` and be sure to add the "maintenance" label.
+4. After the pull request is approved and merged, then navigate to Actions --> Release Gem --> run workflow --> Branch: main --> Run workflow.
 
 ## License
 
-vmpooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.
+VMPooler is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).  See the [LICENSE](LICENSE) file for more details.
diff --git a/Vagrantfile b/Vagrantfile
index aa667a6..38598e4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -2,6 +2,8 @@
 Vagrant.configure("2") do |config|
   config.vm.box = "genebean/centos-7-rvm-multi"
   config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose
+  config.vm.network "forwarded_port", guest: 6379, host: 6379 # Redis
+  config.vm.network "forwarded_port", guest: 8079, host: 8079 # Redis Commander
   config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose
   config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose
   config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose
diff --git a/docker/Dockerfile b/docker/Dockerfile
deleted file mode 100644
index 9f1d15b..0000000
--- a/docker/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2-jdk
-
-ARG vmpooler_version=0.11.3
-
-COPY docker/docker-entrypoint.sh /usr/local/bin/
-
-ENV LOGFILE=/dev/stdout \
-      RACK_ENV=production
-
-RUN apt-get update -qq && \
-    apt-get install -y --no-install-recommends make && \
-    apt-get clean autoclean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN gem install vmpooler -v ${vmpooler_version} && \
-      chmod +x /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/docker/Dockerfile-aio b/docker/Dockerfile-aio
deleted file mode 100644
index 48a62d6..0000000
--- a/docker/Dockerfile-aio
+++ /dev/null
@@ -1,39 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2.9-jdk
-
-RUN mkdir -p /var/lib/vmpooler
-
-WORKDIR /var/lib/vmpooler
-
-RUN echo "deb http://httpredir.debian.org/debian jessie main" >/etc/apt/sources.list.d/jessie-main.list
-
-RUN apt-get update -qq && \
-    apt-get install -y --no-install-recommends make redis-server && \
-    apt-get clean autoclean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/*
-
-ADD Gemfile* /var/lib/vmpooler/
-
-RUN bundle install --system
-
-RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby
-
-COPY . /var/lib/vmpooler
-
-ENV VMPOOLER_LOG /var/log/vmpooler.log
-
-CMD \
-    /etc/init.d/redis-server start \
-    && /var/lib/vmpooler/scripts/vmpooler_init.sh start \
-    && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \
-    tail -f ${VMPOOLER_LOG}
diff --git a/docker/Dockerfile_local b/docker/Dockerfile_local
deleted file mode 100644
index c1b3f4e..0000000
--- a/docker/Dockerfile_local
+++ /dev/null
@@ -1,30 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.2-jdk
-
-COPY docker/docker-entrypoint.sh /usr/local/bin/
-COPY ./ ./
-
-ENV RACK_ENV=production
-
-RUN apt-get update -qq && \
-    apt-get install -y --no-install-recommends make && \
-    apt-get clean autoclean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN gem install bundler && \
-    bundle install && \
-    gem build vmpooler.gemspec && \
-    gem install vmpooler*.gem && \
-    chmod +x /usr/local/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
deleted file mode 100644
index 21b179d..0000000
--- a/docker/docker-compose.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-# For local development run with a dummy provider
-version: '3.8'
-services:
-  vmpooler-api:
-    build:
-      context: ../
-      dockerfile: docker/Dockerfile_local
-    volumes:
-      - type: bind
-        source: ${PWD}/vmpooler.yaml
-        target: /etc/vmpooler/vmpooler.yaml
-    ports:
-      - "8080:4567"
-    networks:
-      - redis-net
-    environment:
-      - VMPOOLER_DEBUG=true # for use of dummy auth
-      - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
-      - REDIS_SERVER=redislocal
-      - LOGFILE=/dev/null
-      - JRUBY_OPTS=-Xinvokedynamic.yield=false
-      - VMPOOLER_TRACING_ENABLED=true
-      - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces
-    image: vmpooler-local
-    command: api
-    depends_on:
-      - redislocal
-  vmpooler-manager:
-    build:
-      context: ../
-      dockerfile: docker/Dockerfile_local
-    volumes:
-      - type: bind
-        source: ${PWD}/vmpooler.yaml
-        target: /etc/vmpooler/vmpooler.yaml
-    ports:
-      - "8081:4567"
-    networks:
-      - redis-net
-    environment:
-      - VMPOOLER_DEBUG=true # for use of dummy auth
-      - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
-      - REDIS_SERVER=redislocal
-      - LOGFILE=/dev/null
-      - JRUBY_OPTS=-Xinvokedynamic.yield=false
-      - VMPOOLER_TRACING_ENABLED=true
-      - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces
-    image: vmpooler-local
-    command: manager
-    depends_on:
-      - redislocal
-  redislocal:
-    image: redis
-    # Uncomment this if you don't want the redis data to persist
-    #command: "redis-server --save '' --appendonly no"
-    ports:
-      - "6379:6379"
-    networks:
-      - redis-net
-  jaeger-aio:
-    image: jaegertracing/all-in-one:1.18
-    ports:
-      - "14250:14250"
-      - "8082:16686"
-    networks:
-      - redis-net
-    user: '1001'
-    read_only: true
-    cap_drop:
-      - ALL
-
-networks:
-  redis-net:
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
deleted file mode 100644
index 5da89df..0000000
--- a/docker/docker-entrypoint.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-set -e
-
-set -- vmpooler "$@"
-
-exec "$@"
diff --git a/docs/API.md b/docs/API-v3.md
similarity index 79%
rename from docs/API.md
rename to docs/API-v3.md
index 038d4e9..ec4feaf 100644
--- a/docs/API.md
+++ b/docs/API-v3.md
@@ -12,6 +12,30 @@
 
 vmpooler provides a REST API for VM management.  The following examples use `curl` for communication.
 
+## Major change in V3 versus V2
+
+The api/v1 and api/v2 endpoints have been removed. Additionally, the generic api endpoint that reroutes to a versioned endpoint has been removed.
+
+The api/v3 endpoint removes the deprecated "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each pool instead of only a single domain for all pools, or a domain restricted to a particular provider.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will now now need to specify the versioned endpoint (api/v3).
+
+## Major change in V2 versus V1
+
+The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each provider instead of only one.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will still be redirected to v1, but this behavior is notw deprecated and will be changed to v2 in the next major version.
+
+### updavig clients from v1 to v2
+
+Clients need to update their paths to using api/v2 instead of api/v1. Please note the API responses that used to return a "domain" key, will no longer have a separate "domain" key and now return
+the FQDN (includes the domain in the hostname).
+
+One way to support both v1 and v2 in the client logic is to look for a "domain" and append it to the hostname if it exists (existing v1 behavior). If the "domain" key does not exist, you can use the hostname 
+as is since it is a FQDN (v2 behavor).
+
 #### Token operations <a name="token"></a>
 
 Token-based authentication can be used when requesting or modifying VMs.  The `/token` route can be used to create, query, or delete tokens.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations.
@@ -26,7 +50,7 @@ Return codes:
 * 404  when config:auth not found or other error
 
 ```
-$ curl -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -48,7 +72,7 @@ Return codes:
 * 404  when config:auth not found
 
 ```
-$ curl -X POST -u jdoe --url vmpooler.example.com/api/v1/token
+$ curl -X POST -u jdoe --url vmpooler.example.com/api/v3/token
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -67,7 +91,7 @@ Return codes:
 * 404  when config:auth or token not found
 
 ```
-$ curl --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 ```
 ```json
 {
@@ -96,7 +120,7 @@ Return codes:
 * 404  when config:auth not found
 
 ```
-$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
 Enter host password for user 'jdoe':
 ```
 ```json
@@ -115,7 +139,7 @@ Return codes:
 * 200  OK
 
 ```
-$ curl --url vmpooler.example.com/api/v1/vm
+$ curl --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 [
@@ -136,21 +160,20 @@ Return codes:
 * 503  when the vm failed to allocate a vm, or the pool is empty
 
 ```
-$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v1/vm
+$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v3/vm
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "o41xtodlvnvu5cw",
-      "khirruvwfjlmx3y"
+      "o41xtodlvnvu5cw.example.com",
+      "khirruvwfjlmx3y.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "y91qbrpbfj6d13q"
-  },
-  "domain": "example.com"
+    "hostname": "y91qbrpbfj6d13q.example.com"
+  }
 }
 ```
 
@@ -166,22 +189,21 @@ Return codes:
 * 503  when the vm failed to allocate a vm, or the pool is empty
 
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386
 ```
 ```json
 {
   "ok": true,
   "debian-7-i386": {
-    "hostname": "fq6qlpjlsskycq6"
-  },
-  "domain": "example.com"
+    "hostname": "fq6qlpjlsskycq6.example.com"
+  }
 }
 ```
 
 Multiple VMs can be requested by using multiple query parameters in the URL:
 
 ```
-$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
+$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
 ```
 
 ```json
@@ -189,14 +211,13 @@ $ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debia
   "ok": true,
   "debian-7-i386": {
     "hostname": [
-      "sc0o4xqtodlul5w",
-      "4m4dkhqiufnjmxy"
+      "sc0o4xqtodlul5w.example.com",
+      "4m4dkhqiufnjmxy.example.com"
     ]
   },
   "debian-7-x86_64": {
-    "hostname": "zb91y9qbrbf6d3q"
-  },
-  "domain": "example.com"
+    "hostname": "zb91y9qbrbf6d3q.example.com"
+  }
 }
 ```
 
@@ -211,7 +232,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ```
-$ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
+$ curl --url vmpooler.example.com/api/v3/vm/pxpmtoonx7fiqg6
 ```
 ```json
 {
@@ -227,7 +248,6 @@ $ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6
       "user": "jdoe"
     },
     "ip": "192.168.0.1",
-    "domain": "example.com",
     "host": "host1.example.com",
     "migrated": "true"
   }
@@ -256,7 +276,7 @@ Return codes:
 * 400  when supplied PUT parameters fail validation
 
 ```
-$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -265,7 +285,7 @@ $ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpj
 ```
 
 ```
-$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -283,7 +303,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ```
-$ curl -X DELETE --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl -X DELETE --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ```
 ```json
 {
@@ -303,7 +323,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname or size is not an integer
 
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/disk/8
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/disk/8
 ````
 ````json
 {
@@ -317,7 +337,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -329,8 +349,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "disk": [
       "+8gb"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 
@@ -348,7 +367,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname
 
 ````
-$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot
 ````
 ````json
 {
@@ -362,7 +381,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e
 Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/<hostname>` query:
 
 ````
-$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
+$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6
 ````
 ````json
 {
@@ -374,8 +393,7 @@ $ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6
     "state": "running",
     "snapshots": [
       "n4eb4kdtp7rwv4x158366vd9jhac8btq"
-    ],
-    "domain": "delivery.puppetlabs.net"
+    ]
   }
 }
 ````
@@ -390,7 +408,7 @@ Return codes:
 * 404  when requesting an invalid VM hostname or snapshot is not valid
 
 ````
-$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
+$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
 ````
 ````json
 {
@@ -405,7 +423,7 @@ $ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.ex
 A "live" status endpoint, representing the current state of the service.
 
 ```
-$ curl --url vmpooler.example.com/api/v1/status
+$ curl --url vmpooler.example.com/api/v3/status
 ```
 ```json
 {
@@ -457,7 +475,7 @@ The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "s
 If the query parameter 'view' is provided, it will be used to select which top level
 element to compute and return. Select them by specifying which one you want in a comma 
 separated list.
-For example `vmpooler.example.com/api/v1/status?view=capacity,boot`
+For example `vmpooler.example.com/api/v3/status?view=capacity,boot`
 
 ##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]]
 
@@ -474,7 +492,7 @@ Return codes:
 
 
 ```
-$ curl --url vmpooler.example.com/api/v1/summary
+$ curl --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -566,7 +584,7 @@ $ curl --url vmpooler.example.com/api/v1/summary
 
 
 ```
-$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v1/summary
+$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v3/summary
 ```
 ```json
 {
@@ -630,9 +648,9 @@ $ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api
 ```
 
 You can also query only the specific top level section you want by including it after `summary/`.
-The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v1/summary/boot/`.
+The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v3/summary/boot/`.
 You can further drill-down the data by specifying the second level parameter to query eg
-`vmpooler.example.com/api/v1/summary/tag/created_by`
+`vmpooler.example.com/api/v3/summary/tag/created_by`
 
 ##### GET /poolstat?pool=FOO
 
@@ -644,7 +662,7 @@ Return codes
 * 200  OK 
 
 ```
-$ curl https://vmpooler.example.com/api/v1/poolstat?pool=centos-6-x86_64
+$ curl https://vmpooler.example.com/api/v3/poolstat?pool=centos-6-x86_64
 ```
 ```json
 {
@@ -669,7 +687,7 @@ Return codes
 * 200  OK 
 
 ```
-$ curl https://vmpooler.example.com/api/v1/totalrunning
+$ curl https://vmpooler.example.com/api/v3/totalrunning
 ```
 
 ```json
@@ -691,7 +709,7 @@ Return codes
 * 400  No configuration found  
 
 ```
-$ curl https://vmpooler.example.com/api/v1/config
+$ curl https://vmpooler.example.com/api/v3/config
 ```
 ```json
 {
@@ -735,7 +753,7 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v1/config/poolsize
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v3/config/poolsize
 ```
 ```json
 {
@@ -743,6 +761,28 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","deb
 }
 ```
 
+##### DELETE /config/poolsize/&lt;pool&gt;
+
+Delete an overridden pool size. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - size reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/poolsize/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "pool_size_before_overrides": 2,
+    "pool_size_before_reset": 4
+}
+```
+
 ##### POST /config/pooltemplate
 
 Change the template configured for a pool, and replenish the pool with instances built from the new template.
@@ -767,7 +807,7 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v1/config/pooltemplate
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v3/config/pooltemplate
 ```
 ```json
 {
@@ -775,6 +815,28 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templat
 }
 ```
 
+##### DELETE /config/pooltemplate/&lt;pool&gt;
+
+Delete an overridden pool template. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - template reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/pooltemplate/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2",
+    "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta"
+}
+```
+
 ##### POST /poolreset
 
 Clear all pending and ready instances in a pool, and deploy replacements
@@ -793,7 +855,7 @@ Responses:
 * 404 - An unknown error occurred
 * 405 - The endpoint is disabled because experimental features are disabled
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v1/poolreset
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v3/poolreset
 ```
 ```json
 {
@@ -821,7 +883,7 @@ Responses:
 * 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred.
 * 409 - A request of the matching ID has already been created
 ```
-$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v1/ondemandvm
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v3/ondemandvm
 ```
 ```json
 {
@@ -843,7 +905,7 @@ Responses:
 * 202 - The request is not ready yet
 * 404 - The request can not be found, or an unknown error occurred
 ```
-$ curl https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+$ curl https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
 ```
 ```json
 {
@@ -883,7 +945,7 @@ Responses:
 * 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
 * 404 - The request can not be found, or an unknown error occurred.
 ```
-$ curl -X DELETE https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+$ curl -X DELETE https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
 ```
 ```json
 {
diff --git a/docs/PROVIDER_API.md b/docs/PROVIDER_API.md
new file mode 100644
index 0000000..fb0bcdb
--- /dev/null
+++ b/docs/PROVIDER_API.md
@@ -0,0 +1,100 @@
+# Provider API
+
+Providers facilitate VMPooler interacting with some other system that can create virtual machines. A single VMPooler instance can utilize one or more providers and can have multiple instances of the same provider. An example of having multiple instances of the same provider is when you need to interact with multiple vCenters from the same VMPooler instance.
+
+## Known Providers
+
+- `vmpooler-provider-vsphere` provides the ability to use VMware as a source of VMs. Its code can be found in the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository.
+
+Know of others? Please submit a pull request to update this list or reach out to us on the Puppet community Slack.
+
+Want to create a new one? See below!
+
+## Create a new provider gem from scratch
+
+### Requirements
+
+1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
+2. the main provider code file should be named the same at the name of the provider. For example, the `vpshere` provider's main file is `lib/vmpooler/providers/vsphere.rb`.
+3. The gem must be installed on the same machine as VMPooler
+4. The provider name must be referenced in the VMPooler config file in order for it to be loaded.
+5. Your gem name and repository name should be `vmpooler-provider-<provider name>` so the community can easily search provider plugins.
+
+The resulting directory structure should resemble this:
+
+```bash
+lib/
+├── vmpooler/
+│   └── providers/
+│       └── <provider name>.rb
+└── vmpooler-provider-<provider name>/
+    └── version.rb
+```
+
+### 1. Use bundler to create the provider scaffolding
+
+```bash
+bundler gem --test=rspec --no-exe --no-ext vmpooler-provider-spoof
+cd vmpooler-providers-spoof/
+mkdir -p ./lib/vmpooler/providers
+touch ./lib/vmpooler/providers/spoof.rb
+mkdir ./lib/vmpooler-providers-spoof
+touch ./lib/vmpooler-providers-spoof/version.rb
+```
+
+There may be some boilerplate files generated, just delete those.
+
+### 2. Create the main provider file
+
+Ensure the main provider file uses the following code.
+
+```ruby
+# lib/vmpooler/providers/spoof.rb
+require 'yaml'
+require 'vmpooler/providers/base'
+
+module Vmpooler
+  class PoolManager
+    class Provider
+      class Spoof < Vmpooler::PoolManager::Provider::Base
+        # At this time it is not documented which methods should be implemented
+        # have a look at the https://github.com/puppetlabs/vmpooler-provider-vsphere
+        #for an example
+      end
+    end
+  end
+end
+```
+
+### 3. Create the version file
+
+Ensure you have a version file similar this:
+
+```ruby
+# frozen_string_literal: true
+# lib/vmpooler-provider-vsphere/version.rb 
+
+module VmpoolerProviderSpoof
+  VERSION = '1.0.0'
+end
+```
+
+### 4. Fill out your gemspec
+
+Ensure you fill out your gemspec file to your specifications.  If you need a dependency, please make sure you require it.
+
+`spec.add_dependency "foo", "~> 1.15"`.
+
+At a minimum you may want to add the `vmpooler` gem as a dev dependency so you can use it during testing.
+
+`spec.add_dev_dependency "vmpooler", "~> 2.0"`
+
+Also make sure this dependency can be loaded by JRuby.  If the dependency cannot be used by JRuby don't use it.
+
+### 5. Create some tests
+
+Your provider code should be tested before releasing. Copy and refactor some tests from the `vmpooler` gem under `spec/unit/providers/dummy_spec.rb`.
+
+### 6. Publish
+
+Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index de378a1..0000000
--- a/docs/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Documentation for vmpooler
-
-
-* [Setting up a Development Environment](dev-setup.md)
-* [API Documentation](API.md)
diff --git a/docs/configuration.md b/docs/configuration.md
index d49debb..560c328 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -19,11 +19,6 @@ Provide the entire configuration as a blob of yaml. Individual parameters passed
 
 Path to a the file to use when loading the vmpooler configuration. This is only evaluated if `VMPOOLER_CONFIG` has not been specified.
 
-### DOMAIN
-
-If set, returns a top-level 'domain' JSON key in POST requests
-(optional)
-
 ### REDIS\_SERVER
 
 The redis server to use for vmpooler.
@@ -170,9 +165,12 @@ This can also be set per pool.
 
 ### PURGE\_UNCONFIGURED\_FOLDERS
 
-Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+Deprecated, see PURGE\_UNCONFIGURED\_RESOURCES
+
+### PURGE\_UNCONFIGURED\_RESOURCES
+
+Enable purging of VMs (and other resources) detected within the provider that are not explicitly configured.
+Implementation is provider-dependent
 When enabled in the global configuration then purging is enabled for all providers
 Expects a boolean value
 (optional; default: false)
@@ -248,6 +246,18 @@ This can be a string providing a single DN. For multiple DNs please specify the
 The LDAP object-type used to designate a user object.
 (optional)
 
+### LDAP\_SERVICE_ACCOUNT\_HASH
+
+A hash containing the following parameters for a service account to perform the
+initial bind. After the initial bind, then a search query is performed using the
+'base' and 'user_object', then re-binds as the returned user.
+
+- :user_dn: The full distinguished name (DN) of the service account used to bind.
+
+- :password: The password for the service account used to bind.
+
+(optional)
+
 ### SITE\_NAME
 
 The name of your deployment.
diff --git a/docs/dev-setup.md b/docs/dev-setup.md
deleted file mode 100644
index 5999f09..0000000
--- a/docs/dev-setup.md
+++ /dev/null
@@ -1,236 +0,0 @@
-# Setting up a vmpooler development environment
-
-## Docker is the preferred development environment
-
-The docker compose file is the easiest way to get vmpooler running with any local code changes. The docker compose file expects to find a vmpooler.yaml configuration file in the root vmpooler directory. The file is mapped into the running container for the vmpooler application. This file primarily contains the pools configuration. Nearly all other configuration can be supplied with environment variables.
-
-## Requirements for local installation directly on your system (not recommended)
-
-* Supported on OSX, Windows and Linux
-
-* Ruby or JRuby
-
-  Note - It is recommended to user Bundler instead of installing gems into the system repository
-
-* A local Redis server
-
-  Either a containerized instance of Redis or a local version is fine.
-
-## Setup source and ruby
-
-* Clone repository, either from your own fork or the original source
-
-* Perform a bundle install
-
-```
-~/ > git clone https://github.com/puppetlabs/vmpooler.git
-Cloning into 'vmpooler'...
-remote: Counting objects: 3411, done.
-...
-
-~/ > cd vmpooler
-
-~/vmpooler/ > bundle install
-Fetching gem metadata from https://rubygems.org/.........
-Fetching version metadata from https://rubygems.org/..
-Resolving dependencies...
-Installing rake 12.0.0
-...
-Bundle complete! 16 Gemfile dependencies, 37 gems now installed.
-```
-
-## Setup environment variables
-
-### `VMPOOLER_DEBUG`
-
-Setting the `VMPOOLER_DEBUG` environment variable will instruct vmpooler to:
-
-* Output log messages to STDOUT
-
-* Allow the use of the dummy authentication method
-
-* Add interrupt traps so you can stop vmpooler when run interactively
-
-Linux, OSX
-```bash
-~/vmpooler/ > export VMPOOLER_DEBUG=true
-```
-
-Windows (PowerShell)
-```powershell
-C:\vmpooler > $ENV:VMPOOLER_DEBUG = 'true'
-```
-
-
-### `VMPOOLER_CONFIG`
-
-When `VMPOOLER_CONFIG` is set, vmpooler will read its configuration from the content of the environment variable.
-
-Note that this variable does not point to a different configuration file, but stores the contents of a configuration file. You may use `VMPOOLER_CONFIG_FILE` instead to specify a filename.
-
-
-### `VMPOOLER_CONFIG_FILE`
-
-When `VMPOOLER_CONFIG_FILE` is set, vmpooler will read its configuration from the file specified in the environment variable.
-
-Note that this variable points to a different configuration file, unlike `VMPOOLER_CONFIG`.
-
-
-## Setup vmpooler Configuration
-
-You can create a `vmpooler.yaml` file, set the `VMPOOLER_CONFIG` environment variable with the equivalent content, or set the `VMPOOLER_CONFIG_FILE` environment variable with the name of another configuration file to use. `VMPOOLER_CONFIG` takes precedence over `VMPOOLER_CONFIG_FILE`.
-
-Example minimal configuration file:
-```yaml
-
----
-:providers:
-  :dummy:
-
-:redis:
-  server: 'localhost'
-
-:auth:
-   provider: dummy
-
-:tagfilter:
-  url: '(.*)\/'
-
-:config:
-  site_name: 'vmpooler'
-  # Need to change this on Windows
-  logfile: '/var/log/vmpooler.log'
-  task_limit: 10
-  timeout: 15
-  vm_lifetime: 12
-  vm_lifetime_auth: 24
-  allowed_tags:
-    - 'created_by'
-    - 'project'
-  domain: 'example.com'
-  prefix: 'poolvm-'
-
-# Uncomment the lines below to suppress metrics to STDOUT
-# :statsd:
-#   server: 'localhost'
-#   prefix: 'vmpooler'
-#   port: 8125
-
-:pools:
-  - name: 'pool01'
-    size: 5
-    provider: dummy
-  - name: 'pool02'
-    size: 5
-    provider: dummy
-```
-
-## Running vmpooler locally
-
-* Run `bundle exec ruby vmpooler`
-
-  If using JRuby, you may need to use `bundle exec jruby vmpooler`
-
-You should see output similar to:
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-[2017-06-16 14:50:31] starting vmpooler
-[2017-06-16 14:50:31] [!] Creating provider 'dummy'
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection pool of size 1 with timeout 10
-[2017-06-16 14:50:31] [*] [disk_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [snapshot_manager] starting worker thread
-[2017-06-16 14:50:31] [*] [pool01] starting worker thread
-[2017-06-16 14:50:31] [*] [pool02] starting worker thread
-[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection object ID 1784
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-[2017-06-16 14:50:31] [!] [pool02] is empty
-[2017-06-16 14:50:31] [!] [pool01] is empty
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nexs1w50m4djap5'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-r543eibo4b6tjer'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-neqmu7wj7aukyjy'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nsdnrhhy22lnemo'
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-r543eibo4b6tjer' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-neqmu7wj7aukyjy' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool02] 'poolvm-nexs1w50m4djap5' is being cloned from ''
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-edzlp954lyiozli'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nb0uci0yrwbxr6x'
-[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-y2yxgnovaneymvy'
-[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nur59d25s1y8jko'
-...
-```
-
-### Common Errors
-
-* Forget to set VMPOOLER_DEBUG environment variable
-
-vmpooler will fail to start with an error similar to below
-```
-~/vmpooler/ > bundle exec ruby vmpooler
-
-~/vmpooler/lib/vmpooler.rb:44:in `config': Dummy authentication should not be used outside of debug mode; please set environment variable VMPOOLER_DEBUG to 'true' if you want to use dummy authentication (RuntimeError)
-        from vmpooler:8:in `<main>'
-...
-```
-
-* Error in vmpooler configuration
-
-If there is an error in the vmpooler configuration file, or any other fatal error in the Pool Manager, vmpooler will appear to be running but no log information is displayed.  This is due to the error not being displayed until you press `Ctrl-C` and then suddenly you can see the cause of the issue.
-
-For example, when running vmpooler on Windows, but with a unix style filename for the vmpooler log
-
-```powershell
-C:\vmpooler > bundle exec ruby vmpooler
-[2017-06-16 14:49:57] starting vmpooler
-== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma
-*** SIGUSR2 not implemented, signal based restart unavailable!
-*** SIGUSR1 not implemented, signal based restart unavailable!
-*** SIGHUP not implemented, signal based logs reopening unavailable!
-Puma starting in single mode...
-* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller
-* Min threads: 0, max threads: 16
-* Environment: development
-* Listening on tcp://0.0.0.0:4567
-Use Ctrl-C to stop
-
-# [ NOTHING ELSE IS LOGGED ]
-```
-
-Once `Ctrl-C` is pressed the error is shown
-
-```powershell
-...
-== Sinatra has ended his set (crowd applauds)
-Shutting down.
-C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `initialize': No such file or directory @ rb_sysopen - /var/log/vmpooler.log (Errno::ENOENT)
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open'
-        from C:/vmpooler/lib/vmpooler/logger.rb:17:in `log'
-        from C:/vmpooler/lib/vmpooler/pool_manager.rb:709:in `execute!'
-        from vmpooler:26:in `block in <main>'
-```
-
-## Default vmpooler URLs
-
-| Endpoint  | URL                                                                  |
-|-----------|----------------------------------------------------------------------|
-| Dashboard | [http://localhost:4567/dashboard/](http://localhost:4567/dashboard/) |
-| API       | [http://localhost:4567/api/v1]([http://localhost:4567/api/v1)        |
-
-## Use the vmpooler API locally
-
-Once a local vmpooler instance is running you can use any tool you need to interact with the API.  The dummy authentication provider will allow a user to connect if the username and password are not the same:
-
-* Authentication is successful for username `Alice` with password `foo`
-
-* Authentication will fail for username `Alice` with password `Alice`
-
-Like normal vmpooler, tokens will be created for the user and can be used for regular vmpooler operations.
diff --git a/docs/vagrant.md b/docs/vagrant.md
deleted file mode 100644
index 58332b9..0000000
--- a/docs/vagrant.md
+++ /dev/null
@@ -1,45 +0,0 @@
-A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer.
-To use it run:
-
-```
-vagrant up
-vagrant ssh
-docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler
-```
-
-To run vmpooler with the example dummy provider you can replace the above docker command with this:
-
-```
-docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler
-```
-
-Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/).
-
-### Running directly in Vagrant
-
-You can also run vmpooler directly in the Vagrant box. To do so run this:
-
-```
-vagrant up
-vagrant ssh
-cd /vagrant
-
-# Do this if using the dummy provider
-export VMPOOLER_DEBUG=true
-cp vmpooler.yaml.dummy-example vmpooler.yaml
-
-# vmpooler needs a redis server.
-sudo yum -y install redis
-sudo systemctl start redis
-
-# Optional: Choose your ruby version or use jruby
-# ruby 2.4.x is used by default
-rvm list
-rvm use jruby-9.1.7.0
-
-gem install bundler
-bundle install
-bundle exec ruby vmpooler
-```
-
-When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/).
diff --git a/examples/vmpooler.yaml.dummy-example.aliasedpools b/examples/vmpooler.yaml.dummy-example.aliasedpools
index efe0ce2..55bf9ff 100644
--- a/examples/vmpooler.yaml.dummy-example.aliasedpools
+++ b/examples/vmpooler.yaml.dummy-example.aliasedpools
@@ -17,15 +17,20 @@
   logfile: '/Users/samuel/workspace/vmpooler/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 :pools:
   - name: 'debian-7-i386'
     alias: [ 'debian-7-32' ]
@@ -34,8 +39,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-stringalias'
     alias: 'debian-7-32-stringalias'
     template: 'Templates/debian-7-i386'
@@ -43,8 +50,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64'
     alias: [ 'debian-7-64', 'debian-7-amd64' ]
     template: 'Templates/debian-7-x86_64'
@@ -52,16 +61,20 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-i386-noalias'
     template: 'Templates/debian-7-i386'
     folder: 'Pooled VMs/debian-7-i386'
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64-alias-otherpool-extended'
     alias: [ 'debian-7-x86_64' ]
     template: 'Templates/debian-7-x86_64'
@@ -69,6 +82,7 @@
     datastore: 'other-vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
-
+    dns_plugin: 'example'
diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb
index bbc68f8..2fcde30 100644
--- a/lib/vmpooler.rb
+++ b/lib/vmpooler.rb
@@ -3,11 +3,11 @@
 module Vmpooler
   require 'concurrent'
   require 'date'
+  require 'deep_merge'
   require 'json'
   require 'net/ldap'
   require 'open-uri'
   require 'pickup'
-  require 'rbvmomi'
   require 'redis'
   require 'set'
   require 'sinatra/base'
@@ -16,8 +16,8 @@ module Vmpooler
   require 'yaml'
 
   # Dependencies for tracing
-  require 'opentelemetry-api'
   require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-http_client'
   require 'opentelemetry-instrumentation-redis'
   require 'opentelemetry-instrumentation-sinatra'
   require 'opentelemetry-sdk'
@@ -33,8 +33,8 @@ module Vmpooler
     if ENV['VMPOOLER_CONFIG']
       config_string = ENV['VMPOOLER_CONFIG']
       # Parse the YAML config into a Hash
-      # Whitelist the Symbol class
-      parsed_config = YAML.safe_load(config_string, [Symbol])
+      # Allow the Symbol class
+      parsed_config = YAML.safe_load(config_string, permitted_classes: [Symbol])
     else
       # Take the name of the config file either from an ENV variable or from the filepath argument
       config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
@@ -43,8 +43,9 @@ module Vmpooler
       if parsed_config[:config]['extra_config']
         extra_configs = parsed_config[:config]['extra_config'].split(',')
         extra_configs.each do |config|
+          puts "loading extra_config file #{config}"
           extra_config = YAML.load_file(config)
-          parsed_config.merge!(extra_config)
+          parsed_config.deep_merge(extra_config)
         end
       end
     end
@@ -53,17 +54,13 @@ module Vmpooler
 
     # Bail out if someone attempts to start vmpooler with dummy authentication
     # without enbaling debug mode.
-    if parsed_config.key? :auth
-      if parsed_config[:auth]['provider'] == 'dummy'
-        unless ENV['VMPOOLER_DEBUG']
-          warning = [
-            'Dummy authentication should not be used outside of debug mode',
-            'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
-          ]
+    if parsed_config.key?(:auth) && parsed_config[:auth]['provider'] == 'dummy' && !ENV['VMPOOLER_DEBUG']
+      warning = [
+        'Dummy authentication should not be used outside of debug mode',
+        'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
+      ]
 
-          raise warning.join(";\s")
-        end
-      end
+      raise warning.join(";\s")
     end
 
     # Set some configuration defaults
@@ -79,18 +76,25 @@ module Vmpooler
     parsed_config[:config]['prefix']                             = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
     parsed_config[:config]['logfile']                            = ENV['LOGFILE'] if ENV['LOGFILE']
     parsed_config[:config]['site_name']                          = ENV['SITE_NAME'] if ENV['SITE_NAME']
-    parsed_config[:config]['domain']                             = ENV['DOMAIN'] if ENV['DOMAIN']
+    if !parsed_config[:config]['domain'].nil? || !ENV['DOMAIN'].nil?
+      puts '[!] [error] The "domain" config setting has been removed in v3. Please see the docs for migrating the domain config to use a dns plugin at https://github.com/puppetlabs/vmpooler/blob/main/README.md#migrating-to-v3'
+      exit 1
+    end
     parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
     parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['timeout_notification']               = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
     parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
     parsed_config[:config]['create_folders']                     = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
     parsed_config[:config]['experimental_features']              = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
-    parsed_config[:config]['purge_unconfigured_folders']         = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
     parsed_config[:config]['usage_stats']                        = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
     parsed_config[:config]['request_logger']                     = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
     parsed_config[:config]['create_template_delta_disks']        = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_RESOURCES'] if ENV['PURGE_UNCONFIGURED_RESOURCES']
+    parsed_config[:config]['purge_unconfigured_resources']       = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    # ENV PURGE_UNCONFIGURED_FOLDERS deprecated, will be removed in version 3
+    puts '[!] [deprecation] rename ENV var \'PURGE_UNCONFIGURED_FOLDERS\' to \'PURGE_UNCONFIGURED_RESOURCES\'' if ENV['PURGE_UNCONFIGURED_FOLDERS']
     set_linked_clone(parsed_config)
 
     parsed_config[:redis]                            = parsed_config[:redis] || {}
@@ -100,7 +104,7 @@ module Vmpooler
     parsed_config[:redis]['data_ttl']                = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
     parsed_config[:redis]['connection_pool_size']    = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
     parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
-    parsed_config[:redis]['reconnect_attempts']      = string_to_int(ENV['REDIS_RECONNECT_ATTEMPTS']) || parsed_config[:redis]['reconnect_attempts'] || 10
+    parsed_config[:redis]['reconnect_attempts']      = string_array_to_array(ENV['REDIS_RECONNECT_ATTEMPTS']) || parsed_config[:redis]['reconnect_attempts'] || 10
 
     parsed_config[:statsd]           = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
@@ -124,30 +128,45 @@ module Vmpooler
       parsed_config[:auth][:ldap]['port']        = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
       parsed_config[:auth][:ldap]['base']        = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
+      if parsed_config[:auth]['provider'] == 'ldap' && parsed_config[:auth][:ldap].key?('encryption')
+        parsed_config[:auth][:ldap]['encryption'] = parsed_config[:auth][:ldap]['encryption']
+      elsif parsed_config[:auth]['provider'] == 'ldap'
+        parsed_config[:auth][:ldap]['encryption'] = {}
+      end
     end
 
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
     unless parsed_config[:pools]
+      puts 'loading pools configuration from redis, since the config[:pools] is empty'
       redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
       parsed_config[:pools] = load_pools_from_redis(redis)
     end
 
+    # Marshal.dump is paired with Marshal.load to create a copy that has its own memory space
+    # so that each can be edited independently
+    # rubocop:disable Security/MarshalLoad
+
+    # retain a copy of the pools that were observed at startup
+    serialized_pools = Marshal.dump(parsed_config[:pools])
+    parsed_config[:pools_at_startup] = Marshal.load(serialized_pools)
+
     # Create an index of pools by title
     parsed_config[:pool_index] = pool_index(parsed_config[:pools])
+    # rubocop:enable Security/MarshalLoad
 
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
       pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
-        if pool['alias'].is_a?(Array)
+        if pool['alias'].instance_of?(Array)
           pool['alias'].each do |pool_alias|
             parsed_config[:alias] ||= {}
             parsed_config[:alias][pool_alias] = [pool['name']] unless parsed_config[:alias].key? pool_alias
             parsed_config[:alias][pool_alias] << pool['name'] unless parsed_config[:alias][pool_alias].include? pool['name']
             parsed_config[:pool_names] << pool_alias
           end
-        elsif pool['alias'].is_a?(String)
+        elsif pool['alias'].instance_of?(String)
           parsed_config[:alias][pool['alias']] = pool['name']
           parsed_config[:pool_names] << pool['alias']
         end
@@ -191,8 +210,13 @@ module Vmpooler
   end
 
   def self.new_redis(host = 'localhost', port = nil, password = nil, redis_reconnect_attempts = 10)
-    Redis.new(host: host, port: port, password: password, reconnect_attempts: redis_reconnect_attempts, reconnect_delay: 1.5,
-              reconnect_delay_max: 10.0)
+    Redis.new(
+      host: host,
+      port: port,
+      password: password,
+      reconnect_attempts: redis_reconnect_attempts,
+      connect_timeout: 300
+    )
   end
 
   def self.pools(conf)
@@ -217,6 +241,13 @@ module Vmpooler
     Integer(s)
   end
 
+  def self.string_array_to_array(s)
+    # Returns an array from an array like string
+    return if s.nil?
+
+    JSON.parse(s)
+  end
+
   def self.true?(obj)
     obj.to_s.downcase == 'true'
   end
@@ -241,18 +272,19 @@ module Vmpooler
     if tracing_enabled.eql?('false')
       puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'"
       span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
-        exporter: OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
+        OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
       )
     else
       puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}"
       span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
-        exporter: OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
+        OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
       )
     end
 
     OpenTelemetry::SDK.configure do |c|
       c.use 'OpenTelemetry::Instrumentation::Sinatra'
       c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::HttpClient'
       c.use 'OpenTelemetry::Instrumentation::Redis'
 
       c.add_span_processor(span_processor)
diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb
index eb856fc..3b0d9de 100644
--- a/lib/vmpooler/api.rb
+++ b/lib/vmpooler/api.rb
@@ -3,7 +3,7 @@
 module Vmpooler
   class API < Sinatra::Base
     # Load API components
-    %w[helpers dashboard reroute v1 request_logger healthcheck].each do |lib|
+    %w[helpers dashboard v3 request_logger healthcheck].each do |lib|
       require "vmpooler/api/#{lib}"
     end
     # Load dashboard components
@@ -52,8 +52,7 @@ module Vmpooler
 
         use Vmpooler::Dashboard
         use Vmpooler::API::Dashboard
-        use Vmpooler::API::Reroute
-        use Vmpooler::API::V1
+        use Vmpooler::API::V3
       end
 
       # Get thee started O WebServer
diff --git a/lib/vmpooler/api/dashboard.rb b/lib/vmpooler/api/dashboard.rb
index 4c56798..f5e65cf 100644
--- a/lib/vmpooler/api/dashboard.rb
+++ b/lib/vmpooler/api/dashboard.rb
@@ -128,34 +128,32 @@ module Vmpooler
 
         pools.each do |pool|
           running = running_hash[pool['name']]
-          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
           result[pool['major']] ||= {}
           result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
         end
 
-        if params[:history]
-          if graph_url
-            begin
-              buffer = URI.parse(graph_link('.running.*&from=-1hour&format=json')).read
-              JSON.parse(buffer).each do |pool|
-                if pool['target'] =~ /.*\.(.*)$/
-                  pool['name'] = Regexp.last_match[1]
-                  pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
-                  result[pool['major']]['history'] ||= []
+        if params[:history] && graph_url
+          begin
+            buffer = URI.parse(graph_link('.running.*&from=-1hour&format=json')).read
+            JSON.parse(buffer).each do |pool|
+              if pool['target'] =~ /.*\.(.*)$/
+                pool['name'] = Regexp.last_match[1]
+                pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)-/
+                result[pool['major']]['history'] ||= []
 
-                  for i in 0..pool['datapoints'].length
-                    if pool['datapoints'][i] && pool['datapoints'][i][0]
-                      pool['last'] = pool['datapoints'][i][0]
-                      result[pool['major']]['history'][i] ||= 0
-                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
-                    else
-                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
-                    end
+                for i in 0..pool['datapoints'].length
+                  if pool['datapoints'][i] && pool['datapoints'][i][0]
+                    pool['last'] = pool['datapoints'][i][0]
+                    result[pool['major']]['history'][i] ||= 0
+                    result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
+                  else
+                    result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
                   end
                 end
               end
-            rescue StandardError
             end
+          rescue StandardError
           end
         end
         JSON.pretty_generate(result)
diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb
index 1fd4325..747640d 100644
--- a/lib/vmpooler/api/helpers.rb
+++ b/lib/vmpooler/api/helpers.rb
@@ -1,111 +1,69 @@
 # frozen_string_literal: true
 
+require 'vmpooler/api/input_validator'
+
 module Vmpooler
 
   class API
 
     module Helpers
+      include InputValidator
+
+      def tracer
+        @tracer ||= OpenTelemetry.tracer_provider.tracer('api', Vmpooler::VERSION)
+      end
 
       def has_token?
         request.env['HTTP_X_AUTH_TOKEN'].nil? ? false : true
       end
 
       def valid_token?(backend)
-        return false unless has_token?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return false unless has_token?
 
-        backend.exists?('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
+          backend.exists?("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}") ? true : false
+        end
       end
 
       def validate_token(backend)
-        if valid_token?(backend)
-          backend.hset('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'last', Time.now)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          if valid_token?(backend)
+            backend.hset("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'last', Time.now.to_s)
 
-          return true
+            return true
+          end
+
+          content_type :json
+
+          result = { 'ok' => false }
+
+          headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+          halt 401, JSON.pretty_generate(result)
         end
-
-        content_type :json
-
-        result = { 'ok' => false }
-
-        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
-        halt 401, JSON.pretty_generate(result)
       end
 
       def validate_auth(backend)
-        return if authorized?
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return if authorized?
 
-        content_type :json
+          content_type :json
 
-        result = { 'ok' => false }
+          result = { 'ok' => false }
 
-        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
-        halt 401, JSON.pretty_generate(result)
+          headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+          halt 401, JSON.pretty_generate(result)
+        end
       end
 
       def authorized?
-        @auth ||= Rack::Auth::Basic::Request.new(request.env)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          @auth ||= Rack::Auth::Basic::Request.new(request.env)
 
-        if @auth.provided? and @auth.basic? and @auth.credentials
-          username, password = @auth.credentials
+          if @auth.provided? and @auth.basic? and @auth.credentials
+            username, password = @auth.credentials
 
-          if authenticate(Vmpooler::API.settings.config[:auth], username, password)
-            return true
-          end
-        end
-
-        return false
-      end
-
-      def authenticate_ldap(port, host, user_object, base, username_str, password_str)
-        ldap = Net::LDAP.new(
-          :host => host,
-          :port => port,
-          :encryption => {
-            :method => :start_tls,
-            :tls_options => { :ssl_version => 'TLSv1' }
-          },
-          :base => base,
-          :auth => {
-            :method => :simple,
-            :username => "#{user_object}=#{username_str},#{base}",
-            :password => password_str
-          }
-        )
-
-        return true if ldap.bind
-
-        return false
-      end
-
-      def authenticate(auth, username_str, password_str)
-        case auth['provider']
-        when 'dummy'
-          return (username_str != password_str)
-        when 'ldap'
-          ldap_base = auth[:ldap]['base']
-          ldap_port = auth[:ldap]['port'] || 389
-          ldap_user_obj = auth[:ldap]['user_object']
-          ldap_host = auth[:ldap]['host']
-
-          unless ldap_base.is_a? Array
-            ldap_base = ldap_base.split
-          end
-
-          unless ldap_user_obj.is_a? Array
-            ldap_user_obj = ldap_user_obj.split
-          end
-
-          ldap_base.each do |search_base|
-            ldap_user_obj.each do |search_user_obj|
-              result = authenticate_ldap(
-                ldap_port,
-                ldap_host,
-                search_user_obj,
-                search_base,
-                username_str,
-                password_str
-              )
-              return true if result
+            if authenticate(Vmpooler::API.settings.config[:auth], username, password)
+              return true
             end
           end
 
@@ -113,25 +71,128 @@ module Vmpooler
         end
       end
 
-      def export_tags(backend, hostname, tags)
-        tags.each_pair do |tag, value|
-          next if value.nil? or value.empty?
+      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.name' => host,
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp',
+            'enduser.id' => username_str
+          },
+          kind: :client
+        ) do
+          if service_account_hash
+            username = service_account_hash[:user_dn]
+            password = service_account_hash[:password]
+          else
+            username = "#{user_object}=#{username_str},#{base}"
+            password = password_str
+          end
 
-          backend.hset('vmpooler__vm__' + hostname, 'tag:' + tag, value)
-          backend.hset('vmpooler__tag__' + Date.today.to_s, hostname + ':' + tag, value)
+          ldap = Net::LDAP.new(
+            :host => host,
+            :port => port,
+            :encryption => encryption_hash,
+            :base => base,
+            :auth => {
+              :method => :simple,
+              :username => username,
+              :password => password
+            }
+          )
+
+          if service_account_hash
+            return true if ldap.bind_as(
+              :base => base,
+              :filter => "(#{user_object}=#{username_str})",
+              :password => password_str
+            )
+          elsif ldap.bind
+            return true
+          else
+            return false
+          end
+
+          return false
+        end
+      end
+
+      def authenticate(auth, username_str, password_str)
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'enduser.id' => username_str
+          }
+        ) do
+          case auth['provider']
+          when 'dummy'
+            return (username_str != password_str)
+          when 'ldap'
+            ldap_base = auth[:ldap]['base']
+            ldap_port = auth[:ldap]['port'] || 389
+            ldap_user_obj = auth[:ldap]['user_object']
+            ldap_host = auth[:ldap]['host']
+            ldap_encryption_hash = auth[:ldap]['encryption'] || {
+              :method => :start_tls,
+              :tls_options => { :ssl_version => 'TLSv1' }
+            }
+            service_account_hash = auth[:ldap]['service_account_hash']
+
+            unless ldap_base.is_a? Array
+              ldap_base = ldap_base.split
+            end
+
+            unless ldap_user_obj.is_a? Array
+              ldap_user_obj = ldap_user_obj.split
+            end
+
+            ldap_base.each do |search_base|
+              ldap_user_obj.each do |search_user_obj|
+                result = authenticate_ldap(
+                  ldap_port,
+                  ldap_host,
+                  ldap_encryption_hash,
+                  search_user_obj,
+                  search_base,
+                  username_str,
+                  password_str,
+                  service_account_hash
+                )
+                return true if result
+              end
+            end
+
+            return false
+          end
+        end
+      end
+
+      def export_tags(backend, hostname, tags)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          backend.pipelined do |pipeline|
+            tags.each_pair do |tag, value|
+              next if value.nil? or value.empty?
+
+              pipeline.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value)
+              pipeline.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value)
+            end
+          end
         end
       end
 
       def filter_tags(tags)
-        return unless Vmpooler::API.settings.config[:tagfilter]
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          return unless Vmpooler::API.settings.config[:tagfilter]
 
-        tags.each_pair do |tag, value|
-          next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
+          tags.each_pair do |tag, value|
+            next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
 
-          tags[tag] = value.match(filter).captures.join if value.match(filter)
+            tags[tag] = value.match(filter).captures.join if value.match(filter)
+          end
+
+          tags
         end
-
-        tags
       end
 
       def mean(list)
@@ -143,312 +204,347 @@ module Vmpooler
         /^\d{4}-\d{2}-\d{2}$/ === date_str
       end
 
-      def hostname_shorten(hostname, domain=nil)
-        if domain && hostname =~ /^[\w-]+\.#{domain}$/
-          hostname = hostname[/[^\.]+/]
-        end
-
-        hostname
+      def hostname_shorten(hostname)
+        hostname[/[^.]+/]
       end
 
       def get_task_times(backend, task, date_str)
-        backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
+        end
       end
 
       # Takes the pools and a key to run scard on
       # returns an integer for the total count
       def get_total_across_pools_redis_scard(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.scard(key + pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
           end
+          res.inject(0) { |m, x| m + x }.to_i
         end
-        res.inject(0) { |m, x| m + x }.to_i
       end
 
       # Takes the pools and a key to run scard on
       # returns a hash with each pool name as key and the value being the count as integer
       def get_list_across_pools_redis_scard(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        temp_hash = {}
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.scard(key + pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.scard(key + pool['name'])
+            end
           end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_i
+          end
+          temp_hash
         end
-        pools.each_with_index do |pool, i|
-          temp_hash[pool['name']] = res[i].to_i
-        end
-        temp_hash
       end
 
       # Takes the pools and a key to run hget on
       # returns a hash with each pool name as key and the value as string
       def get_list_across_pools_redis_hget(pools, key, backend)
-        # using pipelined is much faster than querying each of the pools and adding them
-        # as we get the result.
-        temp_hash = {}
-        res = backend.pipelined do
-          pools.each do |pool|
-            backend.hget(key, pool['name'])
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          # using pipelined is much faster than querying each of the pools and adding them
+          # as we get the result.
+          temp_hash = {}
+          res = backend.pipelined do |pipeline|
+            pools.each do |pool|
+              pipeline.hget(key, pool['name'])
+            end
           end
+          pools.each_with_index do |pool, i|
+            temp_hash[pool['name']] = res[i].to_s
+          end
+          temp_hash
         end
-        pools.each_with_index do |pool, i|
-          temp_hash[pool['name']] = res[i].to_s
-        end
-        temp_hash
       end
 
       def get_capacity_metrics(pools, backend)
-        capacity = {
-            current: 0,
-            total: 0,
-            percent: 0
-        }
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          capacity = {
+              current: 0,
+              total: 0,
+              percent: 0
+          }
 
-        pools.each do |pool|
-          capacity[:total] += pool['size'].to_i
+          pools.each do |pool|
+            capacity[:total] += pool['size'].to_i
+          end
+
+          capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
+          if capacity[:total] > 0
+            capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1)
+          end
+
+          capacity
         end
-
-        capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-
-        if capacity[:total] > 0
-          capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1)
-        end
-
-        capacity
       end
 
       def get_queue_metrics(pools, backend)
-        queue = {
-            pending: 0,
-            cloning: 0,
-            booting: 0,
-            ready: 0,
-            running: 0,
-            completed: 0,
-            total: 0
-        }
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          queue = {
+              requested: 0,
+              pending: 0,
+              cloning: 0,
+              booting: 0,
+              ready: 0,
+              running: 0,
+              completed: 0,
+              total: 0
+          }
 
-        queue[:pending]   = get_total_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
-        queue[:ready]     = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-        queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-        queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
+          # Use a single pipeline to fetch all queue counts at once for better performance
+          results = backend.pipelined do |pipeline|
+            # Order matters - we'll use indices to extract values
+            pools.each do |pool|
+              pipeline.scard("vmpooler__provisioning__request#{pool['name']}")    # 0..n-1
+              pipeline.scard("vmpooler__provisioning__processing#{pool['name']}") # n..2n-1
+              pipeline.scard("vmpooler__odcreate__task#{pool['name']}")           # 2n..3n-1
+              pipeline.scard("vmpooler__pending__#{pool['name']}")                # 3n..4n-1
+              pipeline.scard("vmpooler__ready__#{pool['name']}")                  # 4n..5n-1
+              pipeline.scard("vmpooler__running__#{pool['name']}")                # 5n..6n-1
+              pipeline.scard("vmpooler__completed__#{pool['name']}")              # 6n..7n-1
+            end
+            pipeline.get('vmpooler__tasks__clone')           # 7n
+            pipeline.get('vmpooler__tasks__ondemandclone')   # 7n+1
+          end
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
-        queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
-        queue[:booting] = 0 if queue[:booting] < 0
-        queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
+          n = pools.length
+          # Safely extract results with default to empty array if slice returns nil
+          queue[:requested] = (results[0...n] || []).sum(&:to_i) +
+                              (results[n...(2 * n)] || []).sum(&:to_i) +
+                              (results[(2 * n)...(3 * n)] || []).sum(&:to_i)
+          queue[:pending]   = (results[(3 * n)...(4 * n)] || []).sum(&:to_i)
+          queue[:ready]     = (results[(4 * n)...(5 * n)] || []).sum(&:to_i)
+          queue[:running]   = (results[(5 * n)...(6 * n)] || []).sum(&:to_i)
+          queue[:completed] = (results[(6 * n)...(7 * n)] || []).sum(&:to_i)
+          queue[:cloning]   = (results[7 * n] || 0).to_i + (results[7 * n + 1] || 0).to_i
+          queue[:booting]   = queue[:pending].to_i - queue[:cloning].to_i
+          queue[:booting]   = 0 if queue[:booting] < 0
+          queue[:total]     = queue[:requested] + queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
 
-        queue
+          queue
+        end
       end
 
       def get_tag_metrics(backend, date_str, opts = {})
-        opts = {:only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:only => false}.merge(opts)
 
-        tags = {}
+          tags = {}
 
-        backend.hgetall('vmpooler__tag__' + date_str).each do |key, value|
-          hostname = 'unknown'
-          tag = 'unknown'
+          backend.hgetall("vmpooler__tag__#{date_str}").each do |key, value|
+            hostname = 'unknown'
+            tag = 'unknown'
 
-          if key =~ /\:/
-            hostname, tag = key.split(':', 2)
+            if key =~ /:/
+              hostname, tag = key.split(':', 2)
+            end
+
+            next if opts[:only] && tag != opts[:only]
+
+            tags[tag] ||= {}
+            tags[tag][value] ||= 0
+            tags[tag][value] += 1
+
+            tags[tag]['total'] ||= 0
+            tags[tag]['total'] += 1
           end
 
-          if opts[:only]
-            next unless tag == opts[:only]
-          end
-
-          tags[tag] ||= {}
-          tags[tag][value] ||= 0
-          tags[tag][value] += 1
-
-          tags[tag]['total'] ||= 0
-          tags[tag]['total'] += 1
+          tags
         end
-
-        tags
       end
 
       def get_tag_summary(backend, from_date, to_date, opts = {})
-        opts = {:only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:only => false}.merge(opts)
 
-        result = {
-          tag: {},
-          daily: []
-        }
-
-        (from_date..to_date).each do |date|
-          daily = {
-            date: date.to_s,
-            tag: get_tag_metrics(backend, date.to_s, opts)
+          result = {
+            tag: {},
+            daily: []
           }
-          result[:daily].push(daily)
-        end
 
-        result[:daily].each do |daily|
-          daily[:tag].each_key do |tag|
-            result[:tag][tag] ||= {}
+          (from_date..to_date).each do |date|
+            daily = {
+              date: date.to_s,
+              tag: get_tag_metrics(backend, date.to_s, opts)
+            }
+            result[:daily].push(daily)
+          end
 
-            daily[:tag][tag].each do |key, value|
-              result[:tag][tag][key] ||= 0
-              result[:tag][tag][key] += value
+          result[:daily].each do |daily|
+            daily[:tag].each_key do |tag|
+              result[:tag][tag] ||= {}
+
+              daily[:tag][tag].each do |key, value|
+                result[:tag][tag][key] ||= 0
+                result[:tag][tag][key] += value
+              end
             end
           end
-        end
 
-        result
+          result
+        end
       end
 
       def get_task_metrics(backend, task_str, date_str, opts = {})
-        opts = {:bypool => false, :only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:bypool => false, :only => false}.merge(opts)
 
-        task = {
-            duration: {
-                average: 0,
-                min: 0,
-                max: 0,
-                total: 0
-            },
-            count: {
-                total: 0
-            }
-        }
+          task = {
+              duration: {
+                  average: 0,
+                  min: 0,
+                  max: 0,
+                  total: 0
+              },
+              count: {
+                  total: 0
+              }
+          }
 
-        task[:count][:total] = backend.hlen('vmpooler__' + task_str + '__' + date_str).to_i
+          task[:count][:total] = backend.hlen("vmpooler__#{task_str}__#{date_str}").to_i
 
-        if task[:count][:total] > 0
-          if opts[:bypool] == true
-            task_times_bypool = {}
+          if task[:count][:total] > 0
+            if opts[:bypool] == true
+              task_times_bypool = {}
 
-            task[:count][:pool]    = {}
-            task[:duration][:pool] = {}
+              task[:count][:pool]    = {}
+              task[:duration][:pool] = {}
 
-            backend.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
-              pool     = 'unknown'
-              hostname = 'unknown'
+              backend.hgetall("vmpooler__#{task_str}__#{date_str}").each do |key, value|
+                pool     = 'unknown'
+                hostname = 'unknown'
 
-              if key =~ /\:/
-                pool, hostname = key.split(':')
-              else
-                hostname = key
+                if key =~ /:/
+                  pool, hostname = key.split(':')
+                else
+                  hostname = key
+                end
+
+                task[:count][:pool][pool]    ||= {}
+                task[:duration][:pool][pool] ||= {}
+
+                task_times_bypool[pool] ||= []
+                task_times_bypool[pool].push(value.to_f)
               end
 
-              task[:count][:pool][pool]    ||= {}
-              task[:duration][:pool][pool] ||= {}
+              task_times_bypool.each_key do |pool|
+                task[:count][:pool][pool][:total] = task_times_bypool[pool].length
 
-              task_times_bypool[pool] ||= []
-              task_times_bypool[pool].push(value.to_f)
+                task[:duration][:pool][pool][:total]                                   = task_times_bypool[pool].reduce(:+).to_f
+                task[:duration][:pool][pool][:average]                                 = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1)
+                task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax
+              end
             end
 
-            task_times_bypool.each_key do |pool|
-              task[:count][:pool][pool][:total] = task_times_bypool[pool].length
+            task_times = get_task_times(backend, task_str, date_str)
 
-              task[:duration][:pool][pool][:total]                                   = task_times_bypool[pool].reduce(:+).to_f
-              task[:duration][:pool][pool][:average]                                 = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1)
-              task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax
+            task[:duration][:total]                      = task_times.reduce(:+).to_f
+            task[:duration][:average]                    = (task[:duration][:total] / task[:count][:total]).round(1)
+            task[:duration][:min], task[:duration][:max] = task_times.minmax
+          end
+
+          if opts[:only]
+            task.each_key do |key|
+              task.delete(key) unless key.to_s == opts[:only]
             end
           end
 
-          task_times = get_task_times(backend, task_str, date_str)
-
-          task[:duration][:total]                      = task_times.reduce(:+).to_f
-          task[:duration][:average]                    = (task[:duration][:total] / task[:count][:total]).round(1)
-          task[:duration][:min], task[:duration][:max] = task_times.minmax
+          task
         end
-
-        if opts[:only]
-          task.each_key do |key|
-            task.delete(key) unless key.to_s == opts[:only]
-          end
-        end
-
-        task
       end
 
       def get_task_summary(backend, task_str, from_date, to_date, opts = {})
-        opts = {:bypool => false, :only => false}.merge(opts)
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          opts = {:bypool => false, :only => false}.merge(opts)
 
-        task_sym = task_str.to_sym
+          task_sym = task_str.to_sym
 
-        result = {
-          task_sym => {},
-          daily: []
-        }
-
-        (from_date..to_date).each do |date|
-          daily = {
-            date: date.to_s,
-            task_sym => get_task_metrics(backend, task_str, date.to_s, opts)
+          result = {
+            task_sym => {},
+            daily: []
           }
-          result[:daily].push(daily)
-        end
 
-        daily_task = {}
-        daily_task_bypool = {} if opts[:bypool] == true
+          (from_date..to_date).each do |date|
+            daily = {
+              date: date.to_s,
+              task_sym => get_task_metrics(backend, task_str, date.to_s, opts)
+            }
+            result[:daily].push(daily)
+          end
 
-        result[:daily].each do |daily|
-          daily[task_sym].each_key do |type|
-            result[task_sym][type] ||= {}
-            daily_task[type] ||= {}
+          daily_task = {}
+          daily_task_bypool = {} if opts[:bypool] == true
 
-            ['min', 'max'].each do |key|
-              if daily[task_sym][type][key]
-                daily_task[type][:data] ||= []
-                daily_task[type][:data].push(daily[task_sym][type][key])
+          result[:daily].each do |daily|
+            daily[task_sym].each_key do |type|
+              result[task_sym][type] ||= {}
+              daily_task[type] ||= {}
+
+              ['min', 'max'].each do |key|
+                if daily[task_sym][type][key]
+                  daily_task[type][:data] ||= []
+                  daily_task[type][:data].push(daily[task_sym][type][key])
+                end
+              end
+
+              result[task_sym][type][:total] ||= 0
+              result[task_sym][type][:total] += daily[task_sym][type][:total]
+
+              if opts[:bypool] == true
+                result[task_sym][type][:pool] ||= {}
+                daily_task_bypool[type] ||= {}
+
+                next unless daily[task_sym][type][:pool]
+
+                daily[task_sym][type][:pool].each_key do |pool|
+                  result[task_sym][type][:pool][pool] ||= {}
+                  daily_task_bypool[type][pool] ||= {}
+
+                  ['min', 'max'].each do |key|
+                    if daily[task_sym][type][:pool][pool][key.to_sym]
+                      daily_task_bypool[type][pool][:data] ||= []
+                      daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym])
+                    end
+                  end
+
+                  result[task_sym][type][:pool][pool][:total] ||= 0
+                  result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total]
+                end
               end
             end
+          end
 
-            result[task_sym][type][:total] ||= 0
-            result[task_sym][type][:total] += daily[task_sym][type][:total]
+          result[task_sym].each_key do |type|
+            if daily_task[type][:data]
+              result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax
+              result[task_sym][type][:average] = mean(daily_task[type][:data])
+            end
 
             if opts[:bypool] == true
-              result[task_sym][type][:pool] ||= {}
-              daily_task_bypool[type] ||= {}
-
-              next unless daily[task_sym][type][:pool]
-
-              daily[task_sym][type][:pool].each_key do |pool|
-                result[task_sym][type][:pool][pool] ||= {}
-                daily_task_bypool[type][pool] ||= {}
-
-                ['min', 'max'].each do |key|
-                  if daily[task_sym][type][:pool][pool][key.to_sym]
-                    daily_task_bypool[type][pool][:data] ||= []
-                    daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym])
+              result[task_sym].each_key do |type|
+                result[task_sym][type][:pool].each_key do |pool|
+                  if daily_task_bypool[type][pool][:data]
+                    result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax
+                    result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data])
                   end
                 end
-
-                result[task_sym][type][:pool][pool][:total] ||= 0
-                result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total]
               end
             end
           end
+
+          result
         end
-
-        result[task_sym].each_key do |type|
-          if daily_task[type][:data]
-            result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax
-            result[task_sym][type][:average] = mean(daily_task[type][:data])
-          end
-
-          if opts[:bypool] == true
-            result[task_sym].each_key do |type|
-              result[task_sym][type][:pool].each_key do |pool|
-                if daily_task_bypool[type][pool][:data]
-                  result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax
-                  result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data])
-                end
-              end
-            end
-          end
-        end
-
-        result
       end
 
       def pool_index(pools)
@@ -462,11 +558,13 @@ module Vmpooler
       end
 
       def template_ready?(pool, backend)
-        prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
-        return false if prepared_template.nil?
-        return true if pool['template'] == prepared_template
+        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
+          prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
+          return false if prepared_template.nil?
+          return true if pool['template'] == prepared_template
 
-        return false
+          return false
+        end
       end
 
       def is_integer?(x)
@@ -477,10 +575,19 @@ module Vmpooler
       end
 
       def open_socket(host, domain = nil, timeout = 1, port = 22, &_block)
-        Timeout.timeout(timeout) do
+        tracer.in_span(
+          "Vmpooler::API::Helpers.#{__method__}",
+          attributes: {
+            'net.peer.port' => port,
+            'net.transport' => 'ip_tcp'
+          },
+          kind: :client
+        ) do
           target_host = host
           target_host = "#{host}.#{domain}" if domain
-          sock = TCPSocket.new target_host, port
+          span = OpenTelemetry::Trace.current_span
+          span.set_attribute('net.peer.name', target_host)
+          sock = TCPSocket.new(target_host, port, connect_timeout: timeout)
           begin
             yield sock if block_given?
           ensure
@@ -488,16 +595,6 @@ module Vmpooler
           end
         end
       end
-
-      def vm_ready?(vm_name, domain = nil)
-        begin
-          open_socket(vm_name, domain)
-        rescue StandardError => _e
-          return false
-        end
-
-        true
-      end
     end
   end
 end
diff --git a/lib/vmpooler/api/input_validator.rb b/lib/vmpooler/api/input_validator.rb
new file mode 100644
index 0000000..add4d6a
--- /dev/null
+++ b/lib/vmpooler/api/input_validator.rb
@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Input validation helpers to enhance security
+    module InputValidator
+      # Maximum lengths to prevent abuse
+      MAX_HOSTNAME_LENGTH = 253
+      MAX_TAG_KEY_LENGTH = 50
+      MAX_TAG_VALUE_LENGTH = 255
+      MAX_REASON_LENGTH = 500
+      MAX_POOL_NAME_LENGTH = 100
+      MAX_TOKEN_LENGTH = 64
+
+      # Valid patterns
+      HOSTNAME_PATTERN = /\A[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)* \z/ix.freeze
+      POOL_NAME_PATTERN = /\A[a-zA-Z0-9_-]+\z/.freeze
+      TAG_KEY_PATTERN = /\A[a-zA-Z0-9_\-.]+\z/.freeze
+      TOKEN_PATTERN = /\A[a-zA-Z0-9\-_]+\z/.freeze
+      INTEGER_PATTERN = /\A\d+\z/.freeze
+
+      class ValidationError < StandardError; end
+
+      # Validate hostname format and length
+      def validate_hostname(hostname)
+        return error_response('Hostname is required') if hostname.nil? || hostname.empty?
+        return error_response('Hostname too long') if hostname.length > MAX_HOSTNAME_LENGTH
+        return error_response('Invalid hostname format') unless hostname.match?(HOSTNAME_PATTERN)
+
+        true
+      end
+
+      # Validate pool/template name
+      def validate_pool_name(pool_name)
+        return error_response('Pool name is required') if pool_name.nil? || pool_name.empty?
+        return error_response('Pool name too long') if pool_name.length > MAX_POOL_NAME_LENGTH
+        return error_response('Invalid pool name format') unless pool_name.match?(POOL_NAME_PATTERN)
+
+        true
+      end
+
+      # Validate tag key and value
+      def validate_tag(key, value)
+        return error_response('Tag key is required') if key.nil? || key.empty?
+        return error_response('Tag key too long') if key.length > MAX_TAG_KEY_LENGTH
+        return error_response('Invalid tag key format') unless key.match?(TAG_KEY_PATTERN)
+
+        if value
+          return error_response('Tag value too long') if value.length > MAX_TAG_VALUE_LENGTH
+
+          # Sanitize value to prevent injection attacks
+          sanitized_value = value.gsub(/[^\w\s\-.@:\/]/, '')
+          return error_response('Tag value contains invalid characters') if sanitized_value != value
+        end
+
+        true
+      end
+
+      # Validate token format
+      def validate_token_format(token)
+        return error_response('Token is required') if token.nil? || token.empty?
+        return error_response('Token too long') if token.length > MAX_TOKEN_LENGTH
+        return error_response('Invalid token format') unless token.match?(TOKEN_PATTERN)
+
+        true
+      end
+
+      # Validate integer parameter
+      def validate_integer(value, name = 'value', min: nil, max: nil)
+        return error_response("#{name} is required") if value.nil?
+
+        value_str = value.to_s
+        return error_response("#{name} must be a valid integer") unless value_str.match?(INTEGER_PATTERN)
+
+        int_value = value.to_i
+        return error_response("#{name} must be at least #{min}") if min && int_value < min
+        return error_response("#{name} must be at most #{max}") if max && int_value > max
+
+        int_value
+      end
+
+      # Validate VM request count
+      def validate_vm_count(count)
+        validated = validate_integer(count, 'VM count', min: 1, max: 100)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate disk size
+      def validate_disk_size(size)
+        validated = validate_integer(size, 'Disk size', min: 1, max: 2048)
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate lifetime (TTL) in hours
+      def validate_lifetime(lifetime)
+        validated = validate_integer(lifetime, 'Lifetime', min: 1, max: 168) # max 1 week
+        return validated if validated.is_a?(Hash) # error response
+
+        validated
+      end
+
+      # Validate reason text
+      def validate_reason(reason)
+        return true if reason.nil? || reason.empty?
+        return error_response('Reason too long') if reason.length > MAX_REASON_LENGTH
+
+        # Sanitize to prevent XSS/injection
+        sanitized = reason.gsub(/[<>"']/, '')
+        return error_response('Reason contains invalid characters') if sanitized != reason
+
+        true
+      end
+
+      # Sanitize JSON body to prevent injection
+      def sanitize_json_body(body)
+        return {} if body.nil? || body.empty?
+
+        begin
+          parsed = JSON.parse(body)
+          return error_response('Request body must be a JSON object') unless parsed.is_a?(Hash)
+
+          # Limit depth and size to prevent DoS
+          return error_response('Request body too complex') if json_depth(parsed) > 5
+          return error_response('Request body too large') if body.length > 10_240 # 10KB max
+
+          parsed
+        rescue JSON::ParserError => e
+          error_response("Invalid JSON: #{e.message}")
+        end
+      end
+
+      # Check if validation result is an error
+      def validation_error?(result)
+        result.is_a?(Hash) && result['ok'] == false
+      end
+
+      private
+
+      def error_response(message)
+        { 'ok' => false, 'error' => message }
+      end
+
+      def json_depth(obj, depth = 0)
+        return depth unless obj.is_a?(Hash) || obj.is_a?(Array)
+        return depth + 1 if obj.empty?
+
+        if obj.is_a?(Hash)
+          depth + 1 + obj.values.map { |v| json_depth(v, 0) }.max
+        else
+          depth + 1 + obj.map { |v| json_depth(v, 0) }.max
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/rate_limiter.rb b/lib/vmpooler/api/rate_limiter.rb
new file mode 100644
index 0000000..8ecfb62
--- /dev/null
+++ b/lib/vmpooler/api/rate_limiter.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    # Rate limiter middleware to protect against abuse
+    # Uses Redis to track request counts per IP and token
+    class RateLimiter
+      DEFAULT_LIMITS = {
+        global_per_ip: { limit: 100, period: 60 }, # 100 requests per minute per IP
+        authenticated: { limit: 500, period: 60 }, # 500 requests per minute with token
+        vm_creation: { limit: 20, period: 60 },    # 20 VM creations per minute
+        vm_deletion: { limit: 50, period: 60 }     # 50 VM deletions per minute
+      }.freeze
+
+      def initialize(app, redis, config = {})
+        @app = app
+        @redis = redis
+        @config = DEFAULT_LIMITS.merge(config[:rate_limits] || {})
+        @enabled = config.fetch(:rate_limiting_enabled, true)
+      end
+
+      def call(env)
+        return @app.call(env) unless @enabled
+
+        request = Rack::Request.new(env)
+        client_id = identify_client(request)
+        endpoint_type = classify_endpoint(request)
+
+        # Check rate limits
+        return rate_limit_response(client_id, endpoint_type) if rate_limit_exceeded?(client_id, endpoint_type, request)
+
+        # Track the request
+        increment_request_count(client_id, endpoint_type)
+
+        @app.call(env)
+      end
+
+      private
+
+      def identify_client(request)
+        # Prioritize token-based identification for authenticated requests
+        token = request.env['HTTP_X_AUTH_TOKEN']
+        return "token:#{token}" if token && !token.empty?
+
+        # Fall back to IP address
+        ip = request.ip || request.env['REMOTE_ADDR'] || 'unknown'
+        "ip:#{ip}"
+      end
+
+      def classify_endpoint(request)
+        path = request.path
+        method = request.request_method
+
+        return :vm_creation if method == 'POST' && path.include?('/vm')
+        return :vm_deletion if method == 'DELETE' && path.include?('/vm')
+        return :authenticated if request.env['HTTP_X_AUTH_TOKEN']
+
+        :global_per_ip
+      end
+
+      def rate_limit_exceeded?(client_id, endpoint_type, _request)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        current_count = @redis.get(key).to_i
+        current_count >= limit_config[:limit]
+      rescue StandardError => e
+        # If Redis fails, allow the request through (fail open)
+        warn "Rate limiter Redis error: #{e.message}"
+        false
+      end
+
+      def increment_request_count(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        @redis.pipelined do |pipeline|
+          pipeline.incr(key)
+          pipeline.expire(key, limit_config[:period])
+        end
+      rescue StandardError => e
+        # Log error but don't fail the request
+        warn "Rate limiter increment error: #{e.message}"
+      end
+
+      def rate_limit_response(client_id, endpoint_type)
+        limit_config = @config[endpoint_type] || @config[:global_per_ip]
+        key = "vmpooler__ratelimit__#{endpoint_type}__#{client_id}"
+
+        begin
+          ttl = @redis.ttl(key)
+        rescue StandardError
+          ttl = limit_config[:period]
+        end
+
+        headers = {
+          'Content-Type' => 'application/json',
+          'X-RateLimit-Limit' => limit_config[:limit].to_s,
+          'X-RateLimit-Remaining' => '0',
+          'X-RateLimit-Reset' => (Time.now.to_i + ttl).to_s,
+          'Retry-After' => ttl.to_s
+        }
+
+        body = JSON.pretty_generate({
+                                      'ok' => false,
+                                      'error' => 'Rate limit exceeded',
+                                      'limit' => limit_config[:limit],
+                                      'period' => limit_config[:period],
+                                      'retry_after' => ttl
+                                    })
+
+        [429, headers, [body]]
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/api/reroute.rb b/lib/vmpooler/api/reroute.rb
deleted file mode 100644
index bc44106..0000000
--- a/lib/vmpooler/api/reroute.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Vmpooler
-  class API
-    class Reroute < Sinatra::Base
-      api_version = '1'
-
-      get '/status/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/status")
-      end
-
-      get '/summary/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary")
-      end
-
-      get '/summary/:route/?:key?/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}")
-      end
-
-      get '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      post '/token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
-      end
-
-      get '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      delete '/token/:token/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
-      end
-
-      get '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
-      end
-
-      post '/vm/:template/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}")
-      end
-
-      get '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      delete '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      put '/vm/:hostname/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
-      end
-
-      post '/vm/:hostname/snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot")
-      end
-
-      post '/vm/:hostname/snapshot/:snapshot/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}")
-      end
-
-      put '/vm/:hostname/disk/:size/?' do
-        call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}")
-      end
-    end
-  end
-end
diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb
deleted file mode 100644
index 066af86..0000000
--- a/lib/vmpooler/api/v1.rb
+++ /dev/null
@@ -1,1483 +0,0 @@
-# frozen_string_literal: true
-
-require 'vmpooler/util/parsing'
-
-module Vmpooler
-  class API
-  class V1 < Sinatra::Base
-    api_version = '1'
-    api_prefix  = "/api/v#{api_version}"
-
-    helpers do
-      include Vmpooler::API::Helpers
-    end
-
-    def backend
-      Vmpooler::API.settings.redis
-    end
-
-    def metrics
-      Vmpooler::API.settings.metrics
-    end
-
-    def config
-      Vmpooler::API.settings.config[:config]
-    end
-
-    def pools
-      Vmpooler::API.settings.config[:pools]
-    end
-
-    def pool_exists?(template)
-      Vmpooler::API.settings.config[:pool_names].include?(template)
-    end
-
-    def need_auth!
-      validate_auth(backend)
-    end
-
-    def need_token!
-      validate_token(backend)
-    end
-
-    def checkoutlock
-      Vmpooler::API.settings.checkoutlock
-    end
-
-    def get_template_aliases(template)
-      result = []
-      aliases = Vmpooler::API.settings.config[:alias]
-      if aliases
-        result += aliases[template] if aliases[template].is_a?(Array)
-        template_backends << aliases[template] if aliases[template].is_a?(String)
-      end
-      result
-    end
-
-    def get_pool_weights(template_backends)
-      pool_index = pool_index(pools)
-      weighted_pools = {}
-      template_backends.each do |t|
-        next unless pool_index.key? t
-
-        index = pool_index[t]
-        clone_target = pools[index]['clone_target'] || config['clone_target']
-        next unless config.key?('backend_weight')
-
-        weight = config['backend_weight'][clone_target]
-        if weight
-          weighted_pools[t] = weight
-        end
-      end
-      weighted_pools
-    end
-
-    def count_selection(selection)
-      result = {}
-      selection.uniq.each do |poolname|
-        result[poolname] = selection.count(poolname)
-      end
-      result
-    end
-
-    def evaluate_template_aliases(template, count)
-      template_backends = []
-      template_backends << template if backend.sismember('vmpooler__pools', template)
-      selection = []
-      aliases = get_template_aliases(template)
-      if aliases
-        template_backends += aliases
-        weighted_pools = get_pool_weights(template_backends)
-
-        if weighted_pools.count > 1 && weighted_pools.count == template_backends.count
-          pickup = Pickup.new(weighted_pools)
-          count.to_i.times do
-            selection << pickup.pick
-          end
-        else
-          count.to_i.times do
-            selection << template_backends.sample
-          end
-        end
-      end
-
-      count_selection(selection)
-    end
-
-    def fetch_single_vm(template)
-      template_backends = [template]
-      aliases = Vmpooler::API.settings.config[:alias]
-      if aliases
-        template_backends += aliases[template] if aliases[template].is_a?(Array)
-        template_backends << aliases[template] if aliases[template].is_a?(String)
-        pool_index = pool_index(pools)
-        weighted_pools = {}
-        template_backends.each do |t|
-          next unless pool_index.key? t
-
-          index = pool_index[t]
-          clone_target = pools[index]['clone_target'] || config['clone_target']
-          next unless config.key?('backend_weight')
-
-          weight = config['backend_weight'][clone_target]
-          if weight
-            weighted_pools[t] = weight
-          end
-        end
-
-        if weighted_pools.count == template_backends.count
-          pickup = Pickup.new(weighted_pools)
-          selection = pickup.pick
-          template_backends.delete(selection)
-          template_backends.unshift(selection)
-        else
-          first = template_backends.sample
-          template_backends.delete(first)
-          template_backends.unshift(first)
-        end
-      end
-
-      checkoutlock.synchronize do
-        template_backends.each do |template_backend|
-          vms = backend.smembers("vmpooler__ready__#{template_backend}")
-          next if vms.empty?
-
-          vms.reverse.each do |vm|
-            ready = vm_ready?(vm, config['domain'])
-            if ready
-              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
-              if smoved
-                return [vm, template_backend, template]
-              else
-                metrics.increment("checkout.smove.failed.#{template_backend}")
-                return [nil, nil, nil]
-              end
-            else
-              backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
-              metrics.increment("checkout.nonresponsive.#{template_backend}")
-            end
-          end
-        end
-        [nil, nil, nil]
-      end
-    end
-
-    def return_vm_to_ready_state(template, vm)
-      backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
-    end
-
-    def account_for_starting_vm(template, vm)
-      backend.sadd("vmpooler__migrating__#{template}", vm)
-      backend.hset("vmpooler__active__#{template}", vm, Time.now)
-      backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
-
-      if Vmpooler::API.settings.config[:auth] and has_token?
-        validate_token(backend)
-
-        backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
-        backend.hset('vmpooler__vm__' + vm, 'token:user',
-                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')
-        )
-
-        if config['vm_lifetime_auth'].to_i > 0
-          backend.hset('vmpooler__vm__' + vm, 'lifetime', config['vm_lifetime_auth'].to_i)
-        end
-      end
-    end
-
-    def update_result_hosts(result, template, vm)
-      result[template] ||= {}
-      if result[template]['hostname']
-        result[template]['hostname'] = Array(result[template]['hostname'])
-        result[template]['hostname'].push(vm)
-      else
-        result[template]['hostname'] = vm
-      end
-    end
-
-    def atomically_allocate_vms(payload)
-      result = { 'ok' => false }
-      failed = false
-      vms = []
-
-      payload.each do |requested, count|
-        count.to_i.times do |_i|
-          vmname, vmpool, vmtemplate = fetch_single_vm(requested)
-          if !vmname
-            failed = true
-            metrics.increment('checkout.empty.' + requested)
-            break
-          else
-            vms << [vmpool, vmname, vmtemplate]
-            metrics.increment('checkout.success.' + vmtemplate)
-          end
-        end
-      end
-
-      if failed
-        vms.each do |(vmpool, vmname, _vmtemplate)|
-          return_vm_to_ready_state(vmpool, vmname)
-        end
-        status 503
-      else
-        vms.each do |(vmpool, vmname, vmtemplate)|
-          account_for_starting_vm(vmpool, vmname)
-          update_result_hosts(result, vmtemplate, vmname)
-        end
-
-        result['ok'] = true
-        result['domain'] = config['domain'] if config['domain']
-      end
-
-      result
-    end
-
-    def update_pool_size(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_sizes
-
-      payload.each do |poolname, size|
-        unless pools[pool_index[poolname]]['size'] == size.to_i
-          pools[pool_index[poolname]]['size'] = size.to_i
-          backend.hset('vmpooler__config__poolsize', poolname, size)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def update_pool_template(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_pool_templates
-
-      payload.each do |poolname, template|
-        unless pools[pool_index[poolname]]['template'] == template
-          pools[pool_index[poolname]]['template'] = template
-          backend.hset('vmpooler__config__template', poolname, template)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def reset_pool(payload)
-      result = { 'ok' => false }
-
-      payload.each do |poolname, _count|
-        backend.sadd('vmpooler__poolreset', poolname)
-      end
-      status 201
-      result['ok'] = true
-      result
-    end
-
-    def update_clone_target(payload)
-      result = { 'ok' => false }
-
-      pool_index = pool_index(pools)
-      pools_updated = 0
-      sync_clone_targets
-
-      payload.each do |poolname, clone_target|
-        unless pools[pool_index[poolname]]['clone_target'] == clone_target
-          pools[pool_index[poolname]]['clone_target'] = clone_target
-          backend.hset('vmpooler__config__clone_target', poolname, clone_target)
-          pools_updated += 1
-          status 201
-        end
-      end
-      status 200 unless pools_updated > 0
-      result['ok'] = true
-      result
-    end
-
-    def sync_pool_templates
-      pool_index = pool_index(pools)
-      template_configs = backend.hgetall('vmpooler__config__template')
-      template_configs&.each do |poolname, template|
-        next unless pool_index.include? poolname
-
-        pools[pool_index[poolname]]['template'] = template
-      end
-    end
-
-    def sync_pool_sizes
-      pool_index = pool_index(pools)
-      poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
-      poolsize_configs&.each do |poolname, size|
-        next unless pool_index.include? poolname
-
-        pools[pool_index[poolname]]['size'] = size.to_i
-      end
-    end
-
-    def sync_clone_targets
-      pool_index = pool_index(pools)
-      clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
-      clone_target_configs&.each do |poolname, clone_target|
-        next unless pool_index.include? poolname
-
-        pools[pool_index[poolname]]['clone_target'] = clone_target
-      end
-    end
-
-    def too_many_requested?(payload)
-      payload&.each do |poolname, count|
-        next unless count.to_i > config['max_ondemand_instances_per_request']
-
-        metrics.increment('ondemandrequest_fail.toomanyrequests.' + poolname)
-        return true
-      end
-      false
-    end
-
-    def generate_ondemand_request(payload)
-      result = { 'ok': false }
-
-      requested_instances = payload.reject { |k, _v| k == 'request_id' }
-      if too_many_requested?(requested_instances)
-        result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
-        status 403
-        return result
-      end
-
-      score = Time.now.to_i
-      request_id = payload['request_id']
-      request_id ||= generate_request_id
-      result['request_id'] = request_id
-
-      if backend.exists?("vmpooler__odrequest__#{request_id}")
-        result['message'] = "request_id '#{request_id}' has already been created"
-        status 409
-        metrics.increment('ondemandrequest_generate.duplicaterequests')
-        return result
-      end
-
-      status 201
-
-      platforms_with_aliases = []
-      requested_instances.each do |poolname, count|
-        selection = evaluate_template_aliases(poolname, count)
-        selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
-      end
-      platforms_string = platforms_with_aliases.join(',')
-
-      return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
-
-      backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
-      if Vmpooler::API.settings.config[:auth] and has_token?
-        backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
-        backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
-                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user'))
-      end
-
-      result['domain'] = config['domain'] if config['domain']
-      result[:ok] = true
-      metrics.increment('ondemandrequest_generate.success')
-      result
-    end
-
-    def generate_request_id
-      SecureRandom.uuid
-    end
-
-    get '/' do
-      sync_pool_sizes
-      redirect to('/dashboard/')
-    end
-
-    # Provide run-time statistics
-    #
-    # Example:
-    #
-    # {
-    #   "boot": {
-    #     "duration": {
-    #       "average": 163.6,
-    #       "min": 65.49,
-    #       "max": 830.07,
-    #       "total": 247744.71000000002
-    #     },
-    #     "count": {
-    #       "total": 1514
-    #     }
-    #   },
-    #   "capacity": {
-    #     "current": 968,
-    #     "total": 975,
-    #     "percent": 99.3
-    #   },
-    #   "clone": {
-    #     "duration": {
-    #       "average": 17.0,
-    #       "min": 4.66,
-    #       "max": 637.96,
-    #       "total": 25634.15
-    #     },
-    #     "count": {
-    #       "total": 1507
-    #     }
-    #   },
-    #   "queue": {
-    #     "pending": 12,
-    #     "cloning": 0,
-    #     "booting": 12,
-    #     "ready": 968,
-    #     "running": 367,
-    #     "completed": 0,
-    #     "total": 1347
-    #   },
-    #   "pools": {
-    #     "ready": 100,
-    #     "running": 120,
-    #     "pending": 5,
-    #     "max": 250,
-    #   }
-    #   "status": {
-    #     "ok": true,
-    #     "message": "Battle station fully armed and operational.",
-    #     "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools
-    #       "redhat-7-x86_64",
-    #       "ubuntu-1404-i386"
-    #     ],
-    #     "uptime": 179585.9
-    # }
-    #
-    # If the query parameter 'view' is provided, it will be used to select which top level
-    # element to compute and return. Select them by specifying them in a comma separated list.
-    # For example /status?view=capacity,boot
-    # would return only the "capacity" and "boot" statistics. "status" is always returned
-
-    get "#{api_prefix}/status/?" do
-      content_type :json
-
-      if params[:view]
-        views = params[:view].split(",")
-      end
-
-      result = {
-        status: {
-          ok: true,
-          message: 'Battle station fully armed and operational.'
-        }
-      }
-
-      sync_pool_sizes
-
-      result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
-      result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
-      result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
-      result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot")
-
-      # Check for empty pools
-      result[:pools] = {} unless views and not views.include?("pools")
-      ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-      running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-      pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
-      lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
-
-      unless views and not views.include?("pools")
-        pools.each do |pool|
-          # REMIND: move this out of the API and into the back-end
-          ready    = ready_hash[pool['name']]
-          running  = running_hash[pool['name']]
-          pending  = pending_hash[pool['name']]
-          max      = pool['size']
-          lastBoot = lastBoot_hash[pool['name']]
-          aka      = pool['alias']
-
-          result[:pools][pool['name']] = {
-            ready: ready,
-            running: running,
-            pending: pending,
-            max: max,
-            lastBoot: lastBoot
-          }
-
-          if aka
-            result[:pools][pool['name']][:alias] = aka
-          end
-
-          # for backwards compatibility, include separate "empty" stats in "status" block
-          if ready == 0 && max != 0
-            result[:status][:empty] ||= []
-            result[:status][:empty].push(pool['name'])
-
-            result[:status][:ok] = false
-            result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
-          end
-        end
-      end
-
-      result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
-
-      JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
-    end
-
-    # request statistics for specific pools by passing parameter 'pool'
-    # with a coma separated list of pools we want to query ?pool=ABC,DEF
-    # returns the ready, max numbers and the aliases (if set)
-    get "#{api_prefix}/poolstat/?" do
-      content_type :json
-
-      result = {}
-
-      poolscopy = []
-
-      if params[:pool]
-        subpool = params[:pool].split(",")
-        poolscopy = pools.select do |p|
-          if subpool.include?(p['name'])
-            true
-          elsif !p['alias'].nil?
-            if p['alias'].is_a?(Array)
-              (p['alias'] & subpool).any?
-            elsif p['alias'].is_a?(String)
-              subpool.include?(p['alias'])
-            end
-          end
-        end
-      end
-
-      result[:pools] = {}
-
-      poolscopy.each do |pool|
-        result[:pools][pool['name']] = {}
-
-        max      = pool['size']
-        aka      = pool['alias']
-
-        result[:pools][pool['name']][:max] = max
-
-        if aka
-          result[:pools][pool['name']][:alias] = aka
-        end
-      end
-
-      ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
-
-      ready_hash.each { |k, v| result[:pools][k][:ready] = v }
-
-      JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
-    end
-
-    # requests the total number of running VMs
-    get "#{api_prefix}/totalrunning/?" do
-      content_type :json
-      queue = {
-          running: 0
-      }
-
-      queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-
-      JSON.pretty_generate(queue)
-    end
-
-    get "#{api_prefix}/summary/?" do
-      content_type :json
-
-      result = {
-        daily: []
-      }
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true)
-      clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true)
-      tag = get_tag_summary(backend, from_date, to_date)
-
-      result[:boot] = boot[:boot]
-      result[:clone] = clone[:clone]
-      result[:tag] = tag[:tag]
-
-      daily = {}
-
-      boot[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:boot] = day[:boot]
-      end
-
-      clone[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:clone] = day[:clone]
-      end
-
-      tag[:daily].each do |day|
-        daily[day[:date]] ||= {}
-        daily[day[:date]][:tag] = day[:tag]
-      end
-
-      daily.each_key do |day|
-        result[:daily].push({
-          date: day,
-          boot: daily[day][:boot],
-          clone: daily[day][:clone],
-          tag: daily[day][:tag]
-        })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/summary/:route/?:key?/?" do
-      content_type :json
-
-      result = {}
-
-      from_param = params[:from] || Date.today.to_s
-      to_param   = params[:to]   || Date.today.to_s
-
-      # Validate date formats
-      [from_param, to_param].each do |param|
-        if !validate_date_str(param.to_s)
-          halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
-        end
-      end
-
-      from_date, to_date = Date.parse(from_param), Date.parse(to_param)
-
-      if to_date < from_date
-        halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
-      elsif from_date > Date.today
-        halt 400, 'Date range is invalid, \'from\' must be in the past.'
-      end
-
-      case params[:route]
-        when 'boot'
-          result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'clone'
-          result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key])
-        when 'tag'
-          result = get_tag_summary(backend, from_date, to_date, :only => params[:key])
-        else
-          halt 404, JSON.pretty_generate({ 'ok' => false })
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        backend.keys('vmpooler__token__*').each do |key|
-          data = backend.hgetall(key)
-
-          if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
-            token = key.split('__').last
-
-            result[token] ||= {}
-
-            result[token]['created'] = data['created']
-            result[token]['last'] = data['last'] || 'never'
-
-            result['ok'] = true
-          end
-        end
-
-        if result['ok']
-          status 200
-        else
-          status 404
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        token = backend.hgetall('vmpooler__token__' + params[:token])
-
-        if not token.nil? and not token.empty?
-          status 200
-
-          pools.each do |pool|
-            backend.smembers('vmpooler__running__' + pool['name']).each do |vm|
-              if backend.hget('vmpooler__vm__' + vm, 'token:token') == params[:token]
-                token['vms'] ||= {}
-                token['vms']['running'] ||= []
-                token['vms']['running'].push(vm)
-              end
-            end
-          end
-
-          result = { 'ok' => true, params[:token] => token }
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/token/:token/?" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        if backend.del('vmpooler__token__' + params[:token]).to_i > 0
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/token" do
-      content_type :json
-
-      status 404
-      result = { 'ok' => false }
-
-      if Vmpooler::API.settings.config[:auth]
-        status 401
-
-        need_auth!
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.hset('vmpooler__token__' + result['token'], 'user', @auth.username)
-        backend.hset('vmpooler__token__' + result['token'], 'created', Time.now)
-
-        status 200
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/?" do
-      content_type :json
-
-      result = []
-
-      pools.each do |pool|
-        result.push(pool['name'])
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/ondemandvm/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.post.ondemand.requestid')
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      result = { 'ok' => false }
-
-      begin
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
-          if invalid.empty?
-            result = generate_ondemand_request(payload)
-          else
-            result[:bad_templates] = invalid
-            invalid.each do |bad_template|
-              metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
-            end
-            status 404
-          end
-        else
-          metrics.increment('ondemandrequest_fail.invalid.unknown')
-          status 404
-        end
-      rescue JSON::ParserError
-        status 400
-        result = {
-          'ok' => false,
-          'message' => 'JSON payload could not be parsed'
-        }
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/ondemandvm/:template/?" do
-      content_type :json
-      result = { 'ok' => false }
-      metrics.increment('http_requests_vm_total.delete.ondemand.template')
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      payload = extract_templates_from_query_params(params[:template])
-
-      if payload
-        invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
-        if invalid.empty?
-          result = generate_ondemand_request(payload)
-        else
-          result[:bad_templates] = invalid
-          invalid.each do |bad_template|
-            metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('ondemandrequest_fail.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/ondemandvm/:requestid/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.get.ondemand.request')
-
-      status 404
-      result = check_ondemand_request(params[:requestid])
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/ondemandvm/:requestid/?" do
-      content_type :json
-      need_token! if Vmpooler::API.settings.config[:auth]
-      metrics.increment('http_requests_vm_total.delete.ondemand.request')
-
-      status 404
-      result = delete_ondemand_request(params[:requestid])
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/?" do
-      content_type :json
-      result = { 'ok' => false }
-      metrics.increment('http_requests_vm_total.post.vm.checkout')
-
-      payload = JSON.parse(request.body.read)
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    def extract_templates_from_query_params(params)
-      payload = {}
-
-      params.split('+').each do |template|
-        payload[template] ||= 0
-        payload[template] += 1
-      end
-
-      payload
-    end
-
-    def invalid_templates(payload)
-      invalid = []
-      payload.keys.each do |template|
-        invalid << template unless pool_exists?(template)
-      end
-      invalid
-    end
-
-    def invalid_template_or_size(payload)
-      invalid = []
-      payload.each do |pool, size|
-        invalid << pool unless pool_exists?(pool)
-        unless is_integer?(size)
-          invalid << pool
-          next
-        end
-        invalid << pool unless Integer(size) >= 0
-      end
-      invalid
-    end
-
-    def invalid_template_or_path(payload)
-      invalid = []
-      payload.each do |pool, template|
-        invalid << pool unless pool_exists?(pool)
-        invalid << pool unless template.include? '/'
-        invalid << pool if template[0] == '/'
-        invalid << pool if template[-1] == '/'
-      end
-      invalid
-    end
-
-    def invalid_pool(payload)
-      invalid = []
-      payload.each do |pool, _clone_target|
-        invalid << pool unless pool_exists?(pool)
-      end
-      invalid
-    end
-
-    def check_ondemand_request(request_id)
-      result = { 'ok' => false }
-      request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
-      if request_hash.empty?
-        result['message'] = "no request found for request_id '#{request_id}'"
-        return result
-      end
-
-      result['request_id'] = request_id
-      result['ready'] = false
-      result['ok'] = true
-      status 202
-
-      if request_hash['status'] == 'ready'
-        result['ready'] = true
-        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
-          instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
-
-          if result.key?(platform_alias)
-            result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
-          else
-            result[platform_alias] = { 'hostname': instances }
-          end
-        end
-        result['domain'] = config['domain'] if config['domain']
-        status 200
-      elsif request_hash['status'] == 'failed'
-        result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
-        status 200
-      elsif request_hash['status'] == 'deleted'
-        result['message'] = 'The request has been deleted'
-        status 200
-      else
-        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
-          instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
-          instances_pending = count.to_i - instance_count.to_i
-
-          if result.key?(platform_alias) && result[platform_alias].key?(:ready)
-            result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
-            result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
-          else
-            result[platform_alias] = {
-              'ready': instance_count.to_s,
-              'pending': instances_pending.to_s
-            }
-          end
-        end
-      end
-
-      result
-    end
-
-    def delete_ondemand_request(request_id)
-      result = { 'ok' => false }
-
-      platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
-      unless platforms
-        result['message'] = "no request found for request_id '#{request_id}'"
-        return result
-      end
-
-      if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
-        result['message'] = 'the request has already been deleted'
-      else
-        backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
-
-        Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
-          backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
-            backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
-          end
-          backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
-        end
-        backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
-      end
-      status 200
-      result['ok'] = true
-      result
-    end
-
-    post "#{api_prefix}/vm/:template/?" do
-      content_type :json
-      result = { 'ok' => false }
-      metrics.increment('http_requests_vm_total.get.vm.template')
-
-      payload = extract_templates_from_query_params(params[:template])
-
-      if payload
-        invalid = invalid_templates(payload)
-        if invalid.empty?
-          result = atomically_allocate_vms(payload)
-        else
-          invalid.each do |bad_template|
-            metrics.increment('checkout.invalid.' + bad_template)
-          end
-          status 404
-        end
-      else
-        metrics.increment('checkout.invalid.unknown')
-        status 404
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.get.vm.hostname')
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        status 200
-        result['ok'] = true
-
-        result[params[:hostname]] = {}
-
-        result[params[:hostname]]['template'] = rdata['template']
-        result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
-
-        if rdata['destroy']
-          result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['state'] = 'destroyed'
-        elsif rdata['checkout']
-          result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-          result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
-          result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
-          result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
-          result[params[:hostname]]['state'] = 'running'
-        elsif rdata['check']
-          result[params[:hostname]]['state'] = 'ready'
-        else
-          result[params[:hostname]]['state'] = 'pending'
-        end
-
-        rdata.keys.each do |key|
-          if key.match('^tag\:(.+?)$')
-            result[params[:hostname]]['tags'] ||= {}
-            result[params[:hostname]]['tags'][$1] = rdata[key]
-          end
-
-          if key.match('^snapshot\:(.+?)$')
-            result[params[:hostname]]['snapshots'] ||= []
-            result[params[:hostname]]['snapshots'].push($1)
-          end
-        end
-
-        if rdata['disk']
-          result[params[:hostname]]['disk'] = rdata['disk'].split(':')
-        end
-
-        # Look up IP address of the hostname
-        begin
-          ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
-        rescue StandardError
-          ipAddress = ""
-        end
-
-        result[params[:hostname]]['ip'] = ipAddress
-
-        if config['domain']
-          result[params[:hostname]]['domain'] = config['domain']
-        end
-
-        result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
-        result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
-
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    delete "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.delete.vm.hostname')
-
-      result = {}
-
-      status 404
-      result['ok'] = false
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-      unless rdata.empty?
-        need_token! if rdata['token:token']
-
-        if backend.srem('vmpooler__running__' + rdata['template'], params[:hostname])
-          backend.sadd('vmpooler__completed__' + rdata['template'], params[:hostname])
-
-          status 200
-          result['ok'] = true
-          metrics.increment('delete.success')
-        else
-          metrics.increment('delete.failed')
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    put "#{api_prefix}/vm/:hostname/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.put.vm.modify')
-
-      status 404
-      result = { 'ok' => false }
-
-      failure = []
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists?('vmpooler__vm__' + params[:hostname])
-        begin
-          jdata = JSON.parse(request.body.read)
-        rescue StandardError
-          halt 400, JSON.pretty_generate(result)
-        end
-
-        # Validate data payload
-        jdata.each do |param, arg|
-          case param
-            when 'lifetime'
-              need_token! if Vmpooler::API.settings.config[:auth]
-
-              # in hours, defaults to one week
-              max_lifetime_upper_limit = config['max_lifetime_upper_limit']
-              if max_lifetime_upper_limit
-                max_lifetime_upper_limit = max_lifetime_upper_limit.to_i
-                if arg.to_i >= max_lifetime_upper_limit
-                  failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.")
-                end
-              end
-
-              # validate lifetime is within boundaries
-              unless arg.to_i > 0
-                failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
-              end
-
-            when 'tags'
-              unless arg.is_a?(Hash)
-                failure.push("You provided tags (#{arg}) as something other than a hash.")
-              end
-
-              if config['allowed_tags']
-                failure.push("You provided unsuppored tags (#{arg}).") if not (arg.keys - config['allowed_tags']).empty?
-              end
-            else
-              failure.push("Unknown argument #{arg}.")
-          end
-        end
-
-        if !failure.empty?
-          status 400
-          result['failure'] = failure
-        else
-          jdata.each do |param, arg|
-            case param
-              when 'lifetime'
-                need_token! if Vmpooler::API.settings.config[:auth]
-
-                arg = arg.to_i
-
-                backend.hset('vmpooler__vm__' + params[:hostname], param, arg)
-              when 'tags'
-                filter_tags(arg)
-                export_tags(backend, params[:hostname], arg)
-            end
-          end
-
-          status 200
-          result['ok'] = true
-        end
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/disk/:size/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.post.vm.disksize')
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if ((params[:size].to_i > 0 )and (backend.exists?('vmpooler__vm__' + params[:hostname])))
-        result[params[:hostname]] = {}
-        result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
-
-        backend.sadd('vmpooler__tasks__disk', params[:hostname] + ':' + params[:size])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.post.vm.snapshot')
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      if backend.exists?('vmpooler__vm__' + params[:hostname])
-        result[params[:hostname]] = {}
-
-        o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-        result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
-
-        backend.sadd('vmpooler__tasks__snapshot', params[:hostname] + ':' + result[params[:hostname]]['snapshot'])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
-      content_type :json
-      metrics.increment('http_requests_vm_total.post.vm.disksize')
-
-      need_token! if Vmpooler::API.settings.config[:auth]
-
-      status 404
-      result = { 'ok' => false }
-
-      params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
-
-      unless backend.hget('vmpooler__vm__' + params[:hostname], 'snapshot:' + params[:snapshot]).to_i.zero?
-        backend.sadd('vmpooler__tasks__snapshot-revert', params[:hostname] + ':' + params[:snapshot])
-
-        status 202
-        result['ok'] = true
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/poolsize/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_size(payload)
-          if invalid.empty?
-            result = update_pool_size(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:not_configured] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/pooltemplate/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_template_or_path(payload)
-          if invalid.empty?
-            result = update_pool_template(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/poolreset/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        begin
-          payload = JSON.parse(request.body.read)
-          if payload
-            invalid = invalid_templates(payload)
-            if invalid.empty?
-              result = reset_pool(payload)
-            else
-              invalid.each do |bad_pool|
-                metrics.increment("poolreset.invalid.#{bad_pool}")
-              end
-              result[:bad_pools] = invalid
-              status 400
-            end
-          else
-            metrics.increment('poolreset.invalid.unknown')
-            status 404
-          end
-        rescue JSON::ParserError
-          status 400
-          result = {
-            'ok' => false,
-            'message' => 'JSON payload could not be parsed'
-          }
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    post "#{api_prefix}/config/clonetarget/?" do
-      content_type :json
-      result = { 'ok' => false }
-
-      if config['experimental_features']
-        need_token! if Vmpooler::API.settings.config[:auth]
-
-        payload = JSON.parse(request.body.read)
-
-        if payload
-          invalid = invalid_pool(payload)
-          if invalid.empty?
-            result = update_clone_target(payload)
-          else
-            invalid.each do |bad_template|
-              metrics.increment("config.invalid.#{bad_template}")
-            end
-            result[:bad_templates] = invalid
-            status 400
-          end
-        else
-          metrics.increment('config.invalid.unknown')
-          status 404
-        end
-      else
-        status 405
-      end
-
-      JSON.pretty_generate(result)
-    end
-
-    get "#{api_prefix}/config/?" do
-      content_type :json
-      result = { 'ok' => false }
-      status 404
-
-      if pools
-        sync_pool_sizes
-        sync_pool_templates
-
-        pool_configuration = []
-        pools.each do |pool|
-          pool['template_ready'] = template_ready?(pool, backend)
-          pool_configuration << pool
-        end
-
-        result = {
-          pool_configuration: pool_configuration,
-          status: {
-            ok: true
-          }
-        }
-
-        status 200
-      end
-      JSON.pretty_generate(result)
-    end
-  end
-  end
-end
diff --git a/lib/vmpooler/api/v3.rb b/lib/vmpooler/api/v3.rb
new file mode 100644
index 0000000..21bc4e3
--- /dev/null
+++ b/lib/vmpooler/api/v3.rb
@@ -0,0 +1,1876 @@
+# frozen_string_literal: true
+
+require 'vmpooler/util/parsing'
+require 'vmpooler/dns'
+
+module Vmpooler
+  class API
+    class V3 < Sinatra::Base
+      api_version = '3'
+      api_prefix  = "/api/v#{api_version}"
+
+      # Simple in-memory cache for status endpoint
+      # rubocop:disable Style/ClassVars
+      @@status_cache = {}
+      @@status_cache_mutex = Mutex.new
+      # rubocop:enable Style/ClassVars
+      STATUS_CACHE_TTL = 30 # seconds
+
+      # Clear cache (useful for testing)
+      def self.clear_status_cache
+        @@status_cache_mutex.synchronize do
+          @@status_cache.clear
+        end
+      end
+
+      helpers do
+        include Vmpooler::API::Helpers
+      end
+
+      def backend
+        Vmpooler::API.settings.redis
+      end
+
+      def metrics
+        Vmpooler::API.settings.metrics
+      end
+
+      def config
+        Vmpooler::API.settings.config[:config]
+      end
+
+      def full_config
+        Vmpooler::API.settings.config
+      end
+
+      def pools
+        Vmpooler::API.settings.config[:pools]
+      end
+
+      def pools_at_startup
+        Vmpooler::API.settings.config[:pools_at_startup]
+      end
+
+      def pool_exists?(template)
+        Vmpooler::API.settings.config[:pool_names].include?(template)
+      end
+
+      def need_auth!
+        validate_auth(backend)
+      end
+
+      def need_token!
+        validate_token(backend)
+      end
+
+      def checkoutlock
+        Vmpooler::API.settings.checkoutlock
+      end
+
+      def get_template_aliases(template)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = []
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            result += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+          end
+          result
+        end
+      end
+
+      def get_pool_weights(template_backends)
+        pool_index = pool_index(pools)
+        weighted_pools = {}
+        template_backends.each do |t|
+          next unless pool_index.key? t
+
+          index = pool_index[t]
+          clone_target = pools[index]['clone_target'] || config['clone_target']
+          next unless config.key?('backend_weight')
+
+          weight = config['backend_weight'][clone_target]
+          if weight
+            weighted_pools[t] = weight
+          end
+        end
+        weighted_pools
+      end
+
+      def count_selection(selection)
+        result = {}
+        selection.uniq.each do |poolname|
+          result[poolname] = selection.count(poolname)
+        end
+        result
+      end
+
+      def evaluate_template_aliases(template, count)
+        template_backends = []
+        template_backends << template if backend.sismember('vmpooler__pools', template)
+        selection = []
+        aliases = get_template_aliases(template)
+        if aliases
+          template_backends += aliases
+          weighted_pools = get_pool_weights(template_backends)
+
+          if weighted_pools.count > 1 && weighted_pools.count == template_backends.count
+            pickup = Pickup.new(weighted_pools)
+            count.to_i.times do
+              selection << pickup.pick
+            end
+          else
+            count.to_i.times do
+              selection << template_backends.sample
+            end
+          end
+        end
+
+        count_selection(selection)
+      end
+
+      # Fetch a single vm from a pool
+      #
+      # @param [String] template
+      #   The template that the vm should be created from
+      #
+      # @return [Tuple] vmname, vmpool, vmtemplate
+      #   Returns a tuple containing the vm's name, the pool it came from, and
+      #   what template was used, if successful. Otherwise the tuple contains.
+      #   nil values.
+      def fetch_single_vm(template)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          template_backends = [template]
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            template_backends += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+            pool_index = pool_index(pools)
+            weighted_pools = {}
+            template_backends.each do |t|
+              next unless pool_index.key? t
+
+              index = pool_index[t]
+              clone_target = pools[index]['clone_target'] || config['clone_target']
+              next unless config.key?('backend_weight')
+
+              weight = config['backend_weight'][clone_target]
+              if weight
+                weighted_pools[t] = weight
+              end
+            end
+
+            if weighted_pools.count == template_backends.count
+              pickup = Pickup.new(weighted_pools)
+              selection = pickup.pick
+              template_backends.delete(selection)
+              template_backends.unshift(selection)
+            else
+              first = template_backends.sample
+              template_backends.delete(first)
+              template_backends.unshift(first)
+            end
+          end
+
+          checkoutlock.synchronize do
+            template_backends.each do |template_backend|
+              vms = backend.smembers("vmpooler__ready__#{template_backend}")
+              next if vms.empty?
+
+              vm = vms.pop
+              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+              if smoved
+                return [vm, template_backend, template]
+              end
+            end
+            [nil, nil, nil]
+          end
+        end
+      end
+
+      def return_vm_to_ready_state(template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          backend.srem("vmpooler__migrating__#{template}", vm)
+          backend.hdel("vmpooler__active__#{template}", vm)
+          backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
+          backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
+        end
+      end
+
+      def account_for_starting_vm(template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
+          span.set_attribute('enduser.id', user)
+          has_token_result = has_token?
+          backend.sadd("vmpooler__migrating__#{template}", vm)
+          backend.hset("vmpooler__active__#{template}", vm, Time.now.to_s)
+          backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now.to_s)
+
+          if Vmpooler::API.settings.config[:auth] and has_token_result
+            backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
+            backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
+
+            if config['vm_lifetime_auth'].to_i > 0
+              backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
+            end
+          end
+        end
+      end
+
+      def update_result_hosts(result, template, vm)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result[template] ||= {}
+          if result[template]['hostname']
+            result[template]['hostname'] = Array(result[template]['hostname'])
+            result[template]['hostname'].push(vm)
+          else
+            result[template]['hostname'] = vm
+          end
+        end
+      end
+
+      def atomically_allocate_vms(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          result = { 'ok' => false }
+          failed = false
+          vms = [] # vmpool, vmname, vmtemplate
+
+          validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
+
+          payload.each do |requested, count|
+            count.to_i.times do |_i|
+              vmname, vmpool, vmtemplate = fetch_single_vm(requested)
+              if vmname
+                account_for_starting_vm(vmpool, vmname)
+                vms << [vmpool, vmname, vmtemplate]
+                metrics.increment("checkout.success.#{vmpool}")
+                update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
+              else
+                failed = true
+                metrics.increment("checkout.empty.#{requested}")
+                break
+              end
+            end
+          end
+
+          if failed
+            vms.each do |(vmpool, vmname, _vmtemplate)|
+              return_vm_to_ready_state(vmpool, vmname)
+            end
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.atomically_allocate_vms',
+              'error.message' => '503 due to failing to allocate one or more vms'
+            })
+            status 503
+          else
+            vm_names = []
+            vms.each do |(vmpool, vmname, vmtemplate)|
+              vmdomain = Dns.get_domain_for_pool(full_config, vmpool)
+              vmfqdn = "#{vmname}.#{vmdomain}"
+              update_result_hosts(result, vmtemplate, vmfqdn)
+              vm_names.append(vmfqdn)
+            end
+
+            span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
+
+            result['ok'] = true
+          end
+
+          result
+        end
+      end
+
+      def component_to_test(match, labels_string)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          return if labels_string.nil?
+
+          labels_string_parts = labels_string.split(',')
+          labels_string_parts.each do |part|
+            key, value = part.split('=')
+            next if value.nil?
+            return value if key == match
+          end
+          'none'
+        end
+      end
+
+      def update_user_metrics(operation, vmname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          begin
+            jenkins_build_url = backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
+            user = backend.hget("vmpooler__vm__#{vmname}", 'token:user')
+            poolname = backend.hget("vmpooler__vm__#{vmname}", 'template')
+            poolname = poolname.gsub('.', '_')
+
+            if user
+              user = user.gsub('.', '_')
+            else
+              user = 'unauthenticated'
+            end
+            metrics.increment("user.#{user}.#{operation}.#{poolname}")
+
+            if jenkins_build_url
+              if jenkins_build_url.include? 'litmus'
+                # Very simple filter for Litmus jobs - just count them coming through for the moment.
+                metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
+              else
+                url_parts = jenkins_build_url.split('/')[2..]
+                jenkins_instance = url_parts[0].gsub('.', '_')
+                value_stream_parts = url_parts[2].split('_')
+                value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
+                value_stream = value_stream_parts.shift
+                branch = value_stream_parts.pop
+                project = value_stream_parts.shift
+                job_name = value_stream_parts.join('_')
+                build_metadata_parts = url_parts[3]
+                component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
+
+                metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
+                metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
+                metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
+              end
+            end
+          rescue StandardError => e
+            puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
+            span.record_exception(e)
+            span.status = OpenTelemetry::Trace::Status.error(e.to_s)
+            span.add_event('log', attributes: {
+              'log.severity' => 'debug',
+              'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
+            })
+          end
+        end
+      end
+
+      def reset_pool_size(poolname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+
+          pools_updated = 0
+          sync_pool_sizes
+
+          pool_size_now = pools[pool_index[poolname]]['size'].to_i
+          pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
+          result['pool_size_before_reset'] = pool_size_now
+          result['pool_size_before_overrides'] = pool_size_original
+
+          unless pool_size_now == pool_size_original
+            pools[pool_index[poolname]]['size'] = pool_size_original
+            backend.hdel('vmpooler__config__poolsize', poolname)
+            backend.sadd('vmpooler__pool__undo_size_override', poolname)
+            pools_updated += 1
+            status 201
+          end
+
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_pool_size(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_pool_sizes
+
+          payload.each do |poolname, size|
+            unless pools[pool_index[poolname]]['size'] == size.to_i
+              pools[pool_index[poolname]]['size'] = size.to_i
+              backend.hset('vmpooler__config__poolsize', poolname, size)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def reset_pool_template(poolname)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index_live = pool_index(pools)
+          pool_index_original = pool_index(pools_at_startup)
+
+          pools_updated = 0
+          sync_pool_templates
+
+          template_now = pools[pool_index_live[poolname]]['template']
+          template_original = pools_at_startup[pool_index_original[poolname]]['template']
+          result['template_before_reset'] = template_now
+          result['template_before_overrides'] = template_original
+
+          unless template_now == template_original
+            pools[pool_index_live[poolname]]['template'] = template_original
+            backend.hdel('vmpooler__config__template', poolname)
+            backend.sadd('vmpooler__pool__undo_template_override', poolname)
+            pools_updated += 1
+            status 201
+          end
+
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_pool_template(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_pool_templates
+
+          payload.each do |poolname, template|
+            unless pools[pool_index[poolname]]['template'] == template
+              pools[pool_index[poolname]]['template'] = template
+              backend.hset('vmpooler__config__template', poolname, template)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      def reset_pool(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          payload.each do |poolname, _count|
+            backend.sadd('vmpooler__poolreset', poolname)
+          end
+          status 201
+          result['ok'] = true
+          result
+        end
+      end
+
+      def update_clone_target(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          result = { 'ok' => false }
+
+          pool_index = pool_index(pools)
+          pools_updated = 0
+          sync_clone_targets
+
+          payload.each do |poolname, clone_target|
+            unless pools[pool_index[poolname]]['clone_target'] == clone_target
+              pools[pool_index[poolname]]['clone_target'] = clone_target
+              backend.hset('vmpooler__config__clone_target', poolname, clone_target)
+              pools_updated += 1
+              status 201
+            end
+          end
+          status 200 unless pools_updated > 0
+          result['ok'] = true
+          result
+        end
+      end
+
+      # Cache helper methods for status endpoint
+      def get_cached_status(cache_key)
+        @@status_cache_mutex.synchronize do
+          cached = @@status_cache[cache_key]
+          if cached && (Time.now - cached[:timestamp]) < STATUS_CACHE_TTL
+            return cached[:data]
+          end
+
+          nil
+        end
+      end
+
+      def set_cached_status(cache_key, data)
+        @@status_cache_mutex.synchronize do
+          @@status_cache[cache_key] = {
+            data: data,
+            timestamp: Time.now
+          }
+          # Cleanup old cache entries (keep only last 10 unique view combinations)
+          if @@status_cache.size > 10
+            oldest = @@status_cache.min_by { |_k, v| v[:timestamp] }
+            @@status_cache.delete(oldest[0])
+          end
+        end
+      end
+
+      def sync_pool_templates
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          template_configs = backend.hgetall('vmpooler__config__template')
+          template_configs&.each do |poolname, template|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['template'] = template
+          end
+        end
+      end
+
+      def sync_pool_sizes
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
+          poolsize_configs&.each do |poolname, size|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['size'] = size.to_i
+          end
+        end
+      end
+
+      def sync_clone_targets
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          pool_index = pool_index(pools)
+          clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
+          clone_target_configs&.each do |poolname, clone_target|
+            next unless pool_index.include? poolname
+
+            pools[pool_index[poolname]]['clone_target'] = clone_target
+          end
+        end
+      end
+
+      def too_many_requested?(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          payload&.each do |poolname, count|
+            next unless count.to_i > config['max_ondemand_instances_per_request']
+
+            metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
+            return true
+          end
+          false
+        end
+      end
+
+      def generate_ondemand_request(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          result = { 'ok': false }
+
+          requested_instances = payload.reject { |k, _v| k == 'request_id' }
+          if too_many_requested?(requested_instances)
+            e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+            result['message'] = e_message
+            status 403
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.generate_ondemand_request',
+              'error.message' => "403 due to #{e_message}"
+            })
+            return result
+          end
+
+          score = Time.now.to_i
+          request_id = payload['request_id']
+          request_id ||= generate_request_id
+          result['request_id'] = request_id
+          span.set_attribute('vmpooler.request_id', request_id)
+
+          if backend.exists?("vmpooler__odrequest__#{request_id}")
+            e_message = "request_id '#{request_id}' has already been created"
+            result['message'] = e_message
+            status 409
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.generate_ondemand_request',
+              'error.message' => "409 due to #{e_message}"
+            })
+            metrics.increment('ondemandrequest_generate.duplicaterequests')
+            return result
+          end
+
+          status 201
+
+          platforms_with_aliases = []
+          requested_instances.each do |poolname, count|
+            selection = evaluate_template_aliases(poolname, count)
+            selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+          end
+          platforms_string = platforms_with_aliases.join(',')
+
+          return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+          backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+          if Vmpooler::API.settings.config[:auth] and has_token?
+            token_token = request.env['HTTP_X_AUTH_TOKEN']
+            token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
+            span.set_attribute('enduser.id', token_user)
+          end
+
+          result[:ok] = true
+          metrics.increment('ondemandrequest_generate.success')
+          result
+        end
+      end
+
+      def generate_request_id
+        SecureRandom.uuid
+      end
+
+      get '/' do
+        sync_pool_sizes
+        redirect to('/dashboard/')
+      end
+
+      # Provide run-time statistics
+      #
+      # Example:
+      #
+      # {
+      #   "boot": {
+      #     "duration": {
+      #       "average": 163.6,
+      #       "min": 65.49,
+      #       "max": 830.07,
+      #       "total": 247744.71000000002
+      #     },
+      #     "count": {
+      #       "total": 1514
+      #     }
+      #   },
+      #   "capacity": {
+      #     "current": 968,
+      #     "total": 975,
+      #     "percent": 99.3
+      #   },
+      #   "clone": {
+      #     "duration": {
+      #       "average": 17.0,
+      #       "min": 4.66,
+      #       "max": 637.96,
+      #       "total": 25634.15
+      #     },
+      #     "count": {
+      #       "total": 1507
+      #     }
+      #   },
+      #   "queue": {
+      #     "pending": 12,
+      #     "cloning": 0,
+      #     "booting": 12,
+      #     "ready": 968,
+      #     "running": 367,
+      #     "completed": 0,
+      #     "total": 1347
+      #   },
+      #   "pools": {
+      #     "ready": 100,
+      #     "running": 120,
+      #     "pending": 5,
+      #     "max": 250,
+      #   }
+      #   "status": {
+      #     "ok": true,
+      #     "message": "Battle station fully armed and operational.",
+      #     "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools
+      #       "redhat-7-x86_64",
+      #       "ubuntu-1404-i386"
+      #     ],
+      #     "uptime": 179585.9
+      # }
+      #
+      # If the query parameter 'view' is provided, it will be used to select which top level
+      # element to compute and return. Select them by specifying them in a comma separated list.
+      # For example /status?view=capacity,boot
+      # would return only the "capacity" and "boot" statistics. "status" is always returned
+
+      get "#{api_prefix}/status/?" do
+        content_type :json
+
+        # Create cache key based on view parameters
+        cache_key = params[:view] ? "status_#{params[:view]}" : "status_all"
+
+        # Try to get cached response
+        cached_response = get_cached_status(cache_key)
+        return cached_response if cached_response
+
+        if params[:view]
+          views = params[:view].split(",")
+        end
+
+        result = {
+          status: {
+            ok: true,
+            message: 'Battle station fully armed and operational.'
+          }
+        }
+
+        sync_pool_sizes
+
+        result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
+        result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
+        result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
+        result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot")
+
+        # Check for empty pools
+        result[:pools] = {} unless views and not views.include?("pools")
+        ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+        running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+        pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
+        lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
+
+        unless views and not views.include?("pools")
+          pools.each do |pool|
+            # REMIND: move this out of the API and into the back-end
+            ready    = ready_hash[pool['name']]
+            running  = running_hash[pool['name']]
+            pending  = pending_hash[pool['name']]
+            max      = pool['size']
+            lastBoot = lastBoot_hash[pool['name']]
+            aka      = pool['alias']
+
+            result[:pools][pool['name']] = {
+              ready: ready,
+              running: running,
+              pending: pending,
+              max: max,
+              lastBoot: lastBoot
+            }
+
+            if aka
+              result[:pools][pool['name']][:alias] = aka
+            end
+
+            # for backwards compatibility, include separate "empty" stats in "status" block
+            if ready == 0 && max != 0
+              result[:status][:empty] ||= []
+              result[:status][:empty].push(pool['name'])
+
+              result[:status][:ok] = false
+              result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
+            end
+          end
+        end
+
+        result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
+
+        response = JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+
+        # Cache the response
+        set_cached_status(cache_key, response)
+
+        response
+      end
+
+      # request statistics for specific pools by passing parameter 'pool'
+      # with a coma separated list of pools we want to query ?pool=ABC,DEF
+      # returns the ready, max numbers and the aliases (if set)
+      get "#{api_prefix}/poolstat/?" do
+        content_type :json
+
+        result = {}
+
+        poolscopy = []
+
+        if params[:pool]
+          subpool = params[:pool].split(",")
+          poolscopy = pools.select do |p|
+            if subpool.include?(p['name'])
+              true
+            elsif !p['alias'].nil?
+              if p['alias'].instance_of?(Array)
+                (p['alias'] & subpool).any?
+              elsif p['alias'].instance_of?(String)
+                subpool.include?(p['alias'])
+              end
+            end
+          end
+        end
+
+        result[:pools] = {}
+
+        poolscopy.each do |pool|
+          result[:pools][pool['name']] = {}
+
+          max      = pool['size']
+          aka      = pool['alias']
+
+          result[:pools][pool['name']][:max] = max
+
+          if aka
+            result[:pools][pool['name']][:alias] = aka
+          end
+        end
+
+        ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
+
+        ready_hash.each { |k, v| result[:pools][k][:ready] = v }
+
+        JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+      end
+
+      # requests the total number of running VMs
+      get "#{api_prefix}/totalrunning/?" do
+        content_type :json
+        queue = {
+            running: 0
+        }
+
+        queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+
+        JSON.pretty_generate(queue)
+      end
+
+      get "#{api_prefix}/summary/?" do
+        content_type :json
+
+        result = {
+          daily: []
+        }
+
+        from_param = params[:from] || Date.today.to_s
+        to_param   = params[:to]   || Date.today.to_s
+
+        # Validate date formats
+        [from_param, to_param].each do |param|
+          if !validate_date_str(param.to_s)
+            halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
+          end
+        end
+
+        from_date, to_date = Date.parse(from_param), Date.parse(to_param)
+
+        if to_date < from_date
+          halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
+        elsif from_date > Date.today
+          halt 400, 'Date range is invalid, \'from\' must be in the past.'
+        end
+
+        boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true)
+        clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true)
+        tag = get_tag_summary(backend, from_date, to_date)
+
+        result[:boot] = boot[:boot]
+        result[:clone] = clone[:clone]
+        result[:tag] = tag[:tag]
+
+        daily = {}
+
+        boot[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:boot] = day[:boot]
+        end
+
+        clone[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:clone] = day[:clone]
+        end
+
+        tag[:daily].each do |day|
+          daily[day[:date]] ||= {}
+          daily[day[:date]][:tag] = day[:tag]
+        end
+
+        daily.each_key do |day|
+          result[:daily].push({
+            date: day,
+            boot: daily[day][:boot],
+            clone: daily[day][:clone],
+            tag: daily[day][:tag]
+          })
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/summary/:route/?:key?/?" do
+        content_type :json
+
+        result = {}
+
+        from_param = params[:from] || Date.today.to_s
+        to_param   = params[:to]   || Date.today.to_s
+
+        # Validate date formats
+        [from_param, to_param].each do |param|
+          if !validate_date_str(param.to_s)
+            halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD."
+          end
+        end
+
+        from_date, to_date = Date.parse(from_param), Date.parse(to_param)
+
+        if to_date < from_date
+          halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.'
+        elsif from_date > Date.today
+          halt 400, 'Date range is invalid, \'from\' must be in the past.'
+        end
+
+        case params[:route]
+          when 'boot'
+            result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key])
+          when 'clone'
+            result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key])
+          when 'tag'
+            result = get_tag_summary(backend, from_date, to_date, :only => params[:key])
+          else
+            halt 404, JSON.pretty_generate({ 'ok' => false })
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          backend.keys('vmpooler__token__*').each do |key|
+            data = backend.hgetall(key)
+
+            if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
+              span = OpenTelemetry::Trace.current_span
+              span.set_attribute('enduser.id', data['user'])
+              token = key.split('__').last
+
+              result[token] ||= {}
+
+              result[token]['created'] = data['created']
+              result[token]['last'] = data['last'] || 'never'
+
+              result['ok'] = true
+            end
+          end
+
+          if result['ok']
+            status 200
+          else
+            status 404
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/token/:token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          token = backend.hgetall("vmpooler__token__#{params[:token]}")
+
+          if not token.nil? and not token.empty?
+            status 200
+
+            pools.each do |pool|
+              backend.smembers("vmpooler__running__#{pool['name']}").each do |vm|
+                if backend.hget("vmpooler__vm__#{vm}", 'token:token') == params[:token]
+                  token['vms'] ||= {}
+                  token['vms']['running'] ||= []
+                  token['vms']['running'].push(vm)
+                end
+              end
+            end
+
+            result = { 'ok' => true, params[:token] => token }
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/token/:token/?" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          if backend.del("vmpooler__token__#{params[:token]}").to_i > 0
+            status 200
+            result['ok'] = true
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/token" do
+        content_type :json
+
+        status 404
+        result = { 'ok' => false }
+
+        if Vmpooler::API.settings.config[:auth]
+          status 401
+
+          need_auth!
+
+          o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
+          result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
+
+          backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username)
+          backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now.to_s)
+          span = OpenTelemetry::Trace.current_span
+          span.set_attribute('enduser.id', @auth.username)
+
+          status 200
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/vm/?" do
+        content_type :json
+
+        result = []
+
+        pools.each do |pool|
+          result.push(pool['name'])
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.ondemand.requestid')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        result = { 'ok' => false }
+
+        begin
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+            if invalid.empty?
+              result = generate_ondemand_request(payload)
+            else
+              result[:bad_templates] = invalid
+              invalid.each do |bad_template|
+                metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+              end
+              status 404
+            end
+          else
+            metrics.increment('ondemandrequest_fail.invalid.unknown')
+            status 404
+          end
+        rescue JSON::ParserError
+          span = OpenTelemetry::Trace.current_span
+          span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
+          status 400
+          result = {
+            'ok' => false,
+            'message' => 'JSON payload could not be parsed'
+          }
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.delete.ondemand.template')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest_fail.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/ondemandvm/:requestid/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.ondemand.request')
+
+        status 404
+        result = check_ondemand_request(params[:requestid])
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/ondemandvm/:requestid/?" do
+        content_type :json
+        need_token! if Vmpooler::API.settings.config[:auth]
+        metrics.increment('http_requests_vm_total.delete.ondemand.request')
+
+        status 404
+        result = delete_ondemand_request(params[:requestid])
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.post.vm.checkout')
+
+        # Validate and sanitize JSON body
+        payload = sanitize_json_body(request.body.read)
+        if validation_error?(payload)
+          status 400
+          return JSON.pretty_generate(payload)
+        end
+
+        # Validate each template and count
+        payload.each do |template, count|
+          validation = validate_pool_name(template)
+          if validation_error?(validation)
+            status 400
+            return JSON.pretty_generate(validation)
+          end
+
+          validated_count = validate_vm_count(count)
+          if validation_error?(validated_count)
+            status 400
+            return JSON.pretty_generate(validated_count)
+          end
+        end
+
+        if payload && !payload.empty?
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      def extract_templates_from_query_params(params)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          payload = {}
+
+          params.split('+').each do |template|
+            payload[template] ||= 0
+            payload[template] += 1
+          end
+
+          payload
+        end
+      end
+
+      def invalid_templates(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.keys.each do |template|
+            invalid << template unless pool_exists?(template)
+          end
+          invalid
+        end
+      end
+
+      def invalid_template_or_size(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, size|
+            invalid << pool unless pool_exists?(pool)
+            unless is_integer?(size)
+              invalid << pool
+              next
+            end
+            invalid << pool unless Integer(size) >= 0
+          end
+          invalid
+        end
+      end
+
+      def invalid_template_or_path(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, template|
+            invalid << pool unless pool_exists?(pool)
+            invalid << pool unless template.include? '/'
+            invalid << pool if template[0] == '/'
+            invalid << pool if template[-1] == '/'
+          end
+          invalid
+        end
+      end
+
+      def invalid_pool(payload)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do
+          invalid = []
+          payload.each do |pool, _clone_target|
+            invalid << pool unless pool_exists?(pool)
+          end
+          invalid
+        end
+      end
+
+      def delete_ondemand_request(request_id)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          span.set_attribute('vmpooler.request_id', request_id)
+          result = { 'ok' => false }
+
+          platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
+          unless platforms
+            e_message = "no request found for request_id '#{request_id}'"
+            result['message'] = e_message
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.delete_ondemand_request',
+              'error.message' => e_message
+            })
+            return result
+          end
+
+          if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
+            result['message'] = 'the request has already been deleted'
+          else
+            backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+
+            Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
+              backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
+                backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
+              end
+              backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+            end
+            backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
+          end
+          status 200
+          result['ok'] = true
+          result
+        end
+      end
+
+      post "#{api_prefix}/vm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.get.vm.template')
+
+        # Template can contain multiple pools separated by +, so validate after parsing
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.vm.hostname')
+
+        result = {}
+
+        status 404
+        result['ok'] = false
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
+        unless rdata.empty?
+          status 200
+          result['ok'] = true
+
+          result[params[:hostname]] = {}
+
+          result[params[:hostname]]['template'] = rdata['template']
+          result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
+
+          if rdata['destroy']
+            result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout']
+            result[params[:hostname]]['state'] = 'destroyed'
+          elsif rdata['checkout']
+            result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
+            result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
+            result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
+            result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
+            result[params[:hostname]]['state'] = 'running'
+          elsif rdata['check']
+            result[params[:hostname]]['state'] = 'ready'
+          else
+            result[params[:hostname]]['state'] = 'pending'
+          end
+
+          rdata.keys.each do |key|
+            if key.match('^tag\:(.+?)$')
+              result[params[:hostname]]['tags'] ||= {}
+              result[params[:hostname]]['tags'][$1] = rdata[key]
+            end
+
+            if key.match('^snapshot\:(.+?)$')
+              result[params[:hostname]]['snapshots'] ||= []
+              result[params[:hostname]]['snapshots'].push($1)
+            end
+          end
+
+          if rdata['disk']
+            result[params[:hostname]]['disk'] = rdata['disk'].split(':')
+          end
+
+          # Look up IP address of the hostname
+          begin
+            ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
+          rescue StandardError
+            ipAddress = ""
+          end
+
+          result[params[:hostname]]['ip'] = ipAddress
+
+          if rdata['pool']
+            vmdomain = Dns.get_domain_for_pool(full_config, rdata['pool'])
+            if vmdomain
+              result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}"
+            end
+          end
+
+          result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
+          result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
+
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      def check_ondemand_request(request_id)
+        tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
+          span.set_attribute('vmpooler.request_id', request_id)
+          result = { 'ok' => false }
+          request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+          if request_hash.empty?
+            e_message = "no request found for request_id '#{request_id}'"
+            result['message'] = e_message
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V3.check_ondemand_request',
+              'error.message' => e_message
+            })
+            return result
+          end
+
+          result['request_id'] = request_id
+          result['ready'] = false
+          result['ok'] = true
+          status 202
+
+          case request_hash['status']
+          when 'ready'
+            result['ready'] = true
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+              instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              domain = Dns.get_domain_for_pool(full_config, pool)
+              instances.map! { |instance| instance.concat(".#{domain}") }
+
+              if result.key?(platform_alias)
+                result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
+              else
+                result[platform_alias] = { 'hostname': instances }
+              end
+            end
+            status 200
+          when 'failed'
+            result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+            status 200
+          when 'deleted'
+            result['message'] = 'The request has been deleted'
+            status 200
+          else
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+              instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              instances_pending = count.to_i - instance_count.to_i
+
+              if result.key?(platform_alias) && result[platform_alias].key?(:ready)
+                result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
+                result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
+              else
+                result[platform_alias] = {
+                  'ready': instance_count.to_s,
+                  'pending': instances_pending.to_s
+                }
+              end
+            end
+          end
+
+          result
+        end
+      end
+
+      delete "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.delete.vm.hostname')
+
+        result = {}
+
+        status 404
+        result['ok'] = false
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
+        unless rdata.empty?
+          need_token! if rdata['token:token']
+
+          if backend.srem("vmpooler__running__#{rdata['template']}", params[:hostname])
+            backend.sadd("vmpooler__completed__#{rdata['template']}", params[:hostname])
+
+            status 200
+            result['ok'] = true
+            metrics.increment('delete.success')
+            update_user_metrics('destroy', params[:hostname]) if Vmpooler::API.settings.config[:config]['usage_stats']
+          else
+            metrics.increment('delete.failed')
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      put "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.put.vm.modify')
+
+        status 404
+        result = { 'ok' => false }
+
+        failure = []
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          # Validate and sanitize JSON body
+          jdata = sanitize_json_body(request.body.read)
+          if validation_error?(jdata)
+            status 400
+            return JSON.pretty_generate(jdata)
+          end
+
+          # Validate data payload
+          jdata.each do |param, arg|
+            case param
+              when 'lifetime'
+                need_token! if Vmpooler::API.settings.config[:auth]
+
+                # Validate lifetime is a positive integer
+                lifetime_int = arg.to_i
+                if lifetime_int <= 0
+                  failure.push("Lifetime must be a positive integer (got #{arg})")
+                  next
+                end
+
+                # in hours, defaults to one week
+                max_lifetime_upper_limit = config['max_lifetime_upper_limit']
+                if max_lifetime_upper_limit
+                  max_lifetime_upper_limit = max_lifetime_upper_limit.to_i
+                  if arg.to_i >= max_lifetime_upper_limit
+                    failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.")
+                  end
+                end
+
+              when 'tags'
+                failure.push("You provided tags (#{arg}) as something other than a hash.") unless arg.is_a?(Hash)
+
+                # Validate each tag key and value
+                arg.each do |key, value|
+                  tag_validation = validate_tag(key, value)
+                  if validation_error?(tag_validation)
+                    failure.push(tag_validation['error'])
+                  end
+                end
+
+                failure.push("You provided unsuppored tags (#{arg}).") if config['allowed_tags'] && !(arg.keys - config['allowed_tags']).empty?
+              else
+                failure.push("Unknown argument #{arg}.")
+            end
+          end
+
+          if !failure.empty?
+            status 400
+            result['failure'] = failure
+          else
+            jdata.each do |param, arg|
+              case param
+                when 'lifetime'
+                  need_token! if Vmpooler::API.settings.config[:auth]
+
+                  arg = arg.to_i
+
+                  backend.hset("vmpooler__vm__#{params[:hostname]}", param, arg)
+                when 'tags'
+                  filter_tags(arg)
+                  export_tags(backend, params[:hostname], arg)
+              end
+            end
+
+            status 200
+            result['ok'] = true
+          end
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/disk/:size/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.disksize')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        # Validate disk size
+        validated_size = validate_disk_size(params[:size])
+        if validation_error?(validated_size)
+          status 400
+          return JSON.pretty_generate(validated_size)
+        end
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          result[params[:hostname]] = {}
+          result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
+
+          backend.sadd('vmpooler__tasks__disk', "#{params[:hostname]}:#{params[:size]}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/snapshot/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.snapshot')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
+          result[params[:hostname]] = {}
+
+          o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
+          result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join
+
+          backend.sadd('vmpooler__tasks__snapshot', "#{params[:hostname]}:#{result[params[:hostname]]['snapshot']}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.vm.snapshot')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        status 404
+        result = { 'ok' => false }
+
+        params[:hostname] = hostname_shorten(params[:hostname])
+
+        unless backend.hget("vmpooler__vm__#{params[:hostname]}", "snapshot:#{params[:snapshot]}").to_i.zero?
+          backend.sadd('vmpooler__tasks__snapshot-revert', "#{params[:hostname]}:#{params[:snapshot]}")
+
+          status 202
+          result['ok'] = true
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/config/poolsize/:pool/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          if pool_exists?(params[:pool])
+            result = reset_pool_size(params[:pool])
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/poolsize/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_template_or_size(payload)
+            if invalid.empty?
+              result = update_pool_size(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:not_configured] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      delete "#{api_prefix}/config/pooltemplate/:pool/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          if pool_exists?(params[:pool])
+            result = reset_pool_template(params[:pool])
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/pooltemplate/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_template_or_path(payload)
+            if invalid.empty?
+              result = update_pool_template(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:bad_templates] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/poolreset/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          begin
+            payload = JSON.parse(request.body.read)
+            if payload
+              invalid = invalid_templates(payload)
+              if invalid.empty?
+                result = reset_pool(payload)
+              else
+                invalid.each do |bad_pool|
+                  metrics.increment("poolreset.invalid.#{bad_pool}")
+                end
+                result[:bad_pools] = invalid
+                status 400
+              end
+            else
+              metrics.increment('poolreset.invalid.unknown')
+              status 404
+            end
+          rescue JSON::ParserError
+            span = OpenTelemetry::Trace.current_span
+            span.record_exception(e)
+            span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
+            status 400
+            result = {
+              'ok' => false,
+              'message' => 'JSON payload could not be parsed'
+            }
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/config/clonetarget/?" do
+        content_type :json
+        result = { 'ok' => false }
+
+        if config['experimental_features']
+          need_token! if Vmpooler::API.settings.config[:auth]
+
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_pool(payload)
+            if invalid.empty?
+              result = update_clone_target(payload)
+            else
+              invalid.each do |bad_template|
+                metrics.increment("config.invalid.#{bad_template}")
+              end
+              result[:bad_templates] = invalid
+              status 400
+            end
+          else
+            metrics.increment('config.invalid.unknown')
+            status 404
+          end
+        else
+          status 405
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/config/?" do
+        content_type :json
+        result = { 'ok' => false }
+        status 404
+
+        if pools
+          sync_pool_sizes
+          sync_pool_templates
+
+          pool_configuration = []
+          pools.each do |pool|
+            pool['template_ready'] = template_ready?(pool, backend)
+            pool_configuration << pool
+          end
+
+          result = {
+            pool_configuration: pool_configuration,
+            status: {
+              ok: true
+            }
+          }
+
+          status 200
+        end
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/full_config/?" do
+        content_type :json
+
+        result = {
+          full_config: full_config,
+          status: {
+            ok: true
+          }
+        }
+
+        status 200
+        JSON.pretty_generate(result)
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/dns.rb b/lib/vmpooler/dns.rb
new file mode 100644
index 0000000..823fe17
--- /dev/null
+++ b/lib/vmpooler/dns.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Vmpooler
+  class Dns
+    # Load one or more VMPooler DNS plugin gems by name
+    #
+    # @param names [Array<String>] The list of gem names to load
+    def self.load_by_name(names)
+      names = Array(names)
+      instance = new
+      names.map { |name| instance.load_from_gems(name) }.flatten
+    end
+
+    # Returns the plugin class for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns class
+    # @return [String] The plugin class for the specifid dns config
+    def self.get_dns_plugin_class_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      plugin_class = ''
+
+      dns_configs.map do |dns_config_name|
+        plugin_class = config[:dns_configs][dns_config_name]['dns_class'] if dns_config_name.to_s == name
+      end
+
+      plugin_class
+    end
+
+    # Returns the domain for the specified pool
+    #
+    # @param config [String] - the full config structure
+    # @param pool_name [String] - the name of the pool
+    # @return [String] - domain name for pool, which is set via reference to the dns_configs block
+    def self.get_domain_for_pool(config, pool_name)
+      pool = config[:pools].find { |p| p['name'] == pool_name }
+      pool_dns_config = pool['dns_plugin']
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == pool_dns_config
+      end
+    end
+
+    # Returns the plugin domain for the specified dns config by name
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @param name [Symbol] The name of the dns config key to get the dns domain
+    # @return [String] The domain for the specifid dns config
+    def self.get_dns_plugin_domain_by_name(config, name)
+      dns_configs = config[:dns_configs].keys
+      dns_configs.map do |dns_config_name|
+        return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == name
+      end
+    end
+
+    # Returns a list of DNS plugin classes specified in the vmpooler configuration
+    #
+    # @param config [Object] The entire VMPooler config object
+    # @return nil || [Array<String>] A list of DNS plugin classes
+    def self.get_dns_plugin_config_classes(config)
+      return nil unless config[:dns_configs]
+
+      dns_configs = config[:dns_configs].keys
+      dns_plugins = dns_configs.map do |dns_config_name|
+        if config[:dns_configs][dns_config_name] && config[:dns_configs][dns_config_name]['dns_class']
+          config[:dns_configs][dns_config_name]['dns_class'].to_s
+        else
+          dns_config_name.to_s
+        end
+      end.compact.uniq
+
+      # dynamic-dns is not actually a class, it's just used as a value to denote
+      # that dynamic dns is used so no loading or record management is needed
+      dns_plugins.delete('dynamic-dns')
+
+      dns_plugins
+    end
+
+    # Load a single DNS plugin gem by name
+    #
+    # @param name [String] The name of the DNS plugin gem to load
+    # @return [String] The full require path to the specified gem
+    def load_from_gems(name = nil)
+      require_path = "vmpooler/dns/#{name.gsub('-', '/')}"
+      require require_path
+      require_path
+    end
+  end
+end
diff --git a/lib/vmpooler/dns/base.rb b/lib/vmpooler/dns/base.rb
new file mode 100644
index 0000000..61403ef
--- /dev/null
+++ b/lib/vmpooler/dns/base.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class PoolManager
+    class Dns
+      class Base
+        # These defs must be overidden in child classes
+
+        # Helper Methods
+        # Global Logger object
+        attr_reader :logger
+        # Global Metrics object
+        attr_reader :metrics
+        # Provider options passed in during initialization
+        attr_reader :dns_options
+
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          @config = config
+          @logger = logger
+          @metrics = metrics
+          @redis = redis_connection_pool
+          @dns_plugin_name = name
+
+          @dns_options = options
+
+          logger.log('s', "[!] Creating dns plugin '#{name}'")
+        end
+
+        def pool_config(pool_name)
+          # Get the configuration of a specific pool
+          @config[:pools].each do |pool|
+            return pool if pool['name'] == pool_name
+          end
+
+          nil
+        end
+
+        # Returns this dns plugin's configuration
+        #
+        # @returns [Hashtable] This dns plugins's configuration from the config file.  Returns nil if the dns plugin config does not exist
+        def dns_config
+          @config[:dns_configs].each do |dns|
+            # Convert the symbol from the config into a string for comparison
+            return (dns[1].nil? ? {} : dns[1]) if dns[0].to_s == @dns_plugin_name
+          end
+
+          nil
+        end
+
+        def global_config
+          # This entire VM Pooler config
+          @config
+        end
+
+        def name
+          @dns_plugin_name
+        end
+
+        def get_ip(vm_name)
+          @redis.with_metrics do |redis|
+            redis.hget("vmpooler__vm__#{vm_name}", 'ip')
+          end
+        end
+
+        # returns
+        #   Array[String] : Array of pool names this provider services
+        def provided_pools
+          @config[:pools].select { |pool| pool['dns_config'] == name }.map { |pool| pool['name'] }
+        end
+
+        def create_or_replace_record(hostname)
+          raise("#{self.class.name} does not implement create_or_replace_record #{hostname}")
+        end
+
+        def delete_record(hostname)
+          raise("#{self.class.name} does not implement delete_record for #{hostname}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/vmpooler/metrics/dummy_statsd.rb b/lib/vmpooler/metrics/dummy_statsd.rb
index 96f2eaa..ad866b0 100644
--- a/lib/vmpooler/metrics/dummy_statsd.rb
+++ b/lib/vmpooler/metrics/dummy_statsd.rb
@@ -5,9 +5,6 @@ module Vmpooler
     class DummyStatsd < Metrics
       attr_reader :server, :port, :prefix
 
-      def initialize(*)
-      end
-
       def increment(*)
         true
       end
diff --git a/lib/vmpooler/metrics/graphite.rb b/lib/vmpooler/metrics/graphite.rb
index 128805d..88e66e3 100644
--- a/lib/vmpooler/metrics/graphite.rb
+++ b/lib/vmpooler/metrics/graphite.rb
@@ -7,6 +7,7 @@ module Vmpooler
     class Graphite < Metrics
       attr_reader :server, :port, :prefix
 
+      # rubocop:disable Lint/MissingSuper
       def initialize(logger, params = {})
         raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
 
@@ -15,6 +16,7 @@ module Vmpooler
         @prefix = params['prefix'] || 'vmpooler'
         @logger = logger
       end
+      # rubocop:enable Lint/MissingSuper
 
       def increment(label)
         log label, 1
diff --git a/lib/vmpooler/metrics/promstats.rb b/lib/vmpooler/metrics/promstats.rb
index bee432f..d0e1ab9 100644
--- a/lib/vmpooler/metrics/promstats.rb
+++ b/lib/vmpooler/metrics/promstats.rb
@@ -23,6 +23,7 @@ module Vmpooler
       @p_metrics = {}
       @torun = []
 
+      # rubocop:disable Lint/MissingSuper
       def initialize(logger, params = {})
         @prefix = params['prefix'] || 'vmpooler'
         @prometheus_prefix = params['prometheus_prefix'] || 'vmpooler'
@@ -32,6 +33,7 @@ module Vmpooler
         # Setup up prometheus registry and data structures
         @prometheus = Prometheus::Client.registry
       end
+# rubocop:enable Lint/MissingSuper
 
 =begin # rubocop:disable Style/BlockComments
       The Metrics table is used to register metrics and translate/interpret the incoming metrics.
@@ -163,33 +165,33 @@ module Vmpooler
           },
           user: {
             mtype: M_COUNTER,
-            torun: %i[manager],
-            docstring: 'Number of pool instances this user created created',
-            param_labels: %i[user poolname]
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by a user',
+            param_labels: %i[user operation poolname]
           },
           usage_litmus: {
             mtype: M_COUNTER,
-            torun: %i[manager],
-            docstring: 'Pools by Litmus job usage',
-            param_labels: %i[user poolname]
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Litmus jobs',
+            param_labels: %i[user operation poolname]
           },
           usage_jenkins_instance: {
             mtype: M_COUNTER,
-            torun: %i[manager],
-            docstring: 'Pools by Jenkins instance usage',
-            param_labels: %i[jenkins_instance value_stream poolname]
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins instances',
+            param_labels: %i[jenkins_instance value_stream operation poolname]
           },
           usage_branch_project: {
             mtype: M_COUNTER,
-            torun: %i[manager],
-            docstring: 'Pools by branch/project usage',
-            param_labels: %i[branch project poolname]
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins branch/project',
+            param_labels: %i[branch project operation poolname]
           },
           usage_job_component: {
             mtype: M_COUNTER,
-            torun: %i[manager],
-            docstring: 'Pools by job/component usage',
-            param_labels: %i[job_name component_to_test poolname]
+            torun: %i[api],
+            docstring: 'Number of pool instances and the operation performed by Jenkins job/component',
+            param_labels: %i[job_name component_to_test operation poolname]
           },
           checkout: {
             mtype: M_COUNTER,
@@ -327,6 +329,30 @@ module Vmpooler
             buckets: REDIS_CONNECT_BUCKETS,
             docstring: 'vmpooler redis connection wait time',
             param_labels: %i[type provider]
+          },
+          vmpooler_health: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler health check metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_purge: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler purge metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_destroy: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler destroy metrics',
+            param_labels: %i[poolname]
+          },
+          vmpooler_clone: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler clone metrics',
+            param_labels: %i[poolname]
           }
         }
       end
diff --git a/lib/vmpooler/metrics/promstats/collector_middleware.rb b/lib/vmpooler/metrics/promstats/collector_middleware.rb
index 540b1e6..c7e8325 100644
--- a/lib/vmpooler/metrics/promstats/collector_middleware.rb
+++ b/lib/vmpooler/metrics/promstats/collector_middleware.rb
@@ -113,7 +113,7 @@ module Vmpooler
           # Similarly, request IDs are also stripped from the /ondemand path.
           path
             .gsub(%r{/vm/.+$}, '/vm')
-            .gsub(%r{/ondemand/.+$}, '/ondemand')
+            .gsub(%r{/ondemandvm/.+$}, '/ondemandvm')
             .gsub(%r{/token/.+$}, '/token')
             .gsub(%r{/lib/.+$}, '/lib')
             .gsub(%r{/img/.+$}, '/img')
diff --git a/lib/vmpooler/metrics/statsd.rb b/lib/vmpooler/metrics/statsd.rb
index 85b705d..b469c38 100644
--- a/lib/vmpooler/metrics/statsd.rb
+++ b/lib/vmpooler/metrics/statsd.rb
@@ -8,6 +8,7 @@ module Vmpooler
     class Statsd < Metrics
       attr_reader :server, :port, :prefix
 
+      # rubocop:disable Lint/MissingSuper
       def initialize(logger, params = {})
         raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
 
@@ -17,21 +18,22 @@ module Vmpooler
         @server = ::Statsd.new(host, @port)
         @logger = logger
       end
+      # rubocop:enable Lint/MissingSuper
 
       def increment(label)
-        server.increment(prefix + '.' + label)
+        server.increment("#{prefix}.#{label}")
       rescue StandardError => e
         @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
       end
 
       def gauge(label, value)
-        server.gauge(prefix + '.' + label, value)
+        server.gauge("#{prefix}.#{label}", value)
       rescue StandardError => e
         @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
       end
 
       def timing(label, duration)
-        server.timing(prefix + '.' + label, duration)
+        server.timing("#{prefix}.#{label}", duration)
       rescue StandardError => e
         @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
       end
diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
index a7773f1..9c6def6 100644
--- a/lib/vmpooler/pool_manager.rb
+++ b/lib/vmpooler/pool_manager.rb
@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'vmpooler/dns'
 require 'vmpooler/providers'
 require 'vmpooler/util/parsing'
 require 'spicy-proton'
@@ -26,6 +27,9 @@ module Vmpooler
       # VM Provider objects
       $providers = Concurrent::Hash.new
 
+      # VM DNS objects
+      $dns_plugins = Concurrent::Hash.new
+
       # Our thread-tracker object
       $threads = Concurrent::Hash.new
 
@@ -39,6 +43,9 @@ module Vmpooler
 
       # load specified providers from config file
       load_used_providers
+
+      # load specified dns plugins from config file
+      load_used_dns_plugins
     end
 
     def config
@@ -52,7 +59,7 @@ module Vmpooler
         currently_configured_pools = []
         config[:pools].each do |pool|
           currently_configured_pools << pool['name']
-          redis.sadd('vmpooler__pools', pool['name'])
+          redis.sadd('vmpooler__pools', pool['name'].to_s)
           pool_keys = pool.keys
           pool_keys.delete('alias')
           to_set = {}
@@ -60,11 +67,13 @@ module Vmpooler
             to_set[k] = pool[k]
           end
           to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
-          redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
+          to_set['domain'] = Vmpooler::Dns.get_domain_for_pool(config, pool['name'])
+
+          redis.hmset("vmpooler__pool__#{pool['name']}", *to_set.to_a.flatten) unless to_set.empty?
         end
         previously_configured_pools.each do |pool|
           unless currently_configured_pools.include? pool
-            redis.srem('vmpooler__pools', pool)
+            redis.srem('vmpooler__pools', pool.to_s)
             redis.del("vmpooler__pool__#{pool}")
           end
         end
@@ -73,31 +82,31 @@ module Vmpooler
     end
 
     # Check the state of a VM
-    def check_pending_vm(vm, pool, timeout, provider)
+    def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       Thread.new do
         begin
-          _check_pending_vm(vm, pool, timeout, provider)
+          _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
           @redis.with_metrics do |redis|
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
           raise
         end
       end
     end
 
-    def _check_pending_vm(vm, pool, timeout, provider)
+    def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       mutex = vm_mutex(vm)
       return if mutex.locked?
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
           request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
-          if provider.vm_ready?(pool, vm)
+          if provider.vm_ready?(pool, vm, redis)
             move_pending_vm_to_ready(vm, pool, redis, request_id)
           else
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
         end
       end
@@ -105,70 +114,167 @@ module Vmpooler
 
     def remove_nonexistent_vm(vm, pool, redis)
       redis.srem("vmpooler__pending__#{pool}", vm)
+      dns_plugin = get_dns_plugin_class_for_pool(pool)
+      dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool)
+      domain = get_dns_plugin_domain_for_pool(pool)
+      fqdn = "#{vm}.#{domain}"
+      dns_plugin.delete_record(fqdn) unless dns_plugin_class_name == 'dynamic-dns'
       $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
     end
 
-    def fail_pending_vm(vm, pool, timeout, redis, exists = true)
+    def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
       clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
-
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
-      if time_since_clone > timeout
-        if exists
-          request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
-          pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
-          redis.multi
-          redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
-          redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
-          redis.exec
-          $metrics.increment("errors.markedasfailed.#{pool}")
-          $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-        else
+
+      already_timed_out = time_since_clone > timeout
+      timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
+
+      return true if !already_timed_out && !timing_out_soon
+
+      if already_timed_out
+        unless exists
           remove_nonexistent_vm(vm, pool, redis)
+          return true
         end
+        open_socket_error = handle_timed_out_vm(vm, pool, redis)
       end
+
+      redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
+
+      nonexist_warning = if already_timed_out
+                           "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
+                         elsif timing_out_soon
+                           time_remaining = timeout - timeout_notification
+                           open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+                           "[!] [#{pool}] '#{vm}' impending failure in #{time_remaining} minutes with error: #{open_socket_error}"
+                         else
+                           "[!] [#{pool}] '#{vm}' This error is wholly unexpected"
+                         end
+      $logger.log('d', nonexist_warning)
       true
     rescue StandardError => e
       $logger.log('d', "Fail pending VM failed with an error: #{e}")
       false
     end
 
+    def handle_timed_out_vm(vm, pool, redis)
+      request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
+      open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+      retry_count = redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count').to_i if request_id
+
+      # Move to DLQ before moving to completed queue
+      move_to_dlq(vm, pool, 'pending', 'Timeout',
+                  open_socket_error || 'VM timed out during pending phase',
+                  redis, request_id: request_id, pool_alias: pool_alias, retry_count: retry_count)
+
+      clone_error = redis.hget("vmpooler__vm__#{vm}", 'clone_error')
+      clone_error_class = redis.hget("vmpooler__vm__#{vm}", 'clone_error_class')
+      redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
+
+      if request_id
+        ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
+          # Check retry count and max retry limit before retrying
+          retry_count = (redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count') || '0').to_i
+          max_retries = $config[:config]['max_vm_retries'] || 3
+
+          $logger.log('s', "[!] [#{pool}] '#{vm}' checking retry logic: error='#{clone_error}', error_class='#{clone_error_class}', retry_count=#{retry_count}, max_retries=#{max_retries}")
+
+          # Determine if error is likely permanent (configuration issues)
+          permanent_error = permanent_error?(clone_error, clone_error_class)
+          $logger.log('s', "[!] [#{pool}] '#{vm}' permanent_error check result: #{permanent_error}")
+
+          if retry_count < max_retries && !permanent_error
+            # Increment retry count and retry VM creation
+            redis.hset("vmpooler__odrequest__#{request_id}", 'retry_count', retry_count + 1)
+            redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
+            $logger.log('s', "[!] [#{pool}] '#{vm}' failed, retrying (attempt #{retry_count + 1}/#{max_retries})")
+          else
+            # Max retries exceeded or permanent error, mark request as permanently failed
+            failure_reason = if permanent_error
+                               "Configuration error: #{clone_error}"
+                             else
+                               'Max retry attempts exceeded'
+                             end
+            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+            redis.hset("vmpooler__odrequest__#{request_id}", 'failure_reason', failure_reason)
+            $logger.log('s', "[!] [#{pool}] '#{vm}' permanently failed: #{failure_reason}")
+            $metrics.increment("vmpooler_errors.permanently_failed.#{pool}")
+          end
+        end
+      end
+      $metrics.increment("vmpooler_errors.markedasfailed.#{pool}")
+      open_socket_error || clone_error
+    end
+
+    # Determine if an error is likely permanent (configuration issue) vs transient
+    def permanent_error?(error_message, error_class)
+      return false if error_message.nil? || error_class.nil?
+
+      permanent_error_patterns = [
+        /template.*not found/i,
+        /template.*does not exist/i,
+        /invalid.*path/i,
+        /folder.*not found/i,
+        /datastore.*not found/i,
+        /resource pool.*not found/i,
+        /permission.*denied/i,
+        /authentication.*failed/i,
+        /invalid.*credentials/i,
+        /configuration.*error/i
+      ]
+
+      permanent_error_classes = [
+        'ArgumentError',
+        'NoMethodError',
+        'NameError'
+      ]
+
+      # Check error message patterns
+      permanent_error_patterns.any? { |pattern| error_message.match?(pattern) } ||
+        # Check error class types
+        permanent_error_classes.include?(error_class)
+    end
+
     def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
-      clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
+      clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
       finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
 
       if request_id
         ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
-        if ondemandrequest_hash['status'] == 'failed'
+        case ondemandrequest_hash['status']
+        when 'failed'
           move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
           return nil
-        elsif ondemandrequest_hash['status'] == 'deleted'
+        when 'deleted'
           move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
           return nil
         end
         pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
 
-        redis.pipelined do
-          redis.hset("vmpooler__active__#{pool}", vm, Time.now)
-          redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
+        redis.pipelined do |pipeline|
+          pipeline.hset("vmpooler__active__#{pool}", vm, Time.now.to_s)
+          pipeline.hset("vmpooler__vm__#{vm}", 'checkout', Time.now.to_s)
           if ondemandrequest_hash['token:token']
-            redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
-            redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
-            redis.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
           end
-          redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
+          pipeline.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
         end
         move_vm_queue(pool, vm, 'pending', 'running', redis)
         check_ondemand_request_ready(request_id, redis)
       else
-        redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
+        redis.smove("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm)
       end
 
-      redis.pipelined do
-        redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
-        redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
+      redis.pipelined do |pipeline|
+        pipeline.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself?
+        pipeline.hset("vmpooler__vm__#{vm}", 'ready', Time.now.to_s)
 
         # last boot time is displayed in API, and used by alarming script
-        redis.hset('vmpooler__lastboot', pool, Time.now)
+        pipeline.hset('vmpooler__lastboot', pool, Time.now.to_s)
       end
 
       $metrics.timing("time_to_ready_state.#{pool}", finish)
@@ -178,11 +284,20 @@ module Vmpooler
 
     def vm_still_ready?(pool_name, vm_name, provider, redis)
       # Check if the VM is still ready/available
-      return true if provider.vm_ready?(pool_name, vm_name)
+      return true if provider.vm_ready?(pool_name, vm_name, redis)
 
       raise("VM #{vm_name} is not ready")
-    rescue StandardError
-      move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
+    rescue StandardError => e
+      open_socket_error = redis.hget("vmpooler__vm__#{vm_name}", 'open_socket_error')
+      request_id = redis.hget("vmpooler__vm__#{vm_name}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm_name}", 'pool_alias')
+
+      # Move to DLQ before moving to completed queue
+      move_to_dlq(vm_name, pool_name, 'ready', e.class.name,
+                  open_socket_error || 'VM became unreachable in ready queue',
+                  redis, request_id: request_id, pool_alias: pool_alias)
+
+      move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "removed from 'ready' queue. vm unreachable with error: #{open_socket_error}")
     end
 
     def check_ready_vm(vm, pool_name, ttl, provider)
@@ -203,11 +318,11 @@ module Vmpooler
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
-          check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
+          check_stamp = redis.hget("vmpooler__vm__#{vm}", 'check')
           last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
           break if check_stamp && last_checked_too_soon
 
-          redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
+          redis.hset("vmpooler__vm__#{vm}", 'check', Time.now.to_s)
           # Check if the hosts TTL has expired
           # if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
           boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
@@ -217,7 +332,7 @@ module Vmpooler
             boottime = Time.at(0)
           end
           if (Time.now - boottime).to_i > ttl * 60
-            redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
+            redis.smove("vmpooler__ready__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
 
             $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
             return nil
@@ -256,7 +371,7 @@ module Vmpooler
       return if hostname.empty?
       return if hostname == vm
 
-      redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
+      redis.smove("vmpooler__ready__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
       $logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
       true
     end
@@ -280,7 +395,7 @@ module Vmpooler
         catch :stop_checking do
           @redis.with_metrics do |redis|
             # Check that VM is within defined lifetime
-            checkouttime = redis.hget('vmpooler__active__' + pool, vm)
+            checkouttime = redis.hget("vmpooler__active__#{pool}", vm)
             if checkouttime
               time_since_checkout = Time.now - Time.parse(checkouttime)
               running = time_since_checkout / 60 / 60
@@ -293,7 +408,13 @@ module Vmpooler
               move_vm_queue(pool, vm, 'running', 'completed', redis, 'is listed as running, but has no checkouttime data. Removing from running')
             end
 
-            throw :stop_checking if provider.vm_ready?(pool, vm)
+            # tag VM if not tagged yet, this ensures the method is only called once
+            unless redis.hget("vmpooler__vm__#{vm}", 'user_tagged')
+              success = provider.tag_vm_user(pool, vm)
+              redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true') if success
+            end
+
+            throw :stop_checking if provider.vm_ready?(pool, vm, redis)
 
             throw :stop_checking if provider.get_vm(pool, vm)
 
@@ -308,16 +429,76 @@ module Vmpooler
       $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
     end
 
+    # Dead-Letter Queue (DLQ) helper methods
+    def dlq_enabled?
+      $config[:config] && $config[:config]['dlq_enabled'] == true
+    end
+
+    def dlq_ttl
+      ($config[:config] && $config[:config]['dlq_ttl']) || 168 # default 7 days in hours
+    end
+
+    def dlq_max_entries
+      ($config[:config] && $config[:config]['dlq_max_entries']) || 10_000
+    end
+
+    def move_to_dlq(vm, pool, queue_type, error_class, error_message, redis, request_id: nil, pool_alias: nil, retry_count: 0, skip_metrics: false)
+      return unless dlq_enabled?
+
+      dlq_key = "vmpooler__dlq__#{queue_type}"
+      timestamp = Time.now.to_i
+
+      # Build DLQ entry
+      dlq_entry = {
+        'vm' => vm,
+        'pool' => pool,
+        'queue_from' => queue_type,
+        'error_class' => error_class.to_s,
+        'error_message' => error_message.to_s,
+        'failed_at' => Time.now.iso8601,
+        'retry_count' => retry_count,
+        'request_id' => request_id,
+        'pool_alias' => pool_alias
+      }.compact
+
+      # Use sorted set with timestamp as score for easy age-based queries and TTL
+      dlq_entry_json = dlq_entry.to_json
+      redis.zadd(dlq_key, timestamp, "#{vm}:#{timestamp}:#{dlq_entry_json}")
+
+      # Enforce max entries limit by removing oldest entries
+      current_size = redis.zcard(dlq_key)
+      if current_size > dlq_max_entries
+        remove_count = current_size - dlq_max_entries
+        redis.zremrangebyrank(dlq_key, 0, remove_count - 1)
+        $logger.log('d', "[!] [dlq] Trimmed #{remove_count} oldest entries from #{dlq_key}")
+      end
+
+      # Set expiration on the entire DLQ (will be refreshed on next write)
+      ttl_seconds = dlq_ttl * 3600
+      redis.expire(dlq_key, ttl_seconds)
+
+      $metrics.increment("vmpooler_dlq.#{queue_type}.count") unless skip_metrics
+      $logger.log('d', "[!] [dlq] Moved '#{vm}' from '#{queue_type}' queue to DLQ: #{error_message}")
+    rescue StandardError => e
+      $logger.log('s', "[!] [dlq] Failed to move '#{vm}' to DLQ: #{e}")
+    end
+
     # Clone a VM
-    def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
+    def clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil)
       Thread.new do
         begin
-          _clone_vm(pool_name, provider, request_id, pool_alias)
+          _clone_vm(pool_name, provider, dns_plugin, request_id, pool_alias)
         rescue StandardError => e
           if request_id
             $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
             @redis.with_metrics do |redis|
-              redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
+              # Only re-queue if the request wasn't already marked as failed (e.g., by permanent error detection)
+              request_status = redis.hget("vmpooler__odrequest__#{request_id}", 'status')
+              if request_status != 'failed'
+                redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
+              else
+                $logger.log('s', "[!] [#{pool_name}] Request #{request_id} already marked as failed, not re-queueing")
+              end
             end
           else
             $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
@@ -341,7 +522,7 @@ module Vmpooler
         adjective = @name_generator.adjective(max: 14 - noun.length)
         random_name = [adjective, noun].join('-')
         hostname = $config[:config]['prefix'] + random_name
-        available = redis.hlen('vmpooler__vm__' + hostname) == 0
+        available = redis.hlen("vmpooler__vm__#{hostname}") == 0
 
         [hostname, available]
       end
@@ -354,54 +535,63 @@ module Vmpooler
       max_hostname_retries = 3
       while hostname_retries < max_hostname_retries
         hostname, hostname_available = generate_and_check_hostname
-        domain = $config[:config]['domain']
-        dns_ip, dns_available = check_dns_available(hostname, domain)
+        domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name)
+        fqdn = "#{hostname}.#{domain}"
+
+        # skip dns check if the provider is set to skip_dns_check_before_creating_vm
+        provider = get_provider_for_pool(pool_name)
+        if provider && provider.provider_config['skip_dns_check_before_creating_vm']
+          dns_available = true
+        else
+          dns_ip, dns_available = check_dns_available(fqdn)
+        end
+
         break if hostname_available && dns_available
 
         hostname_retries += 1
 
         if !hostname_available
-          $metrics.increment("errors.duplicatehostname.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
+          $metrics.increment("vmpooler_errors.duplicatehostname.#{pool_name}")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
         elsif !dns_available
-          $metrics.increment("errors.staledns.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
+          $metrics.increment("vmpooler_errors.staledns.#{pool_name}")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS")
         end
       end
 
-      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
+      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available
 
       hostname
     end
 
-    def check_dns_available(vm_name, domain = nil)
-      # Query the DNS for the name we want to create and if it already exists, mark it unavailable
-      # This protects against stale DNS records
-      vm_name = "#{vm_name}.#{domain}" if domain
+    # Query the DNS for the name we want to create and if it already exists, mark it unavailable
+    # This protects against stale DNS records
+    def check_dns_available(vm_name)
       begin
         dns_ip = Resolv.getaddress(vm_name)
       rescue Resolv::ResolvError
         # this is the expected case, swallow the error
-        # eg "no address for blah-daisy"
+        # eg "no address for blah-daisy.example.com"
         return ['', true]
       end
       [dns_ip, false]
     end
 
-    def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
+    def _clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil)
       new_vmname = find_unique_hostname(pool_name)
+      pool_domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name)
       mutex = vm_mutex(new_vmname)
       mutex.synchronize do
         @redis.with_metrics do |redis|
-          # Add VM to Redis inventory ('pending' pool)
-          redis.multi
-          redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
-          redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
-          redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) # This value is used to represent the pool.
-          redis.hset('vmpooler__vm__' + new_vmname, 'pool', pool_name)
-          redis.hset('vmpooler__vm__' + new_vmname, 'request_id', request_id) if request_id
-          redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
-          redis.exec
+          redis.multi do |transaction|
+            transaction.sadd("vmpooler__pending__#{pool_name}", new_vmname)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now.to_s)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool.
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain)
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id
+            transaction.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias
+          end
         end
 
         begin
@@ -409,24 +599,72 @@ module Vmpooler
           start = Time.now
           provider.create_vm(pool_name, new_vmname)
           finish = format('%<time>.2f', time: Time.now - start)
-
-          @redis.with_metrics do |redis|
-            redis.pipelined do
-              redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
-              redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
-            end
-          end
           $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
+          $metrics.gauge("vmpooler_clone.#{pool_name}", finish)
+
+          $logger.log('d', "[ ] [#{pool_name}] Obtaining IP for '#{new_vmname}'")
+          ip_start = Time.now
+          ip = provider.get_vm_ip_address(new_vmname, pool_name)
+          ip_finish = format('%<time>.2f', time: Time.now - ip_start)
+
+          raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
+
+          $logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
 
-          $metrics.timing("clone.#{pool_name}", finish)
-        rescue StandardError
           @redis.with_metrics do |redis|
-            redis.pipelined do
-              redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
-              expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
-              redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
+            redis.pipelined do |pipeline|
+              pipeline.hset("vmpooler__clone__#{Date.today}", "#{pool_name}:#{new_vmname}", finish)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_time', finish)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'ip', ip)
             end
           end
+
+          dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool_name)
+          dns_plugin.create_or_replace_record(new_vmname) unless dns_plugin_class_name == 'dynamic-dns'
+        rescue StandardError => e
+          # Store error details for retry decision making
+          @redis.with_metrics do |redis|
+            # Get retry count before moving to DLQ
+            retry_count = 0
+            if request_id
+              ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+              retry_count = ondemandrequest_hash['retry_count'].to_i if ondemandrequest_hash
+            end
+
+            # Move to DLQ before removing from pending queue
+            move_to_dlq(new_vmname, pool_name, 'clone', e.class.name, e.message,
+                        redis, request_id: request_id, pool_alias: pool_alias, retry_count: retry_count)
+
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__pending__#{pool_name}", new_vmname)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_error', e.message)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_error_class', e.class.name)
+              expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
+              pipeline.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
+            end
+
+            # Handle retry logic for on-demand requests
+            if request_id
+              retry_count = (redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count') || '0').to_i
+              max_retries = $config[:config]['max_vm_retries'] || 3
+              is_permanent = permanent_error?(e.message, e.class.name)
+
+              $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' checking immediate failure retry: error='#{e.message}', error_class='#{e.class.name}', retry_count=#{retry_count}, max_retries=#{max_retries}, permanent_error=#{is_permanent}")
+
+              if is_permanent || retry_count >= max_retries
+                reason = is_permanent ? 'permanent error detected' : 'max retries exceeded'
+                $logger.log('s', "[!] [#{pool_name}] Cancelling request #{request_id} due to #{reason}")
+                redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+                redis.zadd('vmpooler__odcreate__task', 0, "#{pool_alias}:#{pool_name}:0:#{request_id}")
+              else
+                # Increment retry count and re-queue for retry
+                redis.hincrby("vmpooler__odrequest__#{request_id}", 'retry_count', 1)
+                $logger.log('s', "[+] [#{pool_name}] Request #{request_id} will be retried (attempt #{retry_count + 1}/#{max_retries})")
+                redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
+              end
+            end
+          end
+          $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed: #{e.class}: #{e.message}")
           raise
         ensure
           @redis.with_metrics do |redis|
@@ -438,10 +676,10 @@ module Vmpooler
     end
 
     # Destroy a VM
-    def destroy_vm(vm, pool, provider)
+    def destroy_vm(vm, pool, provider, dns_plugin)
       Thread.new do
         begin
-          _destroy_vm(vm, pool, provider)
+          _destroy_vm(vm, pool, provider, dns_plugin)
         rescue StandardError => e
           $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{e}")
           raise
@@ -449,98 +687,48 @@ module Vmpooler
       end
     end
 
-    def _destroy_vm(vm, pool, provider)
+    def _destroy_vm(vm, pool, provider, dns_plugin)
       mutex = vm_mutex(vm)
       return if mutex.locked?
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
-          redis.pipelined do
-            redis.hdel('vmpooler__active__' + pool, vm)
-            redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
+          redis.pipelined do |pipeline|
+            pipeline.hdel("vmpooler__active__#{pool}", vm)
+            pipeline.hset("vmpooler__vm__#{vm}", 'destroy', Time.now.to_s)
 
             # Auto-expire metadata key
-            redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+            pipeline.expire("vmpooler__vm__#{vm}", ($config[:redis]['data_ttl'].to_i * 60 * 60))
           end
 
           start = Time.now
 
           provider.destroy_vm(pool, vm)
+          domain = get_dns_plugin_domain_for_pool(pool)
+          fqdn = "#{vm}.#{domain}"
 
-          redis.srem('vmpooler__completed__' + pool, vm)
+          dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool)
+          dns_plugin.delete_record(fqdn) unless dns_plugin_class_name == 'dynamic-dns'
+
+          redis.srem("vmpooler__completed__#{pool}", vm)
 
           finish = format('%<time>.2f', time: Time.now - start)
           $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
-          $metrics.timing("destroy.#{pool}", finish)
-          get_vm_usage_labels(vm, redis)
+          $metrics.gauge("vmpooler_destroy.#{pool}", finish)
         end
       end
       dereference_mutex(vm)
     end
 
-    def get_vm_usage_labels(vm, redis)
-      return unless $config[:config]['usage_stats']
-
-      redis.multi
-      redis.hget("vmpooler__vm__#{vm}", 'checkout')
-      redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
-      redis.hget("vmpooler__vm__#{vm}", 'token:user')
-      redis.hget("vmpooler__vm__#{vm}", 'template')
-      checkout, jenkins_build_url, user, poolname = redis.exec
-      return if checkout.nil?
-
-      user ||= 'unauthenticated'
-      user = user.gsub('.', '_')
-      $metrics.increment("user.#{user}.#{poolname}")
-
-      return unless jenkins_build_url
-
-      if jenkins_build_url.include? 'litmus'
-        # Very simple filter for Litmus jobs - just count them coming through for the moment.
-        $metrics.increment("usage_litmus.#{user}.#{poolname}")
-        return
-      end
-
-      url_parts = jenkins_build_url.split('/')[2..-1]
-      jenkins_instance = url_parts[0].gsub('.', '_')
-      value_stream_parts = url_parts[2].split('_')
-      value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
-      value_stream = value_stream_parts.shift
-      branch = value_stream_parts.pop
-      project = value_stream_parts.shift
-      job_name = value_stream_parts.join('_')
-      build_metadata_parts = url_parts[3]
-      component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
-
-      $metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}")
-      $metrics.increment("usage_branch_project.#{branch}.#{project}.#{poolname}")
-      $metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{poolname}")
-    rescue StandardError => e
-      $logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
-      raise
-    end
-
-    def component_to_test(match, labels_string)
-      return if labels_string.nil?
-
-      labels_string_parts = labels_string.split(',')
-      labels_string_parts.each do |part|
-        key, value = part.split('=')
-        next if value.nil?
-        return value if key == match
-      end
-      'none'
-    end
-
-    def purge_unused_vms_and_folders
-      global_purge = $config[:config]['purge_unconfigured_folders']
+    def purge_unused_vms_and_resources
+      global_purge = $config[:config]['purge_unconfigured_resources']
       providers = $config[:providers].keys
       providers.each do |provider_key|
-        provider_purge = $config[:providers][provider_key]['purge_unconfigured_folders'] || global_purge
+        provider_purge = $config[:providers][provider_key]['purge_unconfigured_resources'] || global_purge
         if provider_purge
           Thread.new do
             begin
-              purge_vms_and_folders(provider_key)
+              purge_vms_and_resources(provider_key)
             rescue StandardError => e
               $logger.log('s', "[!] failed while purging provider #{provider_key} VMs and folders with an error: #{e}")
             end
@@ -550,33 +738,553 @@ module Vmpooler
       nil
     end
 
-    # Return a list of pool folders
-    def pool_folders(provider_name)
-      folders = {}
-      $config[:pools].each do |pool|
-        next unless pool['provider'] == provider_name.to_s
-
-        folder_parts = pool['folder'].split('/')
-        datacenter = $providers[provider_name.to_s].get_target_datacenter_from_config(pool['name'])
-        folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
-      end
-      folders
-    end
-
-    def get_base_folders(folders)
-      base = []
-      folders.each do |_key, value|
-        base << value
-      end
-      base.uniq
-    end
-
-    def purge_vms_and_folders(provider_name)
+    def purge_vms_and_resources(provider_name)
       provider = $providers[provider_name.to_s]
-      configured_folders = pool_folders(provider_name)
-      base_folders = get_base_folders(configured_folders)
-      whitelist = provider.provider_config['folder_whitelist']
-      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      # Deprecated, will be removed in version 3
+      if provider.provider_config['folder_whitelist']
+        $logger.log('d', "[!] [deprecation] rename configuration 'folder_whitelist' to 'resources_allowlist' for provider #{provider_name}")
+        allowlist = provider.provider_config['folder_whitelist']
+      else
+        allowlist = provider.provider_config['resources_allowlist']
+      end
+      provider.purge_unconfigured_resources(allowlist)
+    end
+
+    # Auto-purge stale queue entries
+    def purge_enabled?
+      $config[:config] && $config[:config]['purge_enabled'] == true
+    end
+
+    def purge_dry_run?
+      $config[:config] && $config[:config]['purge_dry_run'] == true
+    end
+
+    def max_pending_age
+      ($config[:config] && $config[:config]['max_pending_age']) || 7200 # default 2 hours in seconds
+    end
+
+    def max_ready_age
+      ($config[:config] && $config[:config]['max_ready_age']) || 86_400 # default 24 hours in seconds
+    end
+
+    def max_completed_age
+      ($config[:config] && $config[:config]['max_completed_age']) || 3600 # default 1 hour in seconds
+    end
+
+    def max_orphaned_age
+      ($config[:config] && $config[:config]['max_orphaned_age']) || 86_400 # default 24 hours in seconds
+    end
+
+    def purge_stale_queue_entries
+      return unless purge_enabled?
+
+      Thread.new do
+        begin
+          $logger.log('d', '[*] [purge] Starting stale queue entry purge cycle')
+          purge_start = Time.now
+
+          @redis.with_metrics do |redis|
+            total_purged = 0
+
+            # Purge stale entries from each pool
+            $config[:pools].each do |pool|
+              pool_name = pool['name']
+
+              # Purge pending queue
+              purged_pending = purge_pending_queue(pool_name, redis)
+              total_purged += purged_pending
+
+              # Purge ready queue
+              purged_ready = purge_ready_queue(pool_name, redis)
+              total_purged += purged_ready
+
+              # Purge completed queue
+              purged_completed = purge_completed_queue(pool_name, redis)
+              total_purged += purged_completed
+            end
+
+            # Purge orphaned VM metadata
+            purged_orphaned = purge_orphaned_metadata(redis)
+            total_purged += purged_orphaned
+
+            purge_duration = Time.now - purge_start
+            $logger.log('s', "[*] [purge] Completed purge cycle in #{purge_duration.round(2)}s: #{total_purged} entries purged")
+            $metrics.gauge('vmpooler_purge.cycle.duration', purge_duration)
+            $metrics.gauge('vmpooler_purge.total.count', total_purged)
+          end
+        rescue StandardError => e
+          $logger.log('s', "[!] [purge] Failed during purge cycle: #{e}")
+        end
+      end
+    end
+
+    def purge_pending_queue(pool_name, redis)
+      queue_key = "vmpooler__pending__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          clone_time_str = redis.hget("vmpooler__vm__#{vm}", 'clone')
+          next unless clone_time_str
+
+          clone_time = Time.parse(clone_time_str)
+          age = Time.now - clone_time
+
+          if age > max_pending_age
+            request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+            pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
+
+            purged_count += 1
+
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale pending VM '#{vm}' (age: #{age.round(0)}s, max: #{max_pending_age}s)")
+            else
+              # Move to DLQ before removing (skip DLQ metric since we're tracking purge metric)
+              move_to_dlq(vm, pool_name, 'pending', 'Purge',
+                          "Stale pending VM (age: #{age.round(0)}s > max: #{max_pending_age}s)",
+                          redis, request_id: request_id, pool_alias: pool_alias, skip_metrics: true)
+
+              redis.srem(queue_key, vm)
+
+              # Set expiration on VM metadata if data_ttl is configured
+              if $config[:redis] && $config[:redis]['data_ttl']
+                expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
+                redis.expire("vmpooler__vm__#{vm}", expiration_ttl)
+              end
+
+              $logger.log('d', "[!] [purge] Purged stale pending VM '#{vm}' from '#{pool_name}' (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.pending.#{pool_name}.count")
+            end
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking pending VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_ready_queue(pool_name, redis)
+      queue_key = "vmpooler__ready__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          ready_time_str = redis.hget("vmpooler__vm__#{vm}", 'ready')
+          next unless ready_time_str
+
+          ready_time = Time.parse(ready_time_str)
+          age = Time.now - ready_time
+
+          if age > max_ready_age
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale ready VM '#{vm}' (age: #{age.round(0)}s, max: #{max_ready_age}s)")
+            else
+              redis.smove(queue_key, "vmpooler__completed__#{pool_name}", vm)
+              $logger.log('d', "[!] [purge] Moved stale ready VM '#{vm}' from '#{pool_name}' to completed (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.ready.#{pool_name}.count")
+            end
+            purged_count += 1
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking ready VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_completed_queue(pool_name, redis)
+      queue_key = "vmpooler__completed__#{pool_name}"
+      vms = redis.smembers(queue_key)
+      purged_count = 0
+
+      vms.each do |vm|
+        begin
+          # Check destroy time or last activity time
+          destroy_time_str = redis.hget("vmpooler__vm__#{vm}", 'destroy')
+          checkout_time_str = redis.hget("vmpooler__vm__#{vm}", 'checkout')
+
+          # Use the most recent timestamp
+          timestamp_str = destroy_time_str || checkout_time_str
+          next unless timestamp_str
+
+          timestamp = Time.parse(timestamp_str)
+          age = Time.now - timestamp
+
+          if age > max_completed_age
+            if purge_dry_run?
+              $logger.log('d', "[*] [purge][dry-run] Would purge stale completed VM '#{vm}' (age: #{age.round(0)}s, max: #{max_completed_age}s)")
+            else
+              redis.srem(queue_key, vm)
+              $logger.log('d', "[!] [purge] Removed stale completed VM '#{vm}' from '#{pool_name}' (age: #{age.round(0)}s)")
+              $metrics.increment("vmpooler_purge.completed.#{pool_name}.count")
+            end
+            purged_count += 1
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking completed VM '#{vm}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    def purge_orphaned_metadata(redis)
+      # Find VM metadata that doesn't belong to any queue
+      all_vm_keys = redis.keys('vmpooler__vm__*')
+      purged_count = 0
+
+      all_vm_keys.each do |vm_key|
+        begin
+          vm = vm_key.sub('vmpooler__vm__', '')
+
+          # Check if VM exists in any queue
+          pool_name = redis.hget(vm_key, 'pool')
+          next unless pool_name
+
+          in_pending = redis.sismember("vmpooler__pending__#{pool_name}", vm)
+          in_ready = redis.sismember("vmpooler__ready__#{pool_name}", vm)
+          in_running = redis.sismember("vmpooler__running__#{pool_name}", vm)
+          in_completed = redis.sismember("vmpooler__completed__#{pool_name}", vm)
+          in_discovered = redis.sismember("vmpooler__discovered__#{pool_name}", vm)
+          in_migrating = redis.sismember("vmpooler__migrating__#{pool_name}", vm)
+
+          # VM is orphaned if not in any queue
+          unless in_pending || in_ready || in_running || in_completed || in_discovered || in_migrating
+            # Check age
+            clone_time_str = redis.hget(vm_key, 'clone')
+            next unless clone_time_str
+
+            clone_time = Time.parse(clone_time_str)
+            age = Time.now - clone_time
+
+            if age > max_orphaned_age
+              if purge_dry_run?
+                $logger.log('d', "[*] [purge][dry-run] Would purge orphaned metadata for '#{vm}' (age: #{age.round(0)}s, max: #{max_orphaned_age}s)")
+              else
+                expiration_ttl = 3600 # 1 hour
+                redis.expire(vm_key, expiration_ttl)
+                $logger.log('d', "[!] [purge] Set expiration on orphaned metadata for '#{vm}' (age: #{age.round(0)}s)")
+                $metrics.increment('vmpooler_purge.orphaned.count')
+              end
+              purged_count += 1
+            end
+          end
+        rescue StandardError => e
+          $logger.log('d', "[!] [purge] Error checking orphaned metadata '#{vm_key}': #{e}")
+        end
+      end
+
+      purged_count
+    end
+
+    # Health checks for Redis queues
+    def health_check_enabled?
+      $config[:config] && $config[:config]['health_check_enabled'] == true
+    end
+
+    def health_thresholds
+      defaults = {
+        'pending_queue_max' => 100,
+        'ready_queue_max' => 500,
+        'dlq_max_warning' => 100,
+        'dlq_max_critical' => 1000,
+        'stuck_vm_age_threshold' => 7200, # 2 hours
+        'stuck_vm_max_warning' => 10,
+        'stuck_vm_max_critical' => 50
+      }
+
+      if $config[:config] && $config[:config]['health_thresholds']
+        defaults.merge($config[:config]['health_thresholds'])
+      else
+        defaults
+      end
+    end
+
+    def check_queue_health
+      return unless health_check_enabled?
+
+      Thread.new do
+        begin
+          $logger.log('d', '[*] [health] Running queue health check')
+          health_start = Time.now
+
+          @redis.with_metrics do |redis|
+            health_metrics = calculate_health_metrics(redis)
+            health_status = determine_health_status(health_metrics)
+
+            # Store health metrics in Redis for API consumption
+            # Convert nested hash to JSON for storage
+            require 'json'
+            redis.hset('vmpooler__health', 'metrics', health_metrics.to_json)
+            redis.hset('vmpooler__health', 'status', health_status)
+            redis.hset('vmpooler__health', 'last_check', Time.now.iso8601)
+            redis.expire('vmpooler__health', 3600) # Expire after 1 hour
+
+            # Log health summary
+            log_health_summary(health_metrics, health_status)
+
+            # Push metrics
+            push_health_metrics(health_metrics, health_status)
+
+            health_duration = Time.now - health_start
+            $metrics.gauge('vmpooler_health.check.duration', health_duration)
+          end
+        rescue StandardError => e
+          $logger.log('s', "[!] [health] Failed during health check: #{e}")
+        end
+      end
+    end
+
+    def calculate_health_metrics(redis)
+      metrics = {
+        'queues' => {},
+        'tasks' => {},
+        'errors' => {}
+      }
+
+      total_stuck_vms = 0
+      total_dlq_size = 0
+      thresholds = health_thresholds
+
+      # Check each pool's queues
+      $config[:pools].each do |pool|
+        pool_name = pool['name']
+        metrics['queues'][pool_name] = {}
+
+        # Pending queue metrics
+        pending_key = "vmpooler__pending__#{pool_name}"
+        pending_vms = redis.smembers(pending_key)
+        pending_ages = calculate_queue_ages(pending_vms, 'clone', redis)
+        stuck_pending = pending_ages.count { |age| age > thresholds['stuck_vm_age_threshold'] }
+        total_stuck_vms += stuck_pending
+
+        metrics['queues'][pool_name]['pending'] = {
+          'size' => pending_vms.size,
+          'oldest_age' => pending_ages.max || 0,
+          'avg_age' => pending_ages.empty? ? 0 : (pending_ages.sum / pending_ages.size).round(0),
+          'stuck_count' => stuck_pending
+        }
+
+        # Ready queue metrics
+        ready_key = "vmpooler__ready__#{pool_name}"
+        ready_vms = redis.smembers(ready_key)
+        ready_ages = calculate_queue_ages(ready_vms, 'ready', redis)
+
+        metrics['queues'][pool_name]['ready'] = {
+          'size' => ready_vms.size,
+          'oldest_age' => ready_ages.max || 0,
+          'avg_age' => ready_ages.empty? ? 0 : (ready_ages.sum / ready_ages.size).round(0)
+        }
+
+        # Completed queue metrics
+        completed_key = "vmpooler__completed__#{pool_name}"
+        completed_size = redis.scard(completed_key)
+        metrics['queues'][pool_name]['completed'] = { 'size' => completed_size }
+      end
+
+      # Task queue metrics
+      clone_active = redis.get('vmpooler__tasks__clone').to_i
+      ondemand_active = redis.get('vmpooler__tasks__ondemandclone').to_i
+      odcreate_pending = redis.zcard('vmpooler__odcreate__task')
+
+      metrics['tasks']['clone'] = { 'active' => clone_active }
+      metrics['tasks']['ondemand'] = { 'active' => ondemand_active, 'pending' => odcreate_pending }
+
+      # DLQ metrics
+      if dlq_enabled?
+        dlq_keys = redis.keys('vmpooler__dlq__*')
+        dlq_keys.each do |dlq_key|
+          queue_type = dlq_key.sub('vmpooler__dlq__', '')
+          dlq_size = redis.zcard(dlq_key)
+          total_dlq_size += dlq_size
+          metrics['queues']['dlq'] ||= {}
+          metrics['queues']['dlq'][queue_type] = { 'size' => dlq_size }
+        end
+      end
+
+      # Error metrics
+      metrics['errors']['dlq_total_size'] = total_dlq_size
+      metrics['errors']['stuck_vm_count'] = total_stuck_vms
+
+      # Orphaned metadata count
+      orphaned_count = count_orphaned_metadata(redis)
+      metrics['errors']['orphaned_metadata_count'] = orphaned_count
+
+      metrics
+    end
+
+    def calculate_queue_ages(vms, timestamp_field, redis)
+      ages = []
+      vms.each do |vm|
+        begin
+          timestamp_str = redis.hget("vmpooler__vm__#{vm}", timestamp_field)
+          next unless timestamp_str
+
+          timestamp = Time.parse(timestamp_str)
+          age = (Time.now - timestamp).to_i
+          ages << age
+        rescue StandardError
+          # Skip VMs with invalid timestamps
+        end
+      end
+      ages
+    end
+
+    def count_orphaned_metadata(redis)
+      all_vm_keys = redis.keys('vmpooler__vm__*')
+      orphaned_count = 0
+
+      all_vm_keys.each do |vm_key|
+        begin
+          vm = vm_key.sub('vmpooler__vm__', '')
+          pool_name = redis.hget(vm_key, 'pool')
+          next unless pool_name
+
+          in_any_queue = redis.sismember("vmpooler__pending__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__ready__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__running__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__completed__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__discovered__#{pool_name}", vm) ||
+                         redis.sismember("vmpooler__migrating__#{pool_name}", vm)
+
+          orphaned_count += 1 unless in_any_queue
+        rescue StandardError
+          # Skip on error
+        end
+      end
+
+      orphaned_count
+    end
+
+    def determine_health_status(metrics)
+      thresholds = health_thresholds
+
+      # Check DLQ size
+      dlq_size = metrics['errors']['dlq_total_size']
+      return 'unhealthy' if dlq_size > thresholds['dlq_max_critical']
+
+      # Check stuck VM count
+      stuck_count = metrics['errors']['stuck_vm_count']
+      return 'unhealthy' if stuck_count > thresholds['stuck_vm_max_critical']
+
+      # Check queue sizes
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        pending_size = begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        ready_size = begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+
+        return 'unhealthy' if pending_size > thresholds['pending_queue_max'] * 2
+        return 'unhealthy' if ready_size > thresholds['ready_queue_max'] * 2
+      end
+
+      # Check for degraded conditions
+      return 'degraded' if dlq_size > thresholds['dlq_max_warning']
+      return 'degraded' if stuck_count > thresholds['stuck_vm_max_warning']
+
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        pending_size = begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        ready_size = begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+
+        return 'degraded' if pending_size > thresholds['pending_queue_max']
+        return 'degraded' if ready_size > thresholds['ready_queue_max']
+      end
+
+      'healthy'
+    end
+
+    def log_health_summary(metrics, status)
+      summary = "[*] [health] Status: #{status.upcase}"
+
+      # Queue summary
+      total_pending = 0
+      total_ready = 0
+      total_completed = 0
+
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        total_pending += begin
+          queues['pending']['size']
+        rescue StandardError
+          0
+        end
+        total_ready += begin
+          queues['ready']['size']
+        rescue StandardError
+          0
+        end
+        total_completed += begin
+          queues['completed']['size']
+        rescue StandardError
+          0
+        end
+      end
+
+      summary += " | Queues: P=#{total_pending} R=#{total_ready} C=#{total_completed}"
+      summary += " | DLQ=#{metrics['errors']['dlq_total_size']}"
+      summary += " | Stuck=#{metrics['errors']['stuck_vm_count']}"
+      summary += " | Orphaned=#{metrics['errors']['orphaned_metadata_count']}"
+
+      log_level = status == 'healthy' ? 's' : 'd'
+      $logger.log(log_level, summary)
+    end
+
+    def push_health_metrics(metrics, status)
+      # Push error metrics first
+      $metrics.gauge('vmpooler_health.dlq.total_size', metrics['errors']['dlq_total_size'])
+      $metrics.gauge('vmpooler_health.stuck_vms.count', metrics['errors']['stuck_vm_count'])
+      $metrics.gauge('vmpooler_health.orphaned_metadata.count', metrics['errors']['orphaned_metadata_count'])
+
+      # Push per-pool queue metrics
+      metrics['queues'].each do |pool_name, queues|
+        next if pool_name == 'dlq'
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.size", queues['pending']['size'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.oldest_age", queues['pending']['oldest_age'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.pending.stuck_count", queues['pending']['stuck_count'])
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.ready.size", queues['ready']['size'])
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.ready.oldest_age", queues['ready']['oldest_age'])
+
+        $metrics.gauge("vmpooler_health.queue.#{pool_name}.completed.size", queues['completed']['size'])
+      end
+
+      # Push DLQ metrics
+      metrics['queues']['dlq']&.each do |queue_type, dlq_metrics|
+        $metrics.gauge("vmpooler_health.dlq.#{queue_type}.size", dlq_metrics['size'])
+      end
+
+      # Push task metrics
+      $metrics.gauge('vmpooler_health.tasks.clone.active', metrics['tasks']['clone']['active'])
+      $metrics.gauge('vmpooler_health.tasks.ondemand.active', metrics['tasks']['ondemand']['active'])
+      $metrics.gauge('vmpooler_health.tasks.ondemand.pending', metrics['tasks']['ondemand']['pending'])
+
+      # Push status last (0=healthy, 1=degraded, 2=unhealthy)
+      status_value = { 'healthy' => 0, 'degraded' => 1, 'unhealthy' => 2 }[status] || 2
+      $metrics.gauge('vmpooler_health.status', status_value)
     end
 
     def create_vm_disk(pool_name, vm, disk_size, provider)
@@ -603,10 +1311,10 @@ module Vmpooler
 
       if result
         @redis.with_metrics do |redis|
-          rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
+          rdisks = redis.hget("vmpooler__vm__#{vm_name}", 'disk')
           disks = rdisks ? rdisks.split(':') : []
           disks.push("+#{disk_size}gb")
-          redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
+          redis.hset("vmpooler__vm__#{vm_name}", 'disk', disks.join(':'))
         end
 
         $logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
@@ -638,7 +1346,7 @@ module Vmpooler
 
       if result
         @redis.with_metrics do |redis|
-          redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
+          redis.hset("vmpooler__vm__#{vm_name}", "snapshot:#{snapshot_name}", Time.now.to_s)
         end
         $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
       else
@@ -676,28 +1384,47 @@ module Vmpooler
       result
     end
 
+    # load only dns plugins used in config file
+    def load_used_dns_plugins
+      dns_plugins = Vmpooler::Dns.get_dns_plugin_config_classes(config)
+      Vmpooler::Dns.load_by_name(dns_plugins)
+    end
+
     # load only providers used in config file
     def load_used_providers
       Vmpooler::Providers.load_by_name(used_providers)
     end
 
     # @return [Array] - a list of used providers from the config file, defaults to the default providers
-    # ie. ["vsphere", "dummy"]
+    # ie. ["dummy"]
     def used_providers
-      pools = config[:pools] || []
-      @used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers).uniq
+      # create an array of provider classes based on the config
+      if config[:providers]
+        config_provider_names = config[:providers].keys
+        config_providers = config_provider_names.map do |config_provider_name|
+          if config[:providers][config_provider_name] && config[:providers][config_provider_name]['provider_class']
+            config[:providers][config_provider_name]['provider_class'].to_s
+          else
+            config_provider_name.to_s
+          end
+        end.compact.uniq
+      else
+        config_providers = []
+      end
+      # return the unique array of providers from the config and VMPooler defaults
+      @used_providers ||= (config_providers + default_providers).uniq
     end
 
     # @return [Array] - returns a list of providers that should always be loaded
     # note: vsphere is the default if user does not specify although this should not be
     # if vsphere is to no longer be loaded by default please remove
     def default_providers
-      @default_providers ||= %w[vsphere dummy]
+      @default_providers ||= %w[dummy]
     end
 
     def get_pool_name_for_vm(vm_name, redis)
       # the 'template' is a bad name.  Should really be 'poolname'
-      redis.hget('vmpooler__vm__' + vm_name, 'template')
+      redis.hget("vmpooler__vm__#{vm_name}", 'template')
     end
 
     # @param pool_name [String] - the name of the pool
@@ -710,6 +1437,31 @@ module Vmpooler
       $providers[provider_name]
     end
 
+    def get_dns_plugin_class_name_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      Vmpooler::Dns.get_dns_plugin_class_by_name(config, plugin_name)
+    end
+
+    def get_dns_plugin_class_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      plugin_class = Vmpooler::Dns.get_dns_plugin_class_by_name(config, plugin_name)
+      $dns_plugins[plugin_class]
+    end
+
+    def get_dns_plugin_domain_for_pool(pool_name)
+      pool = $config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      plugin_name = pool.fetch('dns_plugin')
+      Vmpooler::Dns.get_dns_plugin_domain_by_name(config, plugin_name)
+    end
+
     def check_disk_queue(maxloop = 0, loop_delay = 5)
       $logger.log('d', '[*] [disk_manager] starting worker thread')
 
@@ -838,16 +1590,21 @@ module Vmpooler
     #   - Fires when a pool reset is requested
     #   - Additional options
     #       :poolname
+    # :undo_override
+    #   - Fires when a pool override removal is requested
+    #   - Additional options
+    #       :poolname
     #
     def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {})
       exit_by = Time.now + loop_delay
       wakeup_by = Time.now + wakeup_period
+
       return if time_passed?(:exit_by, exit_by)
 
       @redis.with_metrics do |redis|
         initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
 
-        initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
+        initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target].to_s) if options[:clone_target_change]
 
         initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
 
@@ -867,16 +1624,12 @@ module Vmpooler
 
             if options[:clone_target_change]
               clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
-              if clone_target
-                break unless clone_target == initial_clone_target
-              end
+              break if clone_target && clone_target != initial_clone_target
             end
 
             if options[:pool_template_change]
               configured_template = redis.hget('vmpooler__config__template', options[:poolname])
-              if configured_template
-                break unless initial_template == configured_template
-              end
+              break if configured_template && initial_template != configured_template
             end
 
             if options[:pool_reset]
@@ -884,17 +1637,21 @@ module Vmpooler
               break if pending
             end
 
+            if options[:undo_override]
+              break if redis.sismember('vmpooler__pool__undo_template_override', options[:poolname])
+              break if redis.sismember('vmpooler__pool__undo_size_override', options[:poolname])
+            end
+
             if options[:pending_vm]
               pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
               break unless pending_vm_count == 0
             end
 
             if options[:ondemand_request]
-              redis.multi
-              redis.zcard('vmpooler__provisioning__request')
-              redis.zcard('vmpooler__provisioning__processing')
-              redis.zcard('vmpooler__odcreate__task')
-              od_request, od_processing, od_createtask = redis.exec
+              od_request = redis.zcard('vmpooler__provisioning__request')
+              od_processing = redis.zcard('vmpooler__provisioning__processing')
+              od_createtask = redis.zcard('vmpooler__odcreate__task')
+
               break unless od_request == 0
               break unless od_processing == 0
               break unless od_createtask == 0
@@ -926,11 +1683,16 @@ module Vmpooler
           loop_count = 1
           loop_delay = loop_delay_min
           provider = get_provider_for_pool(pool['name'])
-          raise("Could not find provider '#{pool['provider']}") if provider.nil?
+          raise("Could not find provider '#{pool['provider']}'") if provider.nil?
 
           sync_pool_template(pool)
           loop do
+            start_time = Time.now
             result = _check_pool(pool, provider)
+            duration = Time.now - start_time
+
+            $metrics.gauge("vmpooler_performance.check_pool.#{pool['name']}", duration)
+            $logger.log('d', "[!] check_pool for #{pool['name']} took #{duration.round(2)}s") if duration > 5
 
             if result[:cloned_vms] > 0 || result[:checked_pending_vms] > 0 || result[:discovered_vms] > 0
               loop_delay = loop_delay_min
@@ -938,7 +1700,7 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true)
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true, undo_override: true)
 
             unless maxloop == 0
               break if loop_count >= maxloop
@@ -970,21 +1732,17 @@ module Vmpooler
     def sync_pool_template(pool)
       @redis.with_metrics do |redis|
         pool_template = redis.hget('vmpooler__config__template', pool['name'])
-        if pool_template
-          pool['template'] = pool_template unless pool['template'] == pool_template
-        end
+        pool['template'] = pool_template if pool_template && pool['template'] != pool_template
       end
     end
 
     def prepare_template(pool, provider, redis)
-      if $config[:config]['create_template_delta_disks']
-        unless redis.sismember('vmpooler__template__deltas', pool['template'])
-          begin
-            provider.create_template_delta_disks(pool)
-            redis.sadd('vmpooler__template__deltas', pool['template'])
-          rescue StandardError => e
-            $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
-          end
+      if $config[:config]['create_template_delta_disks'] && !redis.sismember('vmpooler__template__deltas', pool['template'])
+        begin
+          provider.create_template_delta_disks(pool)
+          redis.sadd('vmpooler__template__deltas', pool['template'])
+        rescue StandardError => e
+          $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
         end
       end
       redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
@@ -1070,10 +1828,8 @@ module Vmpooler
 
     def remove_excess_vms(pool)
       @redis.with_metrics do |redis|
-        redis.multi
-        redis.scard("vmpooler__ready__#{pool['name']}")
-        redis.scard("vmpooler__pending__#{pool['name']}")
-        ready, pending = redis.exec
+        ready = redis.scard("vmpooler__ready__#{pool['name']}")
+        pending = redis.scard("vmpooler__pending__#{pool['name']}")
         total = pending.to_i + ready.to_i
         break if total.nil?
         break if total == 0
@@ -1102,15 +1858,18 @@ module Vmpooler
       return if mutex.locked?
 
       @redis.with_metrics do |redis|
-        poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
-        break if poolsize.nil?
+        pool_size_requested = redis.hget('vmpooler__config__poolsize', pool['name'])
+        break if pool_size_requested.nil?
 
-        poolsize = Integer(poolsize)
-        break if poolsize == pool['size']
+        pool_size_requested = Integer(pool_size_requested)
+        pool_size_currently = pool['size']
+        break if pool_size_requested == pool_size_currently
 
         mutex.synchronize do
-          pool['size'] = poolsize
+          pool['size'] = pool_size_requested
         end
+
+        $logger.log('s', "[*] [#{pool['name']}] size updated from #{pool_size_currently} to #{pool_size_requested}")
       end
     end
 
@@ -1128,6 +1887,38 @@ module Vmpooler
       end
     end
 
+    def undo_override(pool, provider)
+      poolname = pool['name']
+      mutex = pool_mutex(poolname)
+      return if mutex.locked?
+
+      @redis.with_metrics do |redis|
+        break unless redis.sismember('vmpooler__pool__undo_template_override', poolname)
+
+        redis.srem('vmpooler__pool__undo_template_override', poolname)
+        template_now = pool['template']
+        template_original = $config[:pools_at_startup][$config[:pool_index][poolname]]['template']
+
+        mutex.synchronize do
+          update_pool_template(pool, provider, template_original, template_now, redis)
+        end
+      end
+
+      @redis.with_metrics do |redis|
+        break unless redis.sismember('vmpooler__pool__undo_size_override', poolname)
+
+        redis.srem('vmpooler__pool__undo_size_override', poolname)
+        pool_size_now = pool['size']
+        pool_size_original = $config[:pools_at_startup][$config[:pool_index][poolname]]['size']
+
+        mutex.synchronize do
+          pool['size'] = pool_size_original
+        end
+
+        $logger.log('s', "[*] [#{poolname}] size updated from #{pool_size_now} to #{pool_size_original}")
+      end
+    end
+
     def create_inventory(pool, provider, pool_check_response)
       inventory = {}
       begin
@@ -1135,15 +1926,15 @@ module Vmpooler
         mutex.synchronize do
           @redis.with_metrics do |redis|
             provider.vms_in_pool(pool['name']).each do |vm|
-              if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
-                 !redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
-                 !redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
-                 !redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
-                 !redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
-                 !redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
+              if !redis.sismember("vmpooler__running__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__ready__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__pending__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__completed__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__discovered__#{pool['name']}", vm['name']) &&
+                 !redis.sismember("vmpooler__migrating__#{pool['name']}", vm['name'])
 
                 pool_check_response[:discovered_vms] += 1
-                redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
+                redis.sadd("vmpooler__discovered__#{pool['name']}", vm['name'])
 
                 $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
               end
@@ -1164,7 +1955,7 @@ module Vmpooler
         redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
           if inventory[vm]
             begin
-              vm_lifetime = redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
+              vm_lifetime = redis.hget("vmpooler__vm__#{vm}", 'lifetime') || $config[:config]['vm_lifetime'] || 12
               pool_check_response[:checked_running_vms] += 1
               check_running_vm(vm, pool_name, vm_lifetime, provider)
             rescue StandardError => e
@@ -1194,19 +1985,20 @@ module Vmpooler
       end
     end
 
-    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
+    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
       pool_timeout ||= $config[:config]['timeout'] || 15
+      pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
       @redis.with_metrics do |redis|
         redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
           if inventory[vm]
             begin
               pool_check_response[:checked_pending_vms] += 1
-              check_pending_vm(vm, pool_name, pool_timeout, provider)
+              check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
             rescue StandardError => e
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
             end
           else
-            fail_pending_vm(vm, pool_name, pool_timeout, redis, false)
+            fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
           end
         end
       end
@@ -1218,21 +2010,22 @@ module Vmpooler
           if inventory[vm]
             begin
               pool_check_response[:destroyed_vms] += 1
-              destroy_vm(vm, pool_name, provider)
+              dns_plugin = get_dns_plugin_class_for_pool(pool_name)
+              destroy_vm(vm, pool_name, provider, dns_plugin)
             rescue StandardError => e
-              redis.pipelined do
-                redis.srem("vmpooler__completed__#{pool_name}", vm)
-                redis.hdel("vmpooler__active__#{pool_name}", vm)
-                redis.del("vmpooler__vm__#{vm}")
+              redis.pipelined do |pipeline|
+                pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+                pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+                pipeline.del("vmpooler__vm__#{vm}")
               end
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
             end
           else
             $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
-            redis.pipelined do
-              redis.srem("vmpooler__completed__#{pool_name}", vm)
-              redis.hdel("vmpooler__active__#{pool_name}", vm)
-              redis.del("vmpooler__vm__#{vm}")
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+              pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+              pipeline.del("vmpooler__vm__#{vm}")
             end
           end
         end
@@ -1275,16 +2068,17 @@ module Vmpooler
       return if pool_mutex(pool_name).locked?
 
       @redis.with_metrics do |redis|
-        redis.multi
-        redis.scard("vmpooler__ready__#{pool_name}")
-        redis.scard("vmpooler__pending__#{pool_name}")
-        redis.scard("vmpooler__running__#{pool_name}")
-        ready, pending, running = redis.exec
+        ready = redis.scard("vmpooler__ready__#{pool_name}")
+        pending = redis.scard("vmpooler__pending__#{pool_name}")
+        running = redis.scard("vmpooler__running__#{pool_name}")
+
         total = pending.to_i + ready.to_i
 
         $metrics.gauge("ready.#{pool_name}", ready)
         $metrics.gauge("running.#{pool_name}", running)
 
+        dns_plugin = get_dns_plugin_class_for_pool(pool_name)
+
         unless pool_size == 0
           if redis.get("vmpooler__empty__#{pool_name}")
             redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
@@ -1299,7 +2093,7 @@ module Vmpooler
             begin
               redis.incr('vmpooler__tasks__clone')
               pool_check_response[:cloned_vms] += 1
-              clone_vm(pool_name, provider)
+              clone_vm(pool_name, provider, dns_plugin)
             rescue StandardError => e
               $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
               redis.decr('vmpooler__tasks__clone')
@@ -1331,7 +2125,7 @@ module Vmpooler
 
       check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
 
-      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
+      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
 
       check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
 
@@ -1362,6 +2156,9 @@ module Vmpooler
       # Reset a pool when poolreset is requested from the API
       reset_pool(pool)
 
+      # Undo overrides submitted via the api
+      undo_override(pool, provider)
+
       pool_check_response
     end
 
@@ -1379,7 +2176,19 @@ module Vmpooler
 
         return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
       end
-      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
+      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil?
+    end
+
+    def create_dns_object(config, logger, metrics, redis_connection_pool, dns_class, dns_name, options)
+      if defined?(Vmpooler::PoolManager::Dns)
+        dns_klass = Vmpooler::PoolManager::Dns
+        dns_klass.constants.each do |classname|
+          next unless classname.to_s.casecmp(dns_class) == 0
+
+          return dns_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, dns_name, options)
+        end
+        raise("DNS '#{dns_class}' is unknown for pool with dns name '#{dns_name}'") if dns_klass.nil?
+      end
     end
 
     def check_ondemand_requests(maxloop = 0,
@@ -1442,15 +2251,24 @@ module Vmpooler
         redis.zrem('vmpooler__provisioning__request', request_id)
         return
       end
+
+      # Check if request was already marked as failed (e.g., by delete endpoint)
+      request_status = redis.hget("vmpooler__odrequest__#{request_id}", 'status')
+      if request_status == 'failed'
+        $logger.log('s', "Request '#{request_id}' already marked as failed, skipping VM creation")
+        redis.zrem('vmpooler__provisioning__request', request_id)
+        return
+      end
+
       score = redis.zscore('vmpooler__provisioning__request', request_id)
       requested = requested.split(',')
 
-      redis.pipelined do
+      redis.pipelined do |pipeline|
         requested.each do |request|
-          redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
+          pipeline.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
         end
-        redis.zrem('vmpooler__provisioning__request', request_id)
-        redis.zadd('vmpooler__provisioning__processing', score, request_id)
+        pipeline.zrem('vmpooler__provisioning__request', request_id)
+        pipeline.zadd('vmpooler__provisioning__processing', score, request_id)
       end
     end
 
@@ -1465,24 +2283,25 @@ module Vmpooler
         pool_alias, pool, count, request_id = request.split(':')
         count = count.to_i
         provider = get_provider_for_pool(pool)
+        dns_plugin = get_dns_plugin_class_for_pool(pool)
         slots = ondemand_clone_limit - clone_count
         break if slots == 0
 
         if slots >= count
           count.times do
             redis.incr('vmpooler__tasks__ondemandclone')
-            clone_vm(pool, provider, request_id, pool_alias)
+            clone_vm(pool, provider, dns_plugin, request_id, pool_alias)
           end
           redis.zrem(queue_key, request)
         else
           remaining_count = count - slots
           slots.times do
             redis.incr('vmpooler__tasks__ondemandclone')
-            clone_vm(pool, provider, request_id, pool_alias)
+            clone_vm(pool, provider, dns_plugin, request_id, pool_alias)
           end
-          redis.pipelined do
-            redis.zrem(queue_key, request)
-            redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
+          redis.pipelined do |pipeline|
+            pipeline.zrem(queue_key, request)
+            pipeline.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
           end
         end
       end
@@ -1519,11 +2338,9 @@ module Vmpooler
 
       return unless vms_ready?(request_id, redis)
 
-      redis.multi
       redis.hset(ondemand_hash_key, 'status', 'ready')
       redis.expire(ondemand_hash_key, default_expiration)
       redis.zrem(processing_key, request_id)
-      redis.exec
     end
 
     def request_expired?(request_id, score, redis)
@@ -1533,10 +2350,10 @@ module Vmpooler
 
       $logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
       expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
-      redis.pipelined do
-        redis.zrem('vmpooler__provisioning__processing', request_id)
-        redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
-        redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
+      redis.pipelined do |pipeline|
+        pipeline.zrem('vmpooler__provisioning__processing', request_id)
+        pipeline.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+        pipeline.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
       end
       remove_vms_for_failed_request(request_id, expiration_ttl, redis)
       true
@@ -1546,11 +2363,11 @@ module Vmpooler
       request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
       Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
         pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
-        redis.pipelined do
+        redis.pipelined do |pipeline|
           pools_filled&.each do |vm|
-            move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
+            move_vm_queue(pool, vm, 'running', 'completed', pipeline, "moved to completed queue. '#{request_id}' could not be filled in time")
           end
-          redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+          pipeline.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
         end
       end
     end
@@ -1576,8 +2393,8 @@ module Vmpooler
       # Set default provider for all pools that do not have one defined
       $config[:pools].each do |pool|
         if pool['provider'].nil?
-          $logger.log('d', "[!] Setting provider for pool '#{pool['name']}' to 'vsphere' as default")
-          pool['provider'] = 'vsphere'
+          $logger.log('d', "[!] Setting provider for pool '#{pool['name']}' to 'dummy' as default")
+          pool['provider'] = 'dummy'
         end
       end
 
@@ -1593,35 +2410,46 @@ module Vmpooler
       # Create the providers
       $config[:pools].each do |pool|
         provider_name = pool['provider']
+        dns_plugin_name = pool['dns_plugin']
         # The provider_class parameter can be defined in the provider's data eg
-        #:providers:
-        # :vsphere:
-        #  provider_class: 'vsphere'
-        # :another-vsphere:
-        #  provider_class: 'vsphere'
+        # :providers:
+        #   :vsphere:
+        #     provider_class: 'vsphere'
+        #   :another-vsphere:
+        #     provider_class: 'vsphere'
         # the above would create two providers/vsphere.rb class objects named 'vsphere' and 'another-vsphere'
         # each pools would then define which provider definition to use: vsphere or another-vsphere
         #
         # if provider_class is not defined it will try to use the provider_name as the class, this is to be
         # backwards compatible for example when there is only one provider listed
         # :providers:
-        #  :dummy:
-        #   filename: 'db.txs'
+        #   :dummy:
+        #     filename: 'db.txs'
         # the above example would create an object based on the class providers/dummy.rb
         if $config[:providers].nil? || $config[:providers][provider_name.to_sym].nil? || $config[:providers][provider_name.to_sym]['provider_class'].nil?
           provider_class = provider_name
         else
           provider_class = $config[:providers][provider_name.to_sym]['provider_class']
         end
+
         begin
           $providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
         rescue StandardError => e
           $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
           raise
         end
+
+        dns_plugin_class = $config[:dns_configs][dns_plugin_name.to_sym]['dns_class']
+
+        begin
+          $dns_plugins[dns_plugin_class] = create_dns_object($config, $logger, $metrics, @redis, dns_plugin_class, dns_plugin_name, {}) if $dns_plugins[dns_plugin_class].nil?
+        rescue StandardError => e
+          $logger.log('s', "Error while creating dns plugin for pool #{pool['name']}: #{e}")
+          raise
+        end
       end
 
-      purge_unused_vms_and_folders
+      purge_unused_vms_and_resources
 
       loop_count = 1
       loop do
@@ -1655,6 +2483,48 @@ module Vmpooler
           check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
         end
 
+        # Queue purge thread
+        if purge_enabled?
+          purge_interval = ($config[:config] && $config[:config]['purge_interval']) || 3600 # default 1 hour
+          if !$threads['queue_purge']
+            $threads['queue_purge'] = Thread.new do
+              loop do
+                purge_stale_queue_entries
+                sleep(purge_interval)
+              end
+            end
+          elsif !$threads['queue_purge'].alive?
+            $logger.log('d', '[!] [queue_purge] worker thread died, restarting')
+            $threads['queue_purge'] = Thread.new do
+              loop do
+                purge_stale_queue_entries
+                sleep(purge_interval)
+              end
+            end
+          end
+        end
+
+        # Health check thread
+        if health_check_enabled?
+          health_interval = ($config[:config] && $config[:config]['health_check_interval']) || 300 # default 5 minutes
+          if !$threads['health_check']
+            $threads['health_check'] = Thread.new do
+              loop do
+                check_queue_health
+                sleep(health_interval)
+              end
+            end
+          elsif !$threads['health_check'].alive?
+            $logger.log('d', '[!] [health_check] worker thread died, restarting')
+            $threads['health_check'] = Thread.new do
+              loop do
+                check_queue_health
+                sleep(health_interval)
+              end
+            end
+          end
+        end
+
         sleep(loop_delay)
 
         unless maxloop == 0
diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb
index 635dfc7..e5d458e 100644
--- a/lib/vmpooler/providers/base.rb
+++ b/lib/vmpooler/providers/base.rb
@@ -58,6 +58,10 @@ module Vmpooler
           nil
         end
 
+        def dns_config(dns_config_name)
+          Vmpooler::Dns.get_dns_plugin_domain_by_name(@config, dns_config_name)
+        end
+
         # returns
         #   [Hashtable] : The entire VMPooler configuration
         def global_config
@@ -212,6 +216,22 @@ module Vmpooler
           raise("#{self.class.name} does not implement vm_ready?")
         end
 
+        # tag_vm_user This method is called once we know who is using the VM (it is running). This method enables seeing
+        # who is using what in the provider pools.
+        # This method should be implemented in the providers, if it is not implemented, this base method will be called
+        # and should be a noop. The implementation should check if the vm has a user (as per redis) and add a new tag
+        # with the information.
+        # inputs
+        #   [String] pool_name : Name of the pool
+        #   [String] vm_name   : Name of the VM to check if ready
+        # returns
+        #   [Boolean] : true if successful, false if an error occurred and it should retry
+        def tag_vm_user(_pool_name, _vm_name)
+          # noop by design. If the provider does not implement this method, this base method is called (because inherited)
+          # and should basically do nothing.
+          true
+        end
+
         # inputs
         #   [String] pool_name : Name of the pool
         #   [String] vm_name   : Name of the VM to check if it exists
@@ -226,7 +246,7 @@ module Vmpooler
         # returns
         #   nil when successful. Raises error when encountered
         def create_template_delta_disks(_pool)
-          raise("#{self.class.name} does not implement create_template_delta_disks")
+          puts("#{self.class.name} does not implement create_template_delta_disks")
         end
 
         # inputs
@@ -237,8 +257,19 @@ module Vmpooler
           raise("#{self.class.name} does not implement get_target_datacenter_from_config")
         end
 
-        def purge_unconfigured_folders(_base_folders, _configured_folders, _whitelist)
-          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        def purge_unconfigured_resources(_allowlist)
+          raise("#{self.class.name} does not implement purge_unconfigured_resources")
+        end
+
+        def get_vm_ip_address(vm_name, pool_name)
+          raise("#{self.class.name} does not implement get_vm_ip_address for vm #{vm_name} in pool #{pool_name}")
+        end
+
+        # DEPRECATED if a provider does not implement the new method, it will hit this base class method
+        # and return a deprecation message
+        def purge_unconfigured_folders(_deprecated, _deprecated2, allowlist)
+          logger.log('s', '[!] purge_unconfigured_folders was renamed to purge_unconfigured_resources, please update your provider implementation')
+          purge_unconfigured_resources(allowlist)
         end
       end
     end
diff --git a/lib/vmpooler/providers/dummy.rb b/lib/vmpooler/providers/dummy.rb
index c4ea7cc..0a1a60d 100644
--- a/lib/vmpooler/providers/dummy.rb
+++ b/lib/vmpooler/providers/dummy.rb
@@ -79,7 +79,7 @@ module Vmpooler
           return current_vm['vm_host'] if rand(1..100) > provider_config['migratevm_couldmove_percent']
 
           # Simulate a 10 node cluster and randomly pick a different one
-          new_host = 'HOST' + rand(1..10).to_s while new_host == current_vm['vm_host']
+          new_host = "HOST#{rand(1..10)}" while new_host == current_vm['vm_host']
 
           new_host
         end
@@ -95,9 +95,7 @@ module Vmpooler
             end
 
             # Inject clone failure
-            unless provider_config['migratevm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if rand(1..100) <= provider_config['migratevm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['migratevm_fail_percent'].nil? && rand(1..100) <= provider_config['migratevm_fail_percent']
 
             @write_lock.synchronize do
               current_vm = get_dummy_vm(pool_name, vm_name)
@@ -116,27 +114,23 @@ module Vmpooler
             return nil if dummy.nil?
 
             # Randomly power off the VM
-            unless dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?
-              if rand(1..100) <= provider_config['getvm_poweroff_percent']
-                @write_lock.synchronize do
-                  dummy = get_dummy_vm(pool_name, vm_name)
-                  dummy['powerstate'] = 'PoweredOff'
-                  write_backing_file
-                end
-                logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
+            if !(dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil?) && rand(1..100) <= provider_config['getvm_poweroff_percent']
+              @write_lock.synchronize do
+                dummy = get_dummy_vm(pool_name, vm_name)
+                dummy['powerstate'] = 'PoweredOff'
+                write_backing_file
               end
+              logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
             end
 
             # Randomly rename the host
-            unless dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?
-              if rand(1..100) <= provider_config['getvm_rename_percent']
-                @write_lock.synchronize do
-                  dummy = get_dummy_vm(pool_name, vm_name)
-                  dummy['hostname'] = 'DUMMY' + dummy['name']
-                  write_backing_file
-                end
-                logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
+            if !(dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil?) && rand(1..100) <= provider_config['getvm_rename_percent']
+              @write_lock.synchronize do
+                dummy = get_dummy_vm(pool_name, vm_name)
+                dummy['hostname'] = "DUMMY#{dummy['name']}"
+                write_backing_file
               end
+              logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
             end
 
             obj['name'] = dummy['name']
@@ -196,9 +190,7 @@ module Vmpooler
 
             begin
               # Inject clone failure
-              unless provider_config['createvm_fail_percent'].nil?
-                raise('Dummy Failure for createvm_fail_percent') if rand(1..100) <= provider_config['createvm_fail_percent']
-              end
+              raise('Dummy Failure for createvm_fail_percent') if !provider_config['createvm_fail_percent'].nil? && rand(1..100) <= provider_config['createvm_fail_percent']
 
               # Assert the VM is ready for use
               @write_lock.synchronize do
@@ -229,9 +221,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createdisk_fail_percent'].nil?
-              raise('Dummy Failure for createdisk_fail_percent') if rand(1..100) <= provider_config['createdisk_fail_percent']
-            end
+            raise('Dummy Failure for createdisk_fail_percent') if !provider_config['createdisk_fail_percent'].nil? && rand(1..100) <= provider_config['createdisk_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -255,9 +245,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['createsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for createsnapshot_fail_percent') if rand(1..100) <= provider_config['createsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for createsnapshot_fail_percent') if !provider_config['createsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['createsnapshot_fail_percent']
 
             @write_lock.synchronize do
               vm_object = get_dummy_vm(pool_name, vm_name)
@@ -282,9 +270,7 @@ module Vmpooler
             end
 
             # Inject create failure
-            unless provider_config['revertsnapshot_fail_percent'].nil?
-              raise('Dummy Failure for revertsnapshot_fail_percent') if rand(1..100) <= provider_config['revertsnapshot_fail_percent']
-            end
+            raise('Dummy Failure for revertsnapshot_fail_percent') if !provider_config['revertsnapshot_fail_percent'].nil? && rand(1..100) <= provider_config['revertsnapshot_fail_percent']
           end
 
           vm_object['snapshots'].include?(snapshot_name)
@@ -320,9 +306,7 @@ module Vmpooler
             end
 
             # Inject destroy VM failure
-            unless provider_config['destroyvm_fail_percent'].nil?
-              raise('Dummy Failure for migratevm_fail_percent') if rand(1..100) <= provider_config['destroyvm_fail_percent']
-            end
+            raise('Dummy Failure for migratevm_fail_percent') if !provider_config['destroyvm_fail_percent'].nil? && rand(1..100) <= provider_config['destroyvm_fail_percent']
 
             # 'Destroy' the VM
             @write_lock.synchronize do
@@ -354,9 +338,7 @@ module Vmpooler
             # it's ready to receive a connection
             sleep(2)
 
-            unless provider_config['vmready_fail_percent'].nil?
-              raise('Dummy Failure for vmready_fail_percent') if rand(1..100) <= provider_config['vmready_fail_percent']
-            end
+            raise('Dummy Failure for vmready_fail_percent') if !provider_config['vmready_fail_percent'].nil? && rand(1..100) <= provider_config['vmready_fail_percent']
 
             @write_lock.synchronize do
               vm_object['ready'] = true
diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb
deleted file mode 100644
index 63373ec..0000000
--- a/lib/vmpooler/providers/vsphere.rb
+++ /dev/null
@@ -1,1162 +0,0 @@
-# frozen_string_literal: true
-
-require 'vmpooler/providers/base'
-
-module Vmpooler
-  class PoolManager
-    class Provider
-      class VSphere < Vmpooler::PoolManager::Provider::Base
-        # The connection_pool method is normally used only for testing
-        attr_reader :connection_pool
-
-        def initialize(config, logger, metrics, redis_connection_pool, name, options)
-          super(config, logger, metrics, redis_connection_pool, name, options)
-
-          task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i
-          # The default connection pool size is:
-          # Whatever is biggest from:
-          #   - How many pools this provider services
-          #   - Maximum number of cloning tasks allowed
-          #   - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc.
-          default_connpool_size = [provided_pools.count, task_limit, 2].max
-          connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i
-          # The default connection pool timeout should be quite large - 60 seconds
-          connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i
-          logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
-          @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
-            metrics: metrics,
-            connpool_type: 'provider_connection_pool',
-            connpool_provider: name,
-            size: connpool_size,
-            timeout: connpool_timeout
-          ) do
-            logger.log('d', "[#{name}] Connection Pool - Creating a connection object")
-            # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve
-            # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection
-            # object.  Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the
-            # Hash can change, and is preserved across invocations.
-            new_conn = connect_to_vsphere
-            { connection: new_conn }
-          end
-          @provider_hosts = {}
-          @provider_hosts_lock = Mutex.new
-          @redis = redis_connection_pool
-        end
-
-        # name of the provider class
-        def name
-          'vsphere'
-        end
-
-        def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
-          if whitelist
-            return true if whitelist.include?(folder_title)
-          end
-          return false unless configured_folders.keys.include?(folder_title)
-          return false unless configured_folders[folder_title] == base_folder
-
-          true
-        end
-
-        def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
-          try = 0 if try.nil?
-          max_tries = 3
-          @redis.with_metrics do |redis|
-            redis.multi
-            redis.srem("vmpooler__completed__#{pool}", vm_name)
-            redis.hdel("vmpooler__active__#{pool}", vm_name)
-            redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
-
-            # Auto-expire metadata key
-            redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
-            redis.exec
-          end
-
-          start = Time.now
-
-          if vm_object.is_a? RbVmomi::VIM::Folder
-            logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
-            raise('Expected VM, but received a folder object')
-          end
-          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn'
-          vm_object.Destroy_Task.wait_for_completion
-
-          finish = format('%<time>.2f', time: Time.now - start)
-          logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
-          metrics.timing("destroy.#{pool}", finish)
-        rescue RuntimeError
-          raise
-        rescue StandardError => e
-          try += 1
-          logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{e}")
-          try >= max_tries ? raise : retry
-        end
-
-        def destroy_folder_and_children(folder_object)
-          vms = {}
-          data_ttl = $config[:redis]['data_ttl'].to_i
-          folder_name = folder_object.name
-          unless folder_object.childEntity.count == 0
-            folder_object.childEntity.each do |vm|
-              vms[vm.name] = vm
-            end
-
-            vms.each do |vm_name, vm_object|
-              destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
-            end
-          end
-          destroy_folder(folder_object)
-        end
-
-        def destroy_folder(folder_object)
-          try = 0 if try.nil?
-          max_tries = 3
-          logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
-          folder_object.Destroy_Task.wait_for_completion
-        rescue StandardError
-          try += 1
-          try >= max_tries ? raise : retry
-        end
-
-        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-
-            base_folders.each do |base_folder|
-              folder_children = get_folder_children(base_folder, connection)
-              next if folder_children.empty?
-
-              folder_children.each do |folder_hash|
-                folder_hash.each do |folder_title, folder_object|
-                  destroy_folder_and_children(folder_object) unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
-                end
-              end
-            end
-          end
-        end
-
-        def get_folder_children(folder_name, connection)
-          folders = []
-
-          propSpecs = { # rubocop:disable Naming/VariableName
-            entity: self,
-            inventoryPath: folder_name
-          }
-          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs) # rubocop:disable Naming/VariableName
-
-          return folders if folder_object.nil?
-
-          folder_object.childEntity.each do |folder|
-            next unless folder.is_a? RbVmomi::VIM::Folder
-
-            folders << { folder.name => folder }
-          end
-
-          folders
-        end
-
-        def vms_in_pool(pool_name)
-          vms = []
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            folder_object = find_vm_folder(pool_name, connection)
-
-            return vms if folder_object.nil?
-
-            folder_object.childEntity.each do |vm|
-              vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
-            end
-          end
-          vms
-        end
-
-        def select_target_hosts(target, cluster, datacenter)
-          percentage = 100
-          dc = "#{datacenter}_#{cluster}"
-          @provider_hosts_lock.synchronize do
-            begin
-              target[dc] = {} unless target.key?(dc)
-              target[dc]['checking'] = true
-              hosts_hash = find_least_used_hosts(cluster, datacenter, percentage)
-              target[dc] = hosts_hash
-            rescue StandardError
-              target[dc] = {}
-              raise
-            ensure
-              target[dc]['check_time_finished'] = Time.now
-            end
-          end
-        end
-
-        def run_select_hosts(pool_name, target)
-          now = Time.now
-          max_age = @config[:config]['host_selection_max_age'] || 60
-          loop_delay = 5
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-
-          dc = "#{datacenter}_#{cluster}"
-          unless target.key?(dc)
-            select_target_hosts(target, cluster, datacenter)
-            return
-          end
-          wait_for_host_selection(dc, target, loop_delay, max_age) if target[dc].key?('checking')
-          if target[dc].key?('check_time_finished')
-            select_target_hosts(target, cluster, datacenter) if now - target[dc]['check_time_finished'] > max_age
-          end
-        end
-
-        def wait_for_host_selection(dc, target, maxloop = 0, loop_delay = 1, max_age = 60)
-          loop_count = 1
-          until target.key?(dc) && target[dc].key?('check_time_finished')
-            sleep(loop_delay)
-            unless maxloop == 0
-              break if loop_count >= maxloop
-
-              loop_count += 1
-            end
-          end
-          return unless target[dc].key?('check_time_finished')
-
-          loop_count = 1
-          while Time.now - target[dc]['check_time_finished'] > max_age
-            sleep(loop_delay)
-            unless maxloop == 0
-              break if loop_count >= maxloop
-
-              loop_count += 1
-            end
-          end
-        end
-
-        def select_next_host(pool_name, target, architecture = nil)
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-
-          dc = "#{datacenter}_#{cluster}"
-          @provider_hosts_lock.synchronize do
-            if architecture
-              raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('architectures')
-
-              host = target[dc]['architectures'][architecture].shift
-              target[dc]['architectures'][architecture] << host
-              if target[dc]['hosts'].include?(host)
-                target[dc]['hosts'].delete(host)
-                target[dc]['hosts'] << host
-              end
-            else
-              raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('hosts')
-
-              host = target[dc]['hosts'].shift
-              target[dc]['hosts'] << host
-              target[dc]['architectures'].each do |arch|
-                target[dc]['architectures'][arch] = arch.partition { |v| v != host }.flatten if arch.include?(host)
-              end
-            end
-
-            return host
-          end
-        end
-
-        def vm_in_target?(pool_name, parent_host, architecture, target)
-          datacenter = get_target_datacenter_from_config(pool_name)
-          cluster = get_target_cluster_from_config(pool_name)
-          raise("cluster for pool #{pool_name} cannot be identified") if cluster.nil?
-          raise("datacenter for pool #{pool_name} cannot be identified") if datacenter.nil?
-
-          dc = "#{datacenter}_#{cluster}"
-          raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory") unless target[dc].key?('hosts')
-          return true if target[dc]['hosts'].include?(parent_host)
-          return true if target[dc]['architectures'][architecture].include?(parent_host)
-
-          false
-        end
-
-        def get_vm(pool_name, vm_name)
-          vm_hash = nil
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            return vm_hash if vm_object.nil?
-
-            vm_hash = generate_vm_hash(vm_object, pool_name)
-          end
-          vm_hash
-        end
-
-        def create_vm(pool_name, new_vmname)
-          pool = pool_config(pool_name)
-          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
-
-          vm_hash = nil
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            # Assume all pool config is valid i.e. not missing
-            template_path = pool['template']
-            target_folder_path = pool['folder']
-            target_datastore = pool['datastore']
-            target_datacenter_name = get_target_datacenter_from_config(pool_name)
-
-            # Get the template VM object
-            raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless valid_template_path? template_path
-
-            template_vm_object = find_template_vm(pool, connection)
-
-            # Annotate with creation time, origin template, etc.
-            # Add extraconfig options that can be queried by vmtools
-            config_spec = RbVmomi::VIM.VirtualMachineConfigSpec(
-              annotation: JSON.pretty_generate(
-                name: new_vmname,
-                created_by: provider_config['username'],
-                base_template: template_path,
-                creation_timestamp: Time.now.utc
-              ),
-              extraConfig: [
-                { key: 'guestinfo.hostname', value: new_vmname }
-              ]
-            )
-
-            # Check if alternate network configuration is specified and add configuration
-            if pool.key?('network')
-              template_vm_network_device = template_vm_object.config.hardware.device.grep(RbVmomi::VIM::VirtualEthernetCard).first
-              network_name = pool['network']
-              network_device = set_network_device(target_datacenter_name, template_vm_network_device, network_name, connection)
-              config_spec.deviceChange = [{ operation: 'edit', device: network_device }]
-            end
-
-            # Put the VM in the specified folder and resource pool
-            relocate_spec = create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
-
-            # Create a clone spec
-            clone_spec = create_clone_spec(relocate_spec, config_spec)
-
-            begin
-              vm_target_folder = find_vm_folder(pool_name, connection)
-              vm_target_folder ||= create_folder(connection, target_folder_path, target_datacenter_name) if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
-            rescue StandardError
-              if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
-                vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
-              else
-                raise
-              end
-            end
-            raise ArgumentError, "Cannot find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
-
-            # Create the new VM
-            new_vm_object = template_vm_object.CloneVM_Task(
-              folder: vm_target_folder,
-              name: new_vmname,
-              spec: clone_spec
-            ).wait_for_completion
-
-            vm_hash = generate_vm_hash(new_vm_object, pool_name)
-          end
-          vm_hash
-        end
-
-        def create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
-          pool = pool_config(pool_name)
-          target_cluster_name = get_target_cluster_from_config(pool_name)
-
-          relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-            datastore: find_datastore(target_datastore, connection, target_datacenter_name),
-            diskMoveType: get_disk_backing(pool)
-          )
-          manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
-          if manage_host_selection
-            run_select_hosts(pool_name, @provider_hosts)
-            target_host = select_next_host(pool_name, @provider_hosts)
-            host_object = find_host_by_dnsname(connection, target_host)
-            relocate_spec.host = host_object
-          else
-            # Choose a cluster/host to place the new VM on
-            target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
-            relocate_spec.pool = target_cluster_object.resourcePool
-          end
-          relocate_spec
-        end
-
-        def create_clone_spec(relocate_spec, config_spec)
-          RbVmomi::VIM.VirtualMachineCloneSpec(
-            location: relocate_spec,
-            config: config_spec,
-            powerOn: true,
-            template: false
-          )
-        end
-
-        def set_network_device(datacenter_name, template_vm_network_device, network_name, connection)
-          # Retrieve network object
-          datacenter = connection.serviceInstance.find_datacenter(datacenter_name)
-          new_network = datacenter.network.find { |n| n.name == network_name }
-
-          raise("Cannot find network #{network_name} in datacenter #{datacenter_name}") unless new_network
-
-          # Determine network device type
-          # All possible device type options here: https://vdc-download.vmware.com/vmwb-repository/dcr-public/98d63b35-d822-47fe-a87a-ddefd469df06/2e3c7b58-f2bd-486e-8bb1-a75eb0640bee/doc/vim.vm.device.VirtualEthernetCard.html
-          network_device =
-            if template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet2
-              RbVmomi::VIM.VirtualVmxnet2
-            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet3
-              RbVmomi::VIM.VirtualVmxnet3
-            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000
-              RbVmomi::VIM.VirtualE1000
-            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000e
-              RbVmomi::VIM.VirtualE1000e
-            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualSriovEthernetCard
-              RbVmomi::VIM.VirtualSriovEthernetCard
-            else
-              RbVmomi::VIM.VirtualPCNet32
-            end
-
-          # Set up new network device attributes
-          network_device.key = template_vm_network_device.key
-          network_device.deviceInfo = RbVmomi::VIM.Description(
-            label: template_vm_network_device.deviceInfo.label,
-            summary: network_name
-          )
-          network_device.backing = RbVmomi::VIM.VirtualEthernetCardNetworkBackingInfo(
-            deviceName: network_name,
-            network: new_network,
-            useAutoDetect: false
-          )
-          network_device.addressType = 'assigned'
-          network_device.connectable = RbVmomi::VIM.VirtualDeviceConnectInfo(
-            allowGuestControl: true,
-            startConnected: true,
-            connected: true
-          )
-          network_device
-        end
-
-        def create_disk(pool_name, vm_name, disk_size)
-          pool = pool_config(pool_name)
-          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
-
-          datastore_name = pool['datastore']
-          raise("Pool #{pool_name} does not have a datastore defined for the provider #{name}") if datastore_name.nil?
-
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            add_disk(vm_object, disk_size, datastore_name, connection, get_target_datacenter_from_config(pool_name))
-          end
-          true
-        end
-
-        def create_snapshot(pool_name, vm_name, new_snapshot_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            old_snap = find_snapshot(vm_object, new_snapshot_name)
-            raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil?
-
-            vm_object.CreateSnapshot_Task(
-              name: new_snapshot_name,
-              description: 'vmpooler',
-              memory: true,
-              quiesce: true
-            ).wait_for_completion
-          end
-          true
-        end
-
-        def revert_snapshot(pool_name, vm_name, snapshot_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
-
-            snapshot_object = find_snapshot(vm_object, snapshot_name)
-            raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil?
-
-            snapshot_object.RevertToSnapshot_Task.wait_for_completion
-          end
-          true
-        end
-
-        def destroy_vm(pool_name, vm_name)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            vm_object = find_vm(pool_name, vm_name, connection)
-            # If a VM doesn't exist then it is effectively deleted
-            return true if vm_object.nil?
-
-            # Poweroff the VM if it's running
-            vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn'
-
-            # Kill it with fire
-            vm_object.Destroy_Task.wait_for_completion
-          end
-          true
-        end
-
-        def vm_ready?(_pool_name, vm_name)
-          begin
-            open_socket(vm_name, global_config[:config]['domain'])
-          rescue StandardError => _e
-            return false
-          end
-
-          true
-        end
-
-        # VSphere Helper methods
-
-        def get_target_cluster_from_config(pool_name)
-          pool = pool_config(pool_name)
-          return nil if pool.nil?
-
-          return pool['clone_target'] unless pool['clone_target'].nil?
-          return global_config[:config]['clone_target'] unless global_config[:config]['clone_target'].nil?
-
-          nil
-        end
-
-        def get_target_datacenter_from_config(pool_name)
-          pool = pool_config(pool_name)
-          return nil if pool.nil?
-
-          return pool['datacenter']            unless pool['datacenter'].nil?
-          return provider_config['datacenter'] unless provider_config['datacenter'].nil?
-
-          nil
-        end
-
-        # Return a hash of VM data
-        # Provides vmname, hostname, template, poolname, boottime and powerstate information
-        def generate_vm_hash(vm_object, pool_name)
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-
-          hostname = vm_object.summary.guest.hostName if vm_object.summary&.guest && vm_object.summary.guest.hostName
-          boottime = vm_object.runtime.bootTime if vm_object.runtime&.bootTime
-          powerstate = vm_object.runtime.powerState if vm_object.runtime&.powerState
-
-          hash = {
-            'name' => vm_object.name,
-            'hostname' => hostname,
-            'template' => pool_configuration['template'],
-            'poolname' => pool_name,
-            'boottime' => boottime,
-            'powerstate' => powerstate
-          }
-
-          hash
-        end
-
-        # vSphere helper methods
-        ADAPTER_TYPE = 'lsiLogic'
-        DISK_TYPE = 'thin'
-        DISK_MODE = 'persistent'
-
-        def ensured_vsphere_connection(connection_pool_object)
-          connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection])
-          connection_pool_object[:connection]
-        end
-
-        def vsphere_connection_ok?(connection)
-          _result = connection.serviceInstance.CurrentTime
-          true
-        rescue StandardError
-          false
-        end
-
-        def connect_to_vsphere
-          max_tries = global_config[:config]['max_tries'] || 3
-          retry_factor = global_config[:config]['retry_factor'] || 10
-          try = 1
-          begin
-            connection = RbVmomi::VIM.connect host: provider_config['server'],
-                                              user: provider_config['username'],
-                                              password: provider_config['password'],
-                                              insecure: provider_config['insecure'] || false
-            metrics.increment('connect.open')
-            connection
-          rescue StandardError => e
-            metrics.increment('connect.fail')
-            raise e if try >= max_tries
-
-            sleep(try * retry_factor)
-            try += 1
-            retry
-          end
-        end
-
-        # This should supercede the open_socket method in the Pool Manager
-        def open_socket(host, domain = nil, timeout = 5, port = 22, &_block)
-          Timeout.timeout(timeout) do
-            target_host = host
-            target_host = "#{host}.#{domain}" if domain
-            sock = TCPSocket.new target_host, port
-            begin
-              yield sock if block_given?
-            ensure
-              sock.close
-            end
-          end
-        end
-
-        def get_vm_folder_path(vm_object)
-          # This gives an array starting from the root Datacenters folder all the way to the VM
-          # [ [Object, String], [Object, String ] ... ]
-          # It's then reversed so that it now goes from the VM to the Datacenter
-          full_path = vm_object.path.reverse
-
-          # Find the Datacenter object
-          dc_index = full_path.index { |p| p[0].is_a?(RbVmomi::VIM::Datacenter) }
-          return nil if dc_index.nil?
-          # The Datacenter should be at least 2 otherwise there's something
-          # wrong with the array passed in
-          # This is the minimum:
-          # [ VM (0), VM ROOT FOLDER (1), DC (2)]
-          return nil if dc_index <= 1
-
-          # Remove the VM name (Starting position of 1 in the slice)
-          # Up until the Root VM Folder of DataCenter Node (dc_index - 2)
-          full_path = full_path.slice(1..dc_index - 2)
-
-          # Reverse the array back to normal and
-          # then convert the array of paths into a '/' seperated string
-          (full_path.reverse.map { |p| p[1] }).join('/')
-        end
-
-        def add_disk(vm, size, datastore, connection, datacentername)
-          return false unless size.to_i > 0
-
-          vmdk_datastore = find_datastore(datastore, connection, datacentername)
-          raise("Datastore '#{datastore}' does not exist in datacenter '#{datacentername}'") if vmdk_datastore.nil?
-
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          controller = find_disk_controller(vm)
-          disk_unit_number = find_disk_unit_number(vm, controller)
-          disk_count = vm.config.hardware.device.grep(RbVmomi::VIM::VirtualDisk).count
-          vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{disk_count}.vmdk"
-
-          vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec(
-            capacityKb: size.to_i * 1024 * 1024,
-            adapterType: ADAPTER_TYPE,
-            diskType: DISK_TYPE
-          )
-
-          vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo(
-            datastore: vmdk_datastore,
-            diskMode: DISK_MODE,
-            fileName: "[#{datastore}] #{vmdk_file_name}"
-          )
-
-          device = RbVmomi::VIM::VirtualDisk(
-            backing: vmdk_backing,
-            capacityInKB: size.to_i * 1024 * 1024,
-            controllerKey: controller.key,
-            key: -1,
-            unitNumber: disk_unit_number
-          )
-
-          device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec(
-            device: device,
-            operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add')
-          )
-
-          vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec(
-            deviceChange: [device_config_spec]
-          )
-
-          connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
-            datacenter: datacenter,
-            name: "[#{datastore}] #{vmdk_file_name}",
-            spec: vmdk_spec
-          ).wait_for_completion
-
-          vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion
-
-          true
-        end
-
-        def find_datastore(datastorename, connection, datacentername)
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
-
-          datacenter.find_datastore(datastorename)
-        end
-
-        def find_device(vm, device_name)
-          vm.config.hardware.device.each do |device|
-            return device if device.deviceInfo.label == device_name
-          end
-
-          nil
-        end
-
-        def find_disk_controller(vm)
-          devices = find_disk_devices(vm)
-
-          devices.keys.sort.each do |device|
-            return find_device(vm, devices[device]['device'].deviceInfo.label) if devices[device]['children'].length < 15
-          end
-
-          nil
-        end
-
-        def find_disk_devices(vm)
-          devices = {}
-
-          vm.config.hardware.device.each do |device|
-            if device.is_a? RbVmomi::VIM::VirtualSCSIController
-              if devices[device.controllerKey].nil?
-                devices[device.key] = {}
-                devices[device.key]['children'] = []
-              end
-
-              devices[device.key]['device'] = device
-            end
-
-            if device.is_a? RbVmomi::VIM::VirtualDisk
-              if devices[device.controllerKey].nil?
-                devices[device.controllerKey] = {}
-                devices[device.controllerKey]['children'] = []
-              end
-
-              devices[device.controllerKey]['children'].push(device)
-            end
-          end
-
-          devices
-        end
-
-        def find_disk_unit_number(vm, controller)
-          used_unit_numbers = []
-          available_unit_numbers = []
-
-          devices = find_disk_devices(vm)
-
-          devices.keys.sort.each do |c|
-            next unless controller.key == devices[c]['device'].key
-
-            used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber)
-            devices[c]['children'].each do |disk|
-              used_unit_numbers.push(disk.unitNumber)
-            end
-          end
-
-          (0..15).each do |scsi_id|
-            available_unit_numbers.push(scsi_id) if used_unit_numbers.grep(scsi_id).length <= 0
-          end
-
-          available_unit_numbers.min
-        end
-
-        # Finds a folder object by inventory path
-        # Params:
-        # +pool_name+:: the pool to find the folder for
-        # +connection+:: the vsphere connection object
-        # returns a ManagedObjectReference for the folder found or nil if not found
-        def find_vm_folder(pool_name, connection)
-          # Find a folder by its inventory path and return the object
-          # Returns nil when the object found is not a folder
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-
-          folder = pool_configuration['folder']
-          datacenter = get_target_datacenter_from_config(pool_name)
-          return nil if datacenter.nil?
-
-          propSpecs = { # rubocop:disable Naming/VariableName
-            entity: self,
-            inventoryPath: "#{datacenter}/vm/#{folder}"
-          }
-
-          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs) # rubocop:disable Naming/VariableName
-          return nil unless folder_object.class == RbVmomi::VIM::Folder
-
-          folder_object
-        end
-
-        # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference
-        # Params:
-        # +model+:: CPU arch version to match on
-        # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
-        # returns nil if one on these conditions is true:
-        #    the model param is defined and cannot be found
-        #    the host is in maintenance mode
-        #    the host status is not 'green'
-        #    the cpu or memory utilization is bigger than the limit param
-        def get_host_utilization(host, model = nil, limit = 90)
-          limit = @config[:config]['utilization_limit'] if @config[:config].key?('utilization_limit')
-          if model
-            return nil unless host_has_cpu_model?(host, model)
-          end
-          return nil if host.runtime.inMaintenanceMode
-          return nil unless host.overallStatus == 'green'
-          return nil unless host.configIssue.empty?
-
-          cpu_utilization = cpu_utilization_for host
-          memory_utilization = memory_utilization_for host
-
-          return nil if cpu_utilization.nil?
-          return nil if cpu_utilization == 0.0
-          return nil if memory_utilization.nil?
-          return nil if memory_utilization == 0.0
-
-          return nil if cpu_utilization > limit
-          return nil if memory_utilization > limit
-
-          [cpu_utilization, host]
-        end
-
-        def host_has_cpu_model?(host, model)
-          get_host_cpu_arch_version(host) == model
-        end
-
-        def get_host_cpu_arch_version(host)
-          cpu_model = host.hardware.cpuPkg[0].description
-          cpu_model_parts = cpu_model.split
-          arch_version = cpu_model_parts[4]
-          arch_version
-        end
-
-        def cpu_utilization_for(host)
-          cpu_usage = host.summary.quickStats.overallCpuUsage
-          return nil if cpu_usage.nil?
-
-          cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores
-          cpu_usage.fdiv(cpu_size) * 100
-        end
-
-        def memory_utilization_for(host)
-          memory_usage = host.summary.quickStats.overallMemoryUsage
-          return nil if memory_usage.nil?
-
-          memory_size = host.summary.hardware.memorySize / 1024 / 1024
-          memory_usage.fdiv(memory_size) * 100
-        end
-
-        def get_average_cluster_utilization(hosts)
-          utilization_counts = hosts.map { |host| host[0] }
-          utilization_counts.inject(:+) / hosts.count
-        end
-
-        def build_compatible_hosts_lists(hosts, percentage)
-          hosts_with_arch_versions = hosts.map do |h|
-            {
-              'utilization' => h[0],
-              'host_object' => h[1],
-              'architecture' => get_host_cpu_arch_version(h[1])
-            }
-          end
-          versions = hosts_with_arch_versions.map { |host| host['architecture'] }.uniq
-          architectures = {}
-          versions.each do |version|
-            architectures[version] = []
-          end
-
-          hosts_with_arch_versions.each do |h|
-            architectures[h['architecture']] << [h['utilization'], h['host_object'], h['architecture']]
-          end
-
-          versions.each do |version|
-            targets = select_least_used_hosts(architectures[version], percentage)
-            architectures[version] = targets
-          end
-          architectures
-        end
-
-        def select_least_used_hosts(hosts, percentage)
-          raise('Provided hosts list to select_least_used_hosts is empty') if hosts.empty?
-
-          average_utilization = get_average_cluster_utilization(hosts)
-          least_used_hosts = []
-          hosts.each do |host|
-            least_used_hosts << host if host[0] <= average_utilization
-          end
-          hosts_to_select = (hosts.count * (percentage / 100.0)).to_int
-          hosts_to_select = hosts.count - 1 if percentage == 100
-          least_used_hosts.sort[0..hosts_to_select].map { |host| host[1].name }
-        end
-
-        def find_least_used_hosts(cluster, datacentername, percentage)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            cluster_object = find_cluster(cluster, connection, datacentername)
-            raise("Cluster #{cluster} cannot be found") if cluster_object.nil?
-
-            target_hosts = get_cluster_host_utilization(cluster_object)
-            raise("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-
-            architectures = build_compatible_hosts_lists(target_hosts, percentage)
-            least_used_hosts = select_least_used_hosts(target_hosts, percentage)
-            {
-              'hosts' => least_used_hosts,
-              'architectures' => architectures
-            }
-          end
-        end
-
-        def find_host_by_dnsname(connection, dnsname)
-          host_object = connection.searchIndex.FindByDnsName(dnsName: dnsname, vmSearch: false)
-          return nil if host_object.nil?
-
-          host_object
-        end
-
-        def find_least_used_host(cluster, connection, datacentername)
-          cluster_object = find_cluster(cluster, connection, datacentername)
-          target_hosts = get_cluster_host_utilization(cluster_object)
-          raise("There is no host candidate in vcenter that meets all the required conditions, check that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-
-          least_used_host = target_hosts.min[1]
-          least_used_host
-        end
-
-        def find_cluster(cluster, connection, datacentername)
-          datacenter = connection.serviceInstance.find_datacenter(datacentername)
-          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
-
-          # In the event the cluster is not a direct descendent of the
-          # datacenter, we use a ContainerView to leverage its recursive
-          # search. This will find clusters which are, for example, in
-          # folders under the datacenter. This will also find standalone
-          # hosts which are not part of a cluster.
-          cv = connection.serviceContent.viewManager.CreateContainerView(
-            container: datacenter.hostFolder,
-            type: ['ComputeResource', 'ClusterComputeResource'],
-            recursive: true
-          )
-          cluster = cv.view.find { |cluster_object| cluster_object.name == cluster }
-          cv.DestroyView
-          cluster
-        end
-
-        def get_cluster_host_utilization(cluster, model = nil)
-          cluster_hosts = []
-          cluster.host.each do |host|
-            host_usage = get_host_utilization(host, model)
-            cluster_hosts << host_usage if host_usage
-          end
-          cluster_hosts
-        end
-
-        def find_least_used_vpshere_compatible_host(vm)
-          source_host = vm.summary.runtime.host
-          model = get_host_cpu_arch_version(source_host)
-          cluster = source_host.parent
-          target_hosts = get_cluster_host_utilization(cluster, model)
-          raise("There is no host candidate in vcenter that meets all the required conditions, check that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
-
-          target_host = target_hosts.min[1]
-          [target_host, target_host.name]
-        end
-
-        def find_snapshot(vm, snapshotname)
-          get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) if vm.snapshot
-        end
-
-        def build_propSpecs(datacenter, folder, vmname) # rubocop:disable Naming/MethodName
-          propSpecs = { # rubocop:disable Naming/VariableName
-            entity => self,
-            :inventoryPath => "#{datacenter}/vm/#{folder}/#{vmname}"
-          }
-          propSpecs # rubocop:disable Naming/VariableName
-        end
-
-        def find_vm(pool_name, vmname, connection)
-          # Find a VM by its inventory path and return the VM object
-          # Returns nil when a VM, or pool configuration, cannot be found
-          pool_configuration = pool_config(pool_name)
-          return nil if pool_configuration.nil?
-
-          folder = pool_configuration['folder']
-          datacenter = get_target_datacenter_from_config(pool_name)
-          return nil if datacenter.nil?
-
-          propSpecs = { # rubocop:disable Naming/VariableName
-            entity: self,
-            inventoryPath: "#{datacenter}/vm/#{folder}/#{vmname}"
-          }
-
-          connection.searchIndex.FindByInventoryPath(propSpecs) # rubocop:disable Naming/VariableName
-        end
-
-        def get_base_vm_container_from(connection)
-          view_manager = connection.serviceContent.viewManager
-          view_manager.CreateContainerView(
-            container: connection.serviceContent.rootFolder,
-            recursive: true,
-            type: ['VirtualMachine']
-          )
-        end
-
-        def get_snapshot_list(tree, snapshotname)
-          snapshot = nil
-
-          tree.each do |child|
-            if child.name == snapshotname
-              snapshot ||= child.snapshot
-            else
-              snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname)
-            end
-          end
-
-          snapshot
-        end
-
-        def get_vm_details(pool_name, vm_name, connection)
-          vm_object = find_vm(pool_name, vm_name, connection)
-          return nil if vm_object.nil?
-
-          parent_host_object = vm_object.summary.runtime.host if vm_object.summary&.runtime && vm_object.summary.runtime.host
-          raise('Unable to determine which host the VM is running on') if parent_host_object.nil?
-
-          parent_host = parent_host_object.name
-          architecture = get_host_cpu_arch_version(parent_host_object)
-          {
-            'host_name' => parent_host,
-            'object' => vm_object,
-            'architecture' => architecture
-          }
-        end
-
-        def migration_enabled?(config)
-          migration_limit = config[:config]['migration_limit']
-          return false unless migration_limit.is_a? Integer
-          return true if migration_limit > 0
-
-          false
-        end
-
-        def migrate_vm(pool_name, vm_name)
-          @connection_pool.with_metrics do |pool_object|
-            begin
-              connection = ensured_vsphere_connection(pool_object)
-              vm_hash = get_vm_details(pool_name, vm_name, connection)
-              @redis.with_metrics do |redis|
-                redis.hset("vmpooler__vm__#{vm_name}", 'host', vm_hash['host_name'])
-                migration_count = redis.scard('vmpooler__migration')
-                migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
-                if migration_enabled? @config
-                  if migration_count >= migration_limit
-                    logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
-                    break
-                  end
-                  run_select_hosts(pool_name, @provider_hosts)
-                  if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
-                    logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
-                  else
-                    migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-                  end
-                else
-                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
-                end
-              end
-            rescue StandardError
-              logger.log('s', "[!] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
-              raise
-            end
-          end
-        end
-
-        def migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-          @redis.with_metrics do |redis|
-            redis.sadd('vmpooler__migration', vm_name)
-          end
-          target_host_name = select_next_host(pool_name, @provider_hosts, vm_hash['architecture'])
-          target_host_object = find_host_by_dnsname(connection, target_host_name)
-          finish = migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, target_host_name)
-          @redis.with_metrics do |redis|
-            redis.multi
-            redis.hset("vmpooler__vm__#{vm_name}", 'host', target_host_name)
-            redis.hset("vmpooler__vm__#{vm_name}", 'migrated', true)
-            redis.exec
-          end
-          logger.log('s', "[>] [#{pool_name}] '#{vm_name}' migrated from #{vm_hash['host_name']} to #{target_host_name} in #{finish} seconds")
-        ensure
-          @redis.with_metrics do |redis|
-            redis.srem('vmpooler__migration', vm_name)
-          end
-        end
-
-        def migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, dest_host_name)
-          start = Time.now
-          migrate_vm_host(vm_hash['object'], target_host_object)
-          finish = format('%<time>.2f', time: Time.now - start)
-          metrics.timing("migrate.#{pool_name}", finish)
-          metrics.increment("migrate_from.#{vm_hash['host_name']}")
-          metrics.increment("migrate_to.#{dest_host_name}")
-          @redis.with_metrics do |redis|
-            checkout_to_migration = format('%<time>.2f', time: Time.now - Time.parse(redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
-            redis.multi
-            redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)
-            redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration)
-            redis.exec
-          end
-          finish
-        end
-
-        def migrate_vm_host(vm_object, host)
-          relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host)
-          vm_object.RelocateVM_Task(spec: relospec).wait_for_completion
-        end
-
-        def create_folder(connection, new_folder, datacenter)
-          dc = connection.serviceInstance.find_datacenter(datacenter)
-          folder_object = dc.vmFolder.traverse(new_folder, RbVmomi::VIM::Folder, true)
-          raise("Cannot create folder #{new_folder}") if folder_object.nil?
-
-          folder_object
-        end
-
-        def find_template_vm(pool, connection)
-          datacenter = get_target_datacenter_from_config(pool['name'])
-          raise('cannot find datacenter') if datacenter.nil?
-
-          propSpecs = { # rubocop:disable Naming/VariableName
-            entity: self,
-            inventoryPath: "#{datacenter}/vm/#{pool['template']}"
-          }
-
-          template_vm_object = connection.searchIndex.FindByInventoryPath(propSpecs) # rubocop:disable Naming/VariableName
-          raise("Pool #{pool['name']} specifies a template VM of #{pool['template']} which does not exist for the provider #{name}") if template_vm_object.nil?
-
-          template_vm_object
-        end
-
-        def create_template_delta_disks(pool)
-          @connection_pool.with_metrics do |pool_object|
-            connection = ensured_vsphere_connection(pool_object)
-            template_vm_object = find_template_vm(pool, connection)
-
-            template_vm_object.add_delta_disk_layer_on_all_disks
-          end
-        end
-
-        def valid_template_path?(template)
-          return false unless template.include?('/')
-          return false if template[0] == '/'
-          return false if template[-1] == '/'
-
-          true
-        end
-
-        def get_disk_backing(pool)
-          return :moveChildMostDiskBacking if linked_clone?(pool)
-
-          :moveAllDiskBackingsAndConsolidate
-        end
-
-        def linked_clone?(pool)
-          return if pool['create_linked_clone'] == false
-          return true if pool['create_linked_clone']
-          return true if @config[:config]['create_linked_clones']
-        end
-      end
-    end
-  end
-end
diff --git a/lib/vmpooler/public/img/logo.png b/lib/vmpooler/public/img/logo.png
new file mode 100644
index 0000000..0cefaba
Binary files /dev/null and b/lib/vmpooler/public/img/logo.png differ
diff --git a/lib/vmpooler/public/lib/dashboard.js b/lib/vmpooler/public/lib/dashboard.js
index e726b01..bd12074 100644
--- a/lib/vmpooler/public/lib/dashboard.js
+++ b/lib/vmpooler/public/lib/dashboard.js
@@ -29,10 +29,10 @@ Date.prototype.yyyymmdd = function() {
 
 var data_url = {
   'capacity': '/dashboard/stats/vmpooler/pool',
-  'pools'   : '/api/v1/vm',
+  'pools'   : '/api/v3/vm',
   'running' : '/dashboard/stats/vmpooler/running',
-  'status'  : '/api/v1/status',
-  'summary' : '/api/v1/summary'
+  'status'  : '/api/v3/status',
+  'summary' : '/api/v3/summary'
 };
 
 
diff --git a/lib/vmpooler/util/parsing.rb b/lib/vmpooler/util/parsing.rb
index 8a8273f..1ca6b13 100644
--- a/lib/vmpooler/util/parsing.rb
+++ b/lib/vmpooler/util/parsing.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# utility class shared between apps
+# utility class shared between apps api and pool_manager
 module Vmpooler
   class Parsing
     def self.get_platform_pool_count(requested, &_block)
diff --git a/lib/vmpooler/version.rb b/lib/vmpooler/version.rb
index ca4b1be..4469c2a 100644
--- a/lib/vmpooler/version.rb
+++ b/lib/vmpooler/version.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '0.16.3'
+  VERSION = '3.8.1'
 end
diff --git a/release-notes.md b/release-notes.md
new file mode 100644
index 0000000..7e9892e
--- /dev/null
+++ b/release-notes.md
@@ -0,0 +1,15 @@
+## [3.8.1](https://github.com/puppetlabs/vmpooler/tree/3.8.1) (2026-01-14)
+
+[Full Changelog](https://github.com/puppetlabs/vmpooler/compare/3.7.0...3.8.1)
+
+**Implemented enhancements:**
+
+- \(P4DEVOPS-9434\) Add rate limiting and input validation security enhancements [\#690](https://github.com/puppetlabs/vmpooler/pull/690) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8570\) Add Phase 2 optimizations: status API caching and improved Redis pipelining [\#689](https://github.com/puppetlabs/vmpooler/pull/689) ([mahima-singh](https://github.com/mahima-singh))
+- \(P4DEVOPS-8567\) Add DLQ, auto-purge, and health checks for Redis queues [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Add retry logic for immediate clone failures [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
+
+**Fixed bugs:**
+
+- \(P4DEVOPS-8567\) Prevent VM allocation for already-deleted request-ids [\#688](https://github.com/puppetlabs/vmpooler/pull/688) ([mahima-singh](https://github.com/mahima-singh))
+- Prevent re-queueing requests already marked as failed [\#687](https://github.com/puppetlabs/vmpooler/pull/687) ([mahima-singh](https://github.com/mahima-singh))
diff --git a/release-prep b/release-prep
new file mode 100755
index 0000000..a0082de
--- /dev/null
+++ b/release-prep
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+#
+# Update Gemfile.lock
+docker run -t --rm \
+  -v $(pwd):/app \
+  jruby:9.4.12.1-jdk11 \
+  /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase && cd /app && gem install bundler && bundle install --jobs 3; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"'
+
+# Update Changelog
+docker run -t --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \
+  githubchangeloggenerator/github-changelog-generator:1.16.4 \
+  github_changelog_generator --future-release $(grep VERSION lib/vmpooler/version.rb |rev |cut -d "'" -f2 |rev) \
+  --token $CHANGELOG_GITHUB_TOKEN --release-branch main
+
diff --git a/scripts/create_template_deltas.rb b/scripts/create_template_deltas.rb
deleted file mode 100755
index f03ce39..0000000
--- a/scripts/create_template_deltas.rb
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/ruby
-
-require 'rubygems'
-require 'rbvmomi'
-require 'yaml'
-
-def load_configuration( file_array )
-  file_array.each do |file|
-    file = File.expand_path( file )
-
-    if File.exists?( file )
-      return YAML.load_file( file )
-    end
-  end
-
-  return false
-end
-
-def create_template_deltas( folder )
-  config = load_configuration( [ 'vmpooler.yaml', '~/.vmpooler' ] ) || nil
-
-  abort 'No config file (./vmpooler.yaml or ~/.vmpooler) found!' unless config
-
-  vim = RbVmomi::VIM.connect(
-    :host     => config[ :providers ][ :vsphere ][ "server" ],
-    :user     => config[ :providers ][ :vsphere ][ "username" ],
-    :password => config[ :providers ][ :vsphere ][ "password" ],
-    :ssl      => true,
-    :insecure => true,
-  ) or abort "Unable to connect to #{config[ :vsphere ][ "server" ]}!"
-
-  containerView = vim.serviceContent.viewManager.CreateContainerView( {
-    :container => vim.serviceContent.rootFolder,
-    :recursive => true,
-    :type      => [ 'VirtualMachine' ]
-  } )
-
-  datacenter = vim.serviceInstance.find_datacenter
-  base = datacenter.vmFolder
-
-  case base
-    when RbVmomi::VIM::Folder
-      base = base.childEntity.find { |f| f.name == folder }
-    else
-      abort "Unexpected object type encountered (#{base.class}) while finding folder!"
-  end
-
-  unless base
-    abort "Folder #{ARGV[0]} not found!"
-  end
-
-  base.childEntity.each do |vm|
-    print vm.name
-
-    begin
-      disks = vm.config.hardware.device.grep( RbVmomi::VIM::VirtualDisk )
-    rescue
-      puts ' !'
-      next
-    end
-
-    begin
-      disks.select { |d| d.backing.parent == nil }.each do |disk|
-        linkSpec = {
-          :deviceChange => [
-            {
-              :operation => :remove,
-              :device => disk
-            },
-            {
-              :operation => :add,
-              :fileOperation => :create,
-              :device => disk.dup.tap { |x|
-                x.backing = x.backing.dup
-                x.backing.fileName = "[#{disk.backing.datastore.name}]"
-                x.backing.parent = disk.backing
-              }
-            }
-          ]
-        }
-
-        vm.ReconfigVM_Task( :spec => linkSpec ).wait_for_completion
-      end
-
-      puts " \u2713"
-    rescue
-      puts ' !'
-    end
-  end
-
-  vim.close
-end
-
-if ARGV[0]
-  create_template_deltas( ARGV[0] )
-else
-  puts "Usage: #{$0} <folder>"
-end
diff --git a/spec/fixtures/extra_config1.yaml b/spec/fixtures/extra_config1.yaml
new file mode 100644
index 0000000..d19523c
--- /dev/null
+++ b/spec/fixtures/extra_config1.yaml
@@ -0,0 +1,4 @@
+---
+:providers:
+  :alice:
+    foo: "foo"
\ No newline at end of file
diff --git a/spec/fixtures/extra_config2.yaml b/spec/fixtures/extra_config2.yaml
new file mode 100644
index 0000000..af122c2
--- /dev/null
+++ b/spec/fixtures/extra_config2.yaml
@@ -0,0 +1,12 @@
+---
+:providers:
+  :bob:
+    foo: "foo_bob"
+    bar: "bar"
+
+:pools:
+  - name: 'pool05'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
\ No newline at end of file
diff --git a/spec/fixtures/vmpooler.yaml b/spec/fixtures/vmpooler.yaml
index c5c0522..c7dad1b 100644
--- a/spec/fixtures/vmpooler.yaml
+++ b/spec/fixtures/vmpooler.yaml
@@ -23,9 +23,13 @@
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,8 +40,10 @@
   - name: 'pool01'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
     ready_ttl: 5
   - name: 'pool02'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
     ready_ttl: 5
diff --git a/spec/fixtures/vmpooler2.yaml b/spec/fixtures/vmpooler2.yaml
index a016d30..92731a5 100644
--- a/spec/fixtures/vmpooler2.yaml
+++ b/spec/fixtures/vmpooler2.yaml
@@ -23,9 +23,13 @@
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # Uncomment the lines below to suppress metrics to STDOUT
 # :statsd:
 #   server: 'localhost'
@@ -36,8 +40,10 @@
   - name: 'pool03'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
     ready_ttl: 5
   - name: 'pool04'
     size: 5
     provider: dummy
+    dns_plugin: 'example'
     ready_ttl: 5
diff --git a/spec/fixtures/vmpooler_domain.yaml b/spec/fixtures/vmpooler_domain.yaml
new file mode 100644
index 0000000..9a79726
--- /dev/null
+++ b/spec/fixtures/vmpooler_domain.yaml
@@ -0,0 +1,45 @@
+---
+:providers:
+  :dummy:
+
+:redis:
+  server: 'localhost'
+
+:auth:
+   provider: dummy
+
+:tagfilter:
+  url: '(.*)\/'
+
+:config:
+  site_name: 'vmpooler'
+  # Need to change this on Windows
+  logfile: '/var/log/vmpooler.log'
+  task_limit: 10
+  timeout: 15
+  vm_checktime: 1
+  vm_lifetime: 12
+  vm_lifetime_auth: 24
+  allowed_tags:
+    - 'created_by'
+    - 'project'
+  domain: 'example.com'
+  prefix: 'poolvm-'
+
+# Uncomment the lines below to suppress metrics to STDOUT
+# :statsd:
+#   server: 'localhost'
+#   prefix: 'vmpooler'
+#   port: 8125
+
+:pools:
+  - name: 'pool01'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
+  - name: 'pool02'
+    size: 5
+    provider: dummy
+    dns_plugin: 'example'
+    ready_ttl: 5
diff --git a/spec/helpers.rb b/spec/helpers.rb
index 87245db..53af74c 100644
--- a/spec/helpers.rb
+++ b/spec/helpers.rb
@@ -14,6 +14,16 @@ class MockLogger
   end
 end
 
+class MockPoolManagerDnsBase
+  def delete_record(hostname)
+
+  end
+
+  def create_or_replace_record(hostname)
+
+  end
+end
+
 def expect_json(ok = true, http = 200)
   expect(last_response.header['Content-Type']).to eq('application/json')
 
diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v3/config_spec.rb
similarity index 75%
rename from spec/integration/api/v1/config_spec.rb
rename to spec/integration/api/v3/config_spec.rb
index 79874c8..301c4aa 100644
--- a/spec/integration/api/v1/config_spec.rb
+++ b/spec/integration/api/v3/config_spec.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -26,6 +26,10 @@ describe Vmpooler::API::V1 do
         {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
         {'name' => 'pool2', 'size' => 10}
       ],
+      pools_at_startup: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
       statsd: { 'prefix' => 'stats_prefix'},
       alias: { 'poolone' => 'pool1' },
       pool_names: [ 'pool1', 'pool2', 'poolone' ]
@@ -33,7 +37,7 @@ describe Vmpooler::API::V1 do
   }
 
   describe '/config/pooltemplate' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:current_time) { Time.now }
@@ -45,6 +49,47 @@ describe Vmpooler::API::V1 do
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
     end
 
+    describe 'DELETE /config/pooltemplate/:pool' do
+      it 'resets a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          template_before_reset: 'templates/new_template',
+          template_before_overrides: 'templates/pool1'
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when the pool has not been overridden' do
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails on nonexistent pools' do
+        delete "#{prefix}/config/pooltemplate/poolpoolpool"
+        expect_json(ok = false, http = 404)
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/pooltemplate/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+    end
+
     describe 'POST /config/pooltemplate' do
       it 'updates a pool template' do
         post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
@@ -142,6 +187,56 @@ describe Vmpooler::API::V1 do
 
     end
 
+    describe 'DELETE /config/poolsize' do
+      it 'resets a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          pool_size_before_reset: 2,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        delete "#{prefix}/config/poolsize/pool10"
+        expect_json(ok = false, http = 404)
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when a pool has not been overridden' do
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 200)
+        expected = {
+          ok: true,
+          pool_size_before_reset: 5,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/poolsize/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
     describe 'POST /config/poolsize' do
       it 'changes a pool size' do
         post "#{prefix}/config/poolsize", '{"pool1":"2"}'
@@ -293,7 +388,7 @@ describe Vmpooler::API::V1 do
     end
 
     describe 'GET /config' do
-      let(:prefix) { '/api/v1' }
+      let(:prefix) { '/api/v3' }
 
       it 'returns pool configuration when set' do
         get "#{prefix}/config"
diff --git a/spec/integration/api/v1/ondemandvm_spec.rb b/spec/integration/api/v3/ondemandvm_spec.rb
similarity index 89%
rename from spec/integration/api/v1/ondemandvm_spec.rb
rename to spec/integration/api/v3/ondemandvm_spec.rb
index a361f9f..0a043bb 100644
--- a/spec/integration/api/v1/ondemandvm_spec.rb
+++ b/spec/integration/api/v3/ondemandvm_spec.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -15,7 +15,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/ondemandvm' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
@@ -28,16 +28,25 @@ describe Vmpooler::API::V1 do
             'compute2' => 0
           }
         },
+        dns_configs: {
+          :mock => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
         pools: [
-          {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1'},
-          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2'},
-          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1'}
+          {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'},
+          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2', 'dns_plugin' => 'mock'},
+          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1', 'dns_plugin' => 'mock'}
         ],
         alias: {
           'poolone' => ['pool1'],
           'pool2' => ['pool1']
         },
-        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ]
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ],
+        providers: {
+          :dummy => {},
+        }
       }
     }
     let(:current_time) { Time.now }
@@ -111,24 +120,6 @@ describe Vmpooler::API::V1 do
             expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool2:pool1:1')
             post "#{prefix}/ondemandvm", '{"pool2":"1"}'
           end
-
-          context 'with domain set in the config' do
-            let(:domain) { 'example.com' }
-            before(:each) do
-              config[:config]['domain'] = domain
-            end
-
-            it 'should include domain in the return reply' do
-              post "#{prefix}/ondemandvm", '{"poolone":"1"}'
-              expect_json(true, 201)
-              expected = {
-                "ok": true,
-                "request_id": uuid,
-                "domain": domain
-              }
-              expect(last_response.body).to eq(JSON.pretty_generate(expected))
-            end
-          end
         end
 
         context 'with a resource request that exceeds the specified limit' do
@@ -262,35 +253,12 @@ describe Vmpooler::API::V1 do
               "ready": true,
               "pool1": {
                 "hostname": [
-                  vmname
+                  "#{vmname}.example.com"
                 ]
               }
             }
             expect(last_response.body).to eq(JSON.pretty_generate(expected))
           end
-
-          context 'with domain set' do
-            let(:domain) { 'example.com' }
-            before(:each) do
-              config[:config]['domain'] = domain
-            end
-
-            it 'should include the domain in the result' do
-              get "#{prefix}/ondemandvm/#{uuid}"
-              expected = {
-                "ok": true,
-                "request_id": uuid,
-                "ready": true,
-                "pool1": {
-                  "hostname": [
-                    vmname
-                  ]
-                },
-                "domain": domain
-              }
-              expect(last_response.body).to eq(JSON.pretty_generate(expected))
-            end
-          end
         end
 
         context 'with a deleted request' do
diff --git a/spec/integration/api/v1/poolreset.rb b/spec/integration/api/v3/poolreset.rb
similarity index 98%
rename from spec/integration/api/v1/poolreset.rb
rename to spec/integration/api/v3/poolreset.rb
index dfb7f18..74a11e3 100644
--- a/spec/integration/api/v1/poolreset.rb
+++ b/spec/integration/api/v3/poolreset.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -30,7 +30,7 @@ describe Vmpooler::API::V1 do
   }
 
   describe '/poolreset' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:current_time) { Time.now }
diff --git a/spec/integration/api/v1/status_spec.rb b/spec/integration/api/v3/status_spec.rb
similarity index 98%
rename from spec/integration/api/v1/status_spec.rb
rename to spec/integration/api/v3/status_spec.rb
index 76e65df..5a5449c 100644
--- a/spec/integration/api/v1/status_spec.rb
+++ b/spec/integration/api/v3/status_spec.rb
@@ -5,7 +5,7 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -17,10 +17,12 @@ describe Vmpooler::API::V1 do
   # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
   before(:each) do
     app.reset!
+    # Clear status cache to prevent test interference
+    Vmpooler::API::V3.clear_status_cache
   end
 
   describe 'status and metrics endpoints' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
 
     let(:config) {
       {
diff --git a/spec/integration/api/v1/token_spec.rb b/spec/integration/api/v3/token_spec.rb
similarity index 98%
rename from spec/integration/api/v1/token_spec.rb
rename to spec/integration/api/v3/token_spec.rb
index 72ab060..2fd0dac 100644
--- a/spec/integration/api/v1/token_spec.rb
+++ b/spec/integration/api/v3/token_spec.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/token' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:current_time) { Time.now }
     let(:config) { { 
       config: {}
@@ -111,7 +111,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/token/:token' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:current_time) { Time.now }
 
     before(:each) do
diff --git a/spec/integration/api/v1/vm_hostname_spec.rb b/spec/integration/api/v3/vm_hostname_spec.rb
similarity index 99%
rename from spec/integration/api/v1/vm_hostname_spec.rb
rename to spec/integration/api/v3/vm_hostname_spec.rb
index ff5f7a3..b4c064d 100644
--- a/spec/integration/api/v1/vm_hostname_spec.rb
+++ b/spec/integration/api/v3/vm_hostname_spec.rb
@@ -5,7 +5,7 @@ def has_set_tag?(vm, tag, value)
   value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
 end
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/vm/:hostname' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
 
     let(:config) {
diff --git a/spec/integration/api/v1/vm_spec.rb b/spec/integration/api/v3/vm_spec.rb
similarity index 88%
rename from spec/integration/api/v1/vm_spec.rb
rename to spec/integration/api/v3/vm_spec.rb
index 0d06585..0800d7c 100644
--- a/spec/integration/api/v1/vm_spec.rb
+++ b/spec/integration/api/v3/vm_spec.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/vm' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
@@ -24,10 +24,29 @@ describe Vmpooler::API::V1 do
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2
         },
+        providers: {
+          vsphere: {},
+          gce: {},
+          foo: {}
+        },
+        dns_configs: {
+          :one => {
+            'dns_class' => 'mock',
+            'domain' => 'one.example.com'
+          },
+          :two => {
+            'dns_class' => 'mock',
+            'domain' => 'two.example.com'
+          },
+          :three => {
+            'dns_class' => 'mock',
+            'domain' => 'three.example.com'
+          }
+        },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10},
-          {'name' => 'pool3', 'size' => 10}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'one'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'gce', 'dns_plugin' => 'two'},
+          {'name' => 'pool3', 'size' => 10, 'provider' => 'foo', 'dns_plugin' => 'three'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
         alias: { 'poolone' => ['pool1'] },
@@ -79,7 +98,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -97,7 +116,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           poolone: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           }
         }
 
@@ -124,8 +143,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', vmname, redis
-        post "#{prefix}/vm/poolone"
+        create_ready_vm 'pool2', vmname, redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -146,10 +164,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: vmname
+            hostname: "#{vmname}.one.example.com"
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.two.example.com'
           }
         }
 
@@ -177,8 +195,8 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
@@ -206,8 +224,8 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.two.example.com', '2qrstuvwxyz012345.two.example.com', '3qrstuvwxyz012345.two.example.com')
 
         expect_json(ok = true, http = 200)
       end
@@ -232,7 +250,7 @@ describe Vmpooler::API::V1 do
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop', '1qrstuvwxyz012345')
+        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.two.example.com', '1qrstuvwxyz012345.three.example.com')
 
         expect_json(ok = true, http = 200)
       end
@@ -249,7 +267,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           "pool1": {
-            "hostname": "1abcdefghijklmnop"
+            "hostname": "1abcdefghijklmnop.one.example.com"
           }
         }
 
@@ -338,7 +356,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns the second VM when the first fails to respond' do
-        create_ready_vm 'pool1', vmname, redis
+        create_running_vm 'pool1', vmname, redis
         create_ready_vm 'pool1', "2#{vmname}", redis
 
         allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with(vmname, nil).and_raise('mockerror')
@@ -350,7 +368,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: "2#{vmname}"
+            hostname: "2#{vmname}.one.example.com"
           }
         }
 
@@ -375,7 +393,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -401,7 +419,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -422,7 +440,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.one.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
diff --git a/spec/integration/api/v1/vm_template_spec.rb b/spec/integration/api/v3/vm_template_spec.rb
similarity index 88%
rename from spec/integration/api/v1/vm_template_spec.rb
rename to spec/integration/api/v3/vm_template_spec.rb
index 25c0fbd..f809244 100644
--- a/spec/integration/api/v1/vm_template_spec.rb
+++ b/spec/integration/api/v3/vm_template_spec.rb
@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Vmpooler::API::V1 do
+describe Vmpooler::API::V3 do
   include Rack::Test::Methods
 
   def app()
@@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do
   end
 
   describe '/vm/:template' do
-    let(:prefix) { '/api/v1' }
+    let(:prefix) { '/api/v3' }
     let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
     let(:config) {
       {
@@ -24,10 +24,17 @@ describe Vmpooler::API::V1 do
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
         },
+        dns_configs: {
+          :example => {
+            'dns_class' => 'mock',
+            'domain' => 'example.com'
+          }
+        },
+        providers: { vsphere: {} },
         pools: [
-          {'name' => 'pool1', 'size' => 5},
-          {'name' => 'pool2', 'size' => 10},
-          {'name' => 'poolone', 'size' => 0}
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'vsphere', 'dns_plugin' => 'example'},
+          {'name' => 'poolone', 'size' => 1, 'provider' => 'vsphere', 'dns_plugin' => 'example'}
         ],
         statsd: { 'prefix' => 'stats_prefix'},
         alias: { 'poolone' => 'pool1' },
@@ -59,7 +66,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
 
@@ -76,7 +83,7 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           poolone: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           }
         }
         expect_json(ok = true, http = 200)
@@ -104,7 +111,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'abcdefghijklmnop', redis
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }
@@ -125,10 +132,10 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: 'abcdefghijklmnop'
+            hostname: 'abcdefghijklmnop.example.com'
           },
           pool2: {
-            hostname: 'qrstuvwxyz012345'
+            hostname: 'qrstuvwxyz012345.example.com'
           }
         }
 
@@ -150,17 +157,17 @@ describe Vmpooler::API::V1 do
         expected = {
           ok: true,
           pool1: {
-            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+            hostname: [ '1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com' ]
           },
           pool2: {
-            hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ]
+            hostname: [ '1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com' ]
           }
         }
 
         result = JSON.parse(last_response.body)
         expect(result['ok']).to eq(true)
-        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
-        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.example.com', '2abcdefghijklmnop.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.example.com', '2qrstuvwxyz012345.example.com', '3qrstuvwxyz012345.example.com')
         expect_json(ok = true, http = 200)
       end
 
@@ -264,7 +271,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
 
@@ -290,7 +297,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
@@ -310,7 +317,7 @@ describe Vmpooler::API::V1 do
           expected = {
             ok: true,
             pool1: {
-              hostname: 'abcdefghijklmnop'
+              hostname: 'abcdefghijklmnop.example.com'
             }
           }
           expect_json(ok = true, http = 200)
diff --git a/spec/rbvmomi_helper.rb b/spec/rbvmomi_helper.rb
deleted file mode 100644
index f8c8a1a..0000000
--- a/spec/rbvmomi_helper.rb
+++ /dev/null
@@ -1,922 +0,0 @@
-# -----------------------------------------------------------------------------------------------------------------
-# Managed Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-mo_types.html)
-# -----------------------------------------------------------------------------------------------------------------
-
-MockClusterComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-  # From MockClusterComputeResource
-  :actionHistory, :configuration, :drsFault, :drsRecommendation, :migrationHistory, :recommendation,
-  # From ComputeResource
-  :resourcePool,
-  # From ManagedEntity
-  :name
-)
-
-MockComputeResource = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-  # From ComputeResource
-  :configurationEx, :datastore, :host, :network, :resourcePool, :summary,
-  # From ManagedEntity
-  :name
-)
-
-MockContainerView = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-  # From ContainerView
-  :container, :recursive, :type
-) do
-  def _search_tree(layer)
-    results = []
-
-    layer.children.each do |child|
-      if type.any? { |t| child.is_a?(RbVmomi::VIM.const_get(t)) }
-        results << child
-      end
-
-      if recursive && child.respond_to?(:children)
-          results += _search_tree(child)
-      end
-    end
-    results
-  end
-
-  def view
-    _search_tree(container)
-  end
-
-  def DestroyView
-  end
-end
-
-MockDatacenter = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datacenter.html
-  # From Datacenter
-  :datastore, :datastoreFolder, :hostFolder, :network, :networkFolder, :vmFolder,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Datacenter https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Datacenter.rb
-
-  # Find the Datastore with the given +name+.
-  def find_datastore name
-    datastore.find { |x| x.name == name }
-  end
-end
-
-MockNetwork = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Network.html
-  # From Network
-  :host, :name, :summary, :vm
-)
-
-MockVirtualVmxnet3 = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualVmxnet.html
-  # From VirtualEthenetCard
-  :addressType,
-  # From VirtualDevice
-  :key, :deviceInfo, :backing, :connectable
-)
-
-MockDatastore = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-  # From Datastore
-  :browser, :capability, :host, :info, :iormConfiguration, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockFolder = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Folder.html
-  # From Folder
-  :childEntity, :childType,
-  # From ManagedEntity
-  :name
-) do
-  # From RBVMOMI::VIM::Folder https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L107-L110
-  def children
-    childEntity
-  end
-
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L9-L12
-  def find(name, type=Object)
-    # Fake the searchIndex
-    childEntity.each do |child|
-      if child.name == name
-        if child.kind_of?(type)
-          return child
-        else
-          return nil
-        end
-      end
-    end
-
-    nil
-  end
-end
-
-MockHostSystem = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-  # From HostSystem
-  :capability, :config, :configManager, :datastore, :datastoreBrowser, :hardware, :network, :runtime, :summary, :systemResources, :vm,
-  # From ManagedEntity
-  :overallStatus, :name, :parent,
-  # From ManagedObject
-  :configIssue
-)
-
-MockPropertyCollector = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-  # PropertyCollector
-  :filter
-)
-
-MockResourcePool = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-  # From ResourcePool
-  :childConfiguration, :config, :owner, :resourcePool, :runtime, :summary, :vm,
-  # From ManagedEntity
-  :name
-)
-
-MockSearchIndex = Object
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.SearchIndex.html
-
-MockServiceInstance = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-  # From ServiceInstance
-  :capability, :content, :serverClock
-) do
-  # From ServiceInstance 
-  # Mock the CurrentTime method so that it appears the ServiceInstance is valid.
-  def CurrentTime
-    Time.now
-  end
-
-  # From RBVMOMI::VIM::ServiceInstance https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ServiceInstance.rb
-  def find_datacenter(path=nil)
-    # In our mocked instance, DataCenters are always in the root Folder.
-    # If path is nil the first DC is returned otherwise match by name
-    content.rootFolder.childEntity.each do |child|
-      if child.is_a?(RbVmomi::VIM::Datacenter)
-        return child if path.nil? || child.name == path
-      end
-    end
-    nil
-  end
-end
-
-MockTask = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-  # From Task
-  :info,
-) do
-  # From RBVMOMI https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Task.rb
-  # Mock the with 'Not Implemented'
-  def wait_for_completion
-    raise(RuntimeError,'Not Implemented')
-  end
-end
-
-MockViewManager = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ViewManager.html
-  # From ViewManager
-  :viewList,
-) do
-  # From ViewManager 
-  def CreateContainerView(options)
-    mock_RbVmomi_VIM_ContainerView({
-      :container => options[:container],
-      :recursive => options[:recursive],
-      :type => options[:type],
-    })
-  end
-end
-
-MockVirtualDiskManager = Object
-# https://pubs.vmware.com/vsphere-55/index.jsp#com.vmware.wssdk.apiref.doc/vim.VirtualDiskManager.html
-
-MockVirtualMachine = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-  # From VirtualMachine
-  :config, :runtime, :snapshot, :summary,
-  # From ManagedEntity
-  :name,
-  # From RbVmomi::VIM::ManagedEntity
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ManagedEntity.rb
-  :path
-)
-
-MockVirtualMachineSnapshot = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-  # From VirtualMachineSnapshot
-  :config
-)
-
-# -------------------------------------------------------------------------------------------------------------
-# Data Objects (https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/index-do_types.html)
-# -------------------------------------------------------------------------------------------------------------
-
-MockDescription = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Description.html
-  # From Description
-  :label, :summary
-)
-
-MockVirtualEthernetCardNetworkBackingInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualEthernetCard.NetworkBackingInfo.html
-  # From VirtualEthernetCardNetworkBackingInfo
-  :network,
-
-  # From VirtualDeviceBackingInfo
-  :deviceName, :useAutoDetect
-)
-
-MockVirtualDeviceConnectInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.ConnectInfo.html
-  # From VirtualDeviceConnectInfo
-  :allowGuestControl, :connected, :startConnected
-)
-
-MockVirtualMachineConfigSpec = Struct.new(
-  # https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.vm.ConfigSpec.html
-  # From VirtualMachineConfigSpec
-  :deviceChange, :annotation, :extraConfig
-)
-
-MockVirtualMachineRelocateSpec = Struct.new(
-  # https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.vm.RelocateSpec.html
-  # From VirtualMachineRelocateSpec
-  :datastore, :diskMoveType, :pool
-)
-
-MockDynamicProperty = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.DynamicProperty.html
-  # From DynamicProperty
-  :name, :val
-)
-
-MockHostCpuPackage = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.CpuPackage.html
-  # From HostCpuPackage
-  :busHz, :cpuFeature, :description, :hz, :index, :threadId, :vendor
-)
-
-MockHostHardwareSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.HardwareSummary.html
-  # From HostHardwareSummary
-  :cpuMhz, :numCpuCores, :numCpuPkgs, :memorySize
-)
-
-MockHostListSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.html
-  # From HostListSummary
-  :quickStats, :hardware
-)
-
-MockHostListSummaryQuickStats = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.Summary.QuickStats.html
-  # From HostListSummaryQuickStats
-  :overallCpuUsage, :overallMemoryUsage
-)
-
-MockHostRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.RuntimeInfo.html
-  # From HostRuntimeInfo
-  :bootTime, :connectionState, :healthSystemRuntime, :inMaintenanceMode, :powerState, :tpmPcrValues
-)
-
-MockHostSystemHostHardwareInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.host.HardwareInfo.html
-  # From HostHardwareInfo
-  :biosInfo, :cpuFeature, :cpuInfo, :cpuPkg, :cpuPowerManagementInfo, :memorySize, :numaInfo, :pciDevice, :systemInfo
-)
-
-MockObjectContent = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.ObjectContent.html
-  # From ObjectContent
-  :missingSet, :obj, :propSet
-)
-
-MockRetrieveResult = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-  # From RetrieveResult
-  :objects, :token
-)
-
-MockServiceContent = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-  # From ServiceContent
-  :propertyCollector, :rootFolder, :searchIndex, :viewManager, :virtualDiskManager
-)
-
-MockVirtualDevice = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualDisk = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-  # From VirtualDisk
-  :capacityInKB, :shares,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-MockVirtualHardware = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.VirtualHardware.html
-  # From VirtualHardware
-  :device
-)
-
-MockVirtualMachineConfigInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.ConfigInfo.html
-  # From VirtualMachineConfigInfo
-  :hardware
-)
-
-MockVirtualMachineFileLayoutExFileInfo = Struct.new(
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-  # From VirtualMachineFileLayoutExFileInfo
-  :key, :name, :size, :type, :uniqueSize
-)
-
-MockVirtualMachineGuestSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.GuestSummary.html
-  # From VirtualMachineGuestSummary
-  :hostName
-)
-
-MockVirtualMachineRuntimeInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.RuntimeInfo.html
-  # From VirtualMachineRuntimeInfo
-  :bootTime, :cleanPowerOff, :connectionState, :faultToleranceState, :host, :maxCpuUsage, :maxMemoryUsage, :memoryOverhead,
-  :needSecondaryReason, :numMksConnections, :powerState, :question, :recordReplayState, :suspendInterval, :suspendTime, :toolsInstallerMounted
-)
-
-MockVirtualMachineSnapshotInfo = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotInfo.html
-  # From MockVirtualMachineSnapshotInfo
-  :currentSnapshot, :rootSnapshotList
-)
-
-MockVirtualMachineSnapshotTree = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.SnapshotTree.html
-  # From VirtualMachineSnapshotTree
-  :backupManifest, :childSnapshotList, :createTime, :description, :id, :name, :quiesced, :replaySupported,
-  :snapshot, :state, :vm
-)
-
-MockVirtualMachineSummary = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.html
-  # From VirtualMachineSummary
-  :config, :customValue, :guest, :quickStats, :runtime, :storage, :vm
-)
-
-MockVirtualSCSIController = Struct.new(
-  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-  # From VirtualSCSIController
-  :hotAddRemove, :scsiCtlrUnitNumber, :sharedBus,
-  # From VirtualDevice
-  :deviceInfo, :controllerKey, :key, :backing, :connectable, :unitNumber
-)
-
-# --------------------
-# RBVMOMI only Objects
-# --------------------
-MockRbVmomiVIMConnection = Struct.new(
-  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  :serviceInstance, :serviceContent, :rootFolder, :root
-) do
-  # From https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-  # Alias to serviceContent.searchIndex
-  def searchIndex
-    serviceContent.searchIndex
-  end
-  # Alias to serviceContent.propertyCollector
-  def propertyCollector
-    serviceContent.propertyCollector
-  end
-end
-
-# -------------------------------------------------------------------------------------------------------------
-# Mocking Methods
-# -------------------------------------------------------------------------------------------------------------
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ClusterComputeResource.html
-def mock_RbVmomi_VIM_ClusterComputeResource(options = {})
-  options[:name]  = 'Cluster' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockClusterComputeResource.new()
-
-  mock.name = options[:name]
-  # All cluster compute resources have a root Resource Pool
-  mock.resourcePool = mock_RbVmomi_VIM_ResourcePool({:name => options[:name]})
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ClusterComputeResource
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.view.ContainerView.html
-def mock_RbVmomi_VIM_ContainerView(options = {})
-  mock = MockContainerView.new()
-  mock.container = options[:container]
-  mock.recursive = options[:recursive]
-  mock.type = options[:type]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ComputeResource.html
-def mock_RbVmomi_VIM_ComputeResource(options = {})
-  options[:name]  = 'Compute' + rand(65536).to_s if options[:name].nil?
-  options[:hosts] = [{}] if options[:hosts].nil?
-
-  mock = MockComputeResource.new()
-
-  mock.name = options[:name]
-  mock.host = []
-
-  # A compute resource must have at least one host.
-  options[:hosts].each do |host_options|
-    mock_host = mock_RbVmomi_VIM_HostSystem(host_options)
-    mock_host.parent = mock
-    mock.host << mock_host
-  end
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ComputeResource
-  end
-
-  mock
-end
-
-# https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim.rb
-def mock_RbVmomi_VIM_Connection(options = {})
-  options[:serviceInstance] = {} if options[:serviceInstance].nil?
-  options[:serviceContent]  = {} if options[:serviceContent].nil?
-
-  mock = MockRbVmomiVIMConnection.new()
-  mock.serviceContent = mock_RbVmomi_VIM_ServiceContent(options[:serviceContent])
-  options[:serviceInstance][:servicecontent] = mock.serviceContent if options[:serviceInstance][:servicecontent].nil?
-  mock.serviceInstance = mock_RbVmomi_VIM_ServiceInstance(options[:serviceInstance])
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datacenter(options = {})
-  options[:hostfolder_tree] = {} if options[:hostfolder_tree].nil?
-  options[:vmfolder_tree]   = {} if options[:vmfolder_tree].nil?
-  # Currently don't support mocking datastore tree
-  options[:datastores]      = [] if options[:datastores].nil?
-  options[:name]            = 'Datacenter' + rand(65536).to_s if options[:name].nil?
-  options[:networks]        = [] if options[:networks].nil?
-
-  mock = MockDatacenter.new()
-
-  mock.name = options[:name]
-  mock.hostFolder = mock_RbVmomi_VIM_Folder({ :name => 'hostFolderRoot'})
-  mock.vmFolder = mock_RbVmomi_VIM_Folder({ :name => 'vmFolderRoot'})
-  mock.datastore = []
-  mock.network = []
-
-  # Create vmFolder hierarchy
-  recurse_folder_tree(options[:vmfolder_tree],mock.vmFolder.childEntity)
-
-  # Create hostFolder hierarchy
-  recurse_folder_tree(options[:hostfolder_tree],mock.hostFolder.childEntity)
-
-  # Create mock Datastores
-  options[:datastores].each do |datastorename|
-    mock_ds = mock_RbVmomi_VIM_Datastore({ :name => datastorename })
-    mock.datastore << mock_ds
-  end
-
-  # Create mock Networks
-  options[:networks].each do |networkname|
-    mock_nw = mock_RbVmomi_VIM_Network({ :name => networkname })
-    mock.network << mock_nw
-  end
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datacenter
-  end
-
-  mock
-end
-
-def recurse_folder_tree(tree, root_object)
-  tree.keys.each do |foldername|
-    folder_options = tree[foldername].nil? ? {} : tree[foldername]
-    folder_options[:name] = foldername if folder_options[:name].nil?
-
-    case folder_options[:object_type]
-    when 'vm'
-      child_object = mock_RbVmomi_VIM_VirtualMachine({ :name => folder_options[:name]})
-    when 'compute_resource'
-      child_object = mock_RbVmomi_VIM_ComputeResource({ :name => folder_options[:name]})
-    when 'cluster_compute_resource'
-      child_object = mock_RbVmomi_VIM_ClusterComputeResource({ :name => folder_options[:name]})
-    when 'resource_pool'
-      child_object =  mock_RbVmomi_VIM_ResourcePool({ :name => folder_options[:name]})
-    else
-      child_object = mock_RbVmomi_VIM_Folder({:name => foldername})
-    end
-
-    # Recursively create children - Default is the child_object is a Folder
-    case folder_options[:object_type]
-    when 'cluster_compute_resource'
-      # Append children into the root Resource Pool for a cluster, instead of directly into the cluster itself.
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool.resourcePool) unless folder_options[:children].nil?
-    when 'resource_pool'
-      recurse_folder_tree(folder_options[:children],child_object.resourcePool) unless folder_options[:children].nil?
-    else
-      recurse_folder_tree(folder_options[:children],child_object.childEntity) unless folder_options[:children].nil?
-    end
-
-    root_object << child_object
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Datastore(options = {})
-  options[:name] = 'Datastore' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockDatastore.new()
-
-  mock.name = options[:name]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Datastore
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Network.html
-def mock_RbVmomi_VIM_Network(options = {})
-  options[:name] = 'Network' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockNetwork.new()
-
-  mock.name = options[:name]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Network
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualVmxnet3.html
-def mock_RbVmomi_VIM_VirtualVmxnet3(options = {})
-  options[:key] = rand(65536) if options[:key].nil?
-  options[:deviceInfo] = MockDescription.new()
-  options[:backing] = MockVirtualEthernetCardNetworkBackingInfo.new()
-  options[:addressType] = 'assigned'
-  options[:connectable] = MockVirtualDeviceConnectInfo.new()
-
-  mock = MockVirtualVmxnet3.new()
-
-  mock.key = options[:key]
-  mock.deviceInfo = options[:deviceInfo]
-  mock.backing = options[:backing]
-  mock.addressType = options[:addressType]
-  mock.connectable = options[:connectable]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualVmxnet3
-  end
-
-  mock
-end
-
-# https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.vm.RelocateSpec.html
-def mock_RbVmomi_VIM_VirtualMachineRelocateSpec(options = {})
-  options[:datastore] = 'Datastore' + rand(65536).to_s if options[:datastore].nil?
-  options[:diskMoveType] = :moveChildMostDiskBacking
-  options[:pool] = 'Pool' + rand(65536).to_s if options[:pool].nil?
-
-  mock = MockVirtualMachineRelocateSpec.new
-
-  mock.datastore = mock_RbVmomi_VIM_Datastore({ :name => options[:datastore]})
-  mock.diskMoveType = options[:diskMoveType]
-  mock.pool = mock_RbVmomi_VIM_ResourcePool({:name => options[:pool]})
-  allow(mock).to receive(:is_a?).and_return(RbVmomi::VIM::VirtualMachineRelocateSpec)
-  mock
-end
-
-# https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.vm.ConfigSpec.html
-def mock_RbVmomi_VIM_VirtualMachineConfigSpec(options = {})
-  options[:device] = mock_RbVmomi_VIM_VirtualVmxnet3()
-
-
-  mock = MockVirtualMachineConfigSpec.new
-
-  mock.deviceChange = []
-  mock.deviceChange << { operation: :edit, device: options[:device]}
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.Datastore.html
-def mock_RbVmomi_VIM_Folder(options = {})
-  options[:name] = 'Folder' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockFolder.new()
-
-  mock.name = options[:name]
-  mock.childEntity = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::Folder
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.HostSystem.html
-def mock_RbVmomi_VIM_HostSystem(options = {})
-  options[:memory_size]          = 4294967296 if options[:memory_size].nil? # 4GB RAM
-  options[:num_cpu]              = 1 if options[:num_cpu].nil?
-  options[:num_cores_per_cpu]    = 1 if options[:num_cores_per_cpu].nil?
-  options[:cpu_speed]            = 2048 if options[:cpu_speed].nil? # 2.0 GHz 
-  options[:cpu_model]            = 'Intel(R) Xeon(R) CPU E5-2697 v4 @ 2.0GHz' if options[:cpu_model].nil?
-  options[:maintenance_mode]     = false if options[:maintenance_mode].nil?
-  options[:overall_status]       = 'green' if options[:overall_status].nil?
-  options[:overall_cpu_usage]    = 1 if options[:overall_cpu_usage].nil?
-  options[:overall_memory_usage] = 1 if options[:overall_memory_usage].nil?
-  options[:name]                 = 'HOST' + rand(65536).to_s if options[:name].nil?
-  options[:config_issue]         = [] if options[:config_issue].nil?
-
-  mock = MockHostSystem.new()
-  mock.name = options[:name]
-  mock.summary = MockHostListSummary.new()
-  mock.summary.quickStats = MockHostListSummaryQuickStats.new()
-  mock.summary.hardware = MockHostHardwareSummary.new()
-  mock.hardware = MockHostSystemHostHardwareInfo.new()
-  mock.runtime = MockHostRuntimeInfo.new()
-
-  mock.hardware.cpuPkg = []
-  (1..options[:num_cpu]).each do |cpuid|
-    mockcpu = MockHostCpuPackage.new()
-    mockcpu.hz = options[:cpu_speed] * 1024 * 1024
-    mockcpu.description = options[:cpu_model]
-    mockcpu.index = 0
-    mock.hardware.cpuPkg << mockcpu
-  end
-
-  mock.runtime.inMaintenanceMode = options[:maintenance_mode]
-  mock.overallStatus = options[:overall_status]
-  mock.configIssue = options[:config_issue]
-
-  mock.summary.hardware.memorySize = options[:memory_size]
-  mock.hardware.memorySize = options[:memory_size]
-
-  mock.summary.hardware.cpuMhz = options[:cpu_speed]
-  mock.summary.hardware.numCpuCores = options[:num_cpu] * options[:num_cores_per_cpu]
-  mock.summary.hardware.numCpuPkgs = options[:num_cpu]
-  mock.summary.quickStats.overallCpuUsage = options[:overall_cpu_usage]
-  mock.summary.quickStats.overallMemoryUsage = options[:overall_memory_usage]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.RetrieveResult.html
-def mock_RbVmomi_VIM_RetrieveResult(options = {})
-  options[:response] = [] if options[:response].nil?
-  mock = MockRetrieveResult.new()
-
-  mock.objects = []
-
-  options[:response].each do |response|
-    mock_objectdata = MockObjectContent.new()
-
-    mock_objectdata.propSet = []
-
-    mock_objectdata.obj = response[:object] 
-
-    # Mock the object properties
-    response.each do |key,value|
-      unless key == :object
-        mock_property = MockDynamicProperty.new()
-        mock_property.name = key
-        mock_property.val = value
-        mock_objectdata.propSet << mock_property
-      end
-    end
-    mock.objects << mock_objectdata
-  end
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvmodl.query.PropertyCollector.html
-def mock_RbVmomi_VIM_PropertyCollector(options = {})
-  mock = MockPropertyCollector.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ServiceInstanceContent.html#field_detail
-def mock_RbVmomi_VIM_ServiceContent(options = {})
-  options[:propertyCollector] = {} if options[:propertyCollector].nil?
-  options[:datacenters]       = [] if options[:datacenters].nil?
-
-  mock = MockServiceContent.new()
-
-  mock.searchIndex = MockSearchIndex.new()
-  mock.viewManager = MockViewManager.new()
-  mock.virtualDiskManager = MockVirtualDiskManager.new()
-  mock.rootFolder = mock_RbVmomi_VIM_Folder({ :name => 'RootFolder' })
-
-  mock.propertyCollector = mock_RbVmomi_VIM_PropertyCollector(options[:propertyCollector])
-
-  # Create the DCs in this ServiceContent
-  options[:datacenters].each do |dc_options|
-    mock_dc = mock_RbVmomi_VIM_Datacenter(dc_options)
-    mock.rootFolder.childEntity << mock_dc
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.ServiceInstance.html
-def mock_RbVmomi_VIM_ServiceInstance(options = {})
-  mock = MockServiceInstance.new()
-
-  mock.content = options[:servicecontent]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.Task.html
-def mock_RbVmomi_VIM_Task(options = {})
-  mock = MockTask.new()
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.ResourcePool.html
-def mock_RbVmomi_VIM_ResourcePool(options = {})
-  options[:name] = 'ResourcePool' + rand(65536).to_s if options[:name].nil?
-
-  mock = MockResourcePool.new()
-
-  mock.name = options[:name]
-  mock.resourcePool = []
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::ResourcePool
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDisk.html
-def mock_RbVmomi_VIM_VirtualDisk(options = {})
-  options[:controllerKey] = rand(65536) if options[:controllerKey].nil?
-  options[:key]           = rand(65536) if options[:key].nil?
-  options[:label]         = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:unitNumber]    = rand(65536) if options[:unitNumber].nil?
-
-  mock = MockVirtualDisk.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.unitNumber = options[:unitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualDisk
-  end
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
-def mock_RbVmomi_VIM_VirtualMachine(options = {})
-  options[:snapshot_tree] = nil if options[:snapshot_tree].nil?
-  options[:name] = 'VM' + rand(65536).to_s if options[:name].nil?
-  options[:path] = [] if options[:path].nil?
-
-  mock = MockVirtualMachine.new()
-  mock.config = MockVirtualMachineConfigInfo.new()
-  mock.config.hardware = MockVirtualHardware.new([])
-  mock.summary = MockVirtualMachineSummary.new()
-  mock.summary.runtime = MockVirtualMachineRuntimeInfo.new()
-  mock.summary.guest = MockVirtualMachineGuestSummary.new()
-  mock.runtime = mock.summary.runtime
-
-  mock.name = options[:name]
-  mock.summary.guest.hostName = options[:hostname]
-  mock.runtime.bootTime = options[:boottime]
-  mock.runtime.powerState = options[:powerstate]
-
-  unless options[:snapshot_tree].nil?
-    mock.snapshot = MockVirtualMachineSnapshotInfo.new()
-    mock.snapshot.rootSnapshotList = []
-    index = 0
-
-    # Create a recursive snapshot tree
-    recurse_snapshot_tree(options[:snapshot_tree],mock.snapshot.rootSnapshotList,index)
-  end
-
-  # Create an array of items that describe the path of the VM from the root folder
-  # all the way to the VM itself
-  mock.path = []
-  options[:path].each do |path_item|
-    mock_item = nil
-    case path_item[:type]
-    when 'folder'
-      mock_item = mock_RbVmomi_VIM_Folder({ :name => path_item[:name] })
-    when 'datacenter'
-      mock_item = mock_RbVmomi_VIM_Datacenter({ :name => path_item[:name] })
-    else
-      raise("Unknown mock type #{path_item[:type]} for mock_RbVmomi_VIM_VirtualMachine")
-    end
-    mock.path << [mock_item,path_item[:name]]
-  end
-  mock.path << [mock,options[:name]]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualMachine
-  end
-
-  mock
-end
-
-def recurse_snapshot_tree(tree, root_object, index)
-  tree.keys.each do |snapshotname|
-    snap_options = tree[snapshotname].nil? ? {} : tree[snapshotname]
-    snap = MockVirtualMachineSnapshotTree.new()
-    snap.id = index
-    snap.name = snapshotname
-    snap.childSnapshotList = []
-    snap.description = "Snapshot #{snapshotname}"
-    snap.snapshot = snap_options[:ref] unless snap_options[:ref].nil?
-
-    # Recursively create chilren
-    recurse_snapshot_tree(snap_options[:children],snap.childSnapshotList,index) unless snap_options[:children].nil?
-
-    root_object << snap
-    index += 1
-  end
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualDevice.html
-def mock_RbVmomi_VIM_VirtualMachineDevice(options = {})
-  mock = MockVirtualDevice.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-
-  mock
-end
-
-# https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.vm.FileLayoutEx.FileInfo.html
-def mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo(options = {})
-  options[:key] = rand(65536).to_s if options[:key].nil?
-
-  mock = MockVirtualMachineFileLayoutExFileInfo.new()
-
-  mock.key = options[:key]
-  mock.name = options[:name]
-  mock.size = options[:size]
-  mock.type = options[:type]
-  mock.uniqueSize = options[:uniqueSize]
-
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Snapshot.html
-def mock_RbVmomi_VIM_VirtualMachineSnapshot(options = {})
-  mock = MockVirtualMachineSnapshot.new()
-  
-  mock
-end
-
-# https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.device.VirtualSCSIController.html
-def mock_RbVmomi_VIM_VirtualSCSIController(options = {})
-  options[:controllerKey]      = rand(65536) if options[:controllerKey].nil?
-  options[:key]                = rand(65536) if options[:key].nil?
-  options[:label]              = 'SCSI' + rand(65536).to_s if options[:label].nil?
-  options[:scsiCtlrUnitNumber] = 7 if options[:scsiCtlrUnitNumber].nil?
-
-  mock = MockVirtualSCSIController.new()
-  mock.deviceInfo = MockDescription.new()
-
-  mock.deviceInfo.label = options[:label]
-  mock.controllerKey = options[:controllerKey]
-  mock.key = options[:key]
-  mock.scsiCtlrUnitNumber = options[:scsiCtlrUnitNumber]
-
-  allow(mock).to receive(:is_a?) do |expected_type|
-    expected_type == RbVmomi::VIM::VirtualSCSIController
-  end
-
-  mock
-end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 080f797..dc94ee7 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -3,8 +3,6 @@ SimpleCov.start do
   add_filter '/spec/'
 end
 require 'helpers'
-require 'rbvmomi_helper'
-require 'rbvmomi'
 require 'rspec'
 require 'vmpooler'
 require 'redis'
diff --git a/spec/unit/api/helpers_spec.rb b/spec/unit/api/helpers_spec.rb
index 5c7f0ae..5788d5d 100644
--- a/spec/unit/api/helpers_spec.rb
+++ b/spec/unit/api/helpers_spec.rb
@@ -16,14 +16,12 @@ describe Vmpooler::API::Helpers do
 
   describe '#hostname_shorten' do
     [
-        ['example.com', 'not-example.com', 'example.com'],
-        ['example.com', 'example.com', 'example.com'],
-        ['sub.example.com', 'example.com', 'sub'],
-        ['adjective-noun.example.com', 'example.com', 'adjective-noun'],
-        ['abc123.example.com', 'example.com', 'abc123'],
-        ['example.com', nil, 'example.com']
-    ].each do |hostname, domain, expected|
-      it { expect(subject.hostname_shorten(hostname, domain)).to eq expected }
+        ['example.com', 'example'],
+        ['sub.example.com', 'sub'],
+        ['adjective-noun.example.com', 'adjective-noun'],
+        ['abc123.example.com', 'abc123']
+    ].each do |hostname, expected|
+      it { expect(subject.hostname_shorten(hostname)).to eq expected }
     end
   end
 
@@ -118,7 +116,7 @@ describe Vmpooler::API::Helpers do
       allow(redis).to receive(:pipelined).with(no_args).and_return [0]
       allow(redis).to receive(:get).and_return 0
 
-      expect(subject.get_queue_metrics([], redis)).to eq({pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
+      expect(subject.get_queue_metrics([], redis)).to eq({requested: 0, pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
     end
 
     it 'adds pool queues correctly' do
@@ -127,10 +125,14 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return(1,0)
+      # Mock returns 7*2 + 2 = 16 results (7 queue types for 2 pools + 2 global counters)
+      # For each pool: [request, processing, odcreate, pending, ready, running, completed]
+      # Plus 2 global counters: clone (1), ondemandclone (0)
+      # Results array: [1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1,1, 1, 0]
+      #                [req,  proc,  odc,   pend, rdy,  run,  comp, clone, odc]
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 14})
     end
 
     it 'sets booting to 0 when negative calculation' do
@@ -139,10 +141,10 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return(5,0)
+      # Mock returns 7*2 + 2 = 16 results with clone=5 to cause negative booting
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,0]
 
-      expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8})
+      expect(subject.get_queue_metrics(pools, redis)).to eq({requested: 6, pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 14})
     end
   end
 
@@ -264,22 +266,86 @@ describe Vmpooler::API::Helpers do
         }
       }
       let(:default_port) { 389 }
-      it 'should attempt ldap authentication' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str)
-
-        subject.authenticate(auth, username_str, password_str)
+      let(:default_encryption) do
+        {
+          :method => :start_tls,
+          :tls_options => { :ssl_version => 'TLSv1' }
+        }
       end
 
-      it 'should return true when authentication is successful' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(true)
+      context 'without a service account' do
+        it 'should attempt ldap authentication' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil)
 
-        expect(subject.authenticate(auth, username_str, password_str)).to be true
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when authentication is successful' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when authentication fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, nil).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
       end
 
-      it 'should return false when authentication fails' do
-        expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base, username_str, password_str).and_return(false)
+      context 'with a service account' do
+        let(:service_account_hash) do
+          {
+            :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+            :password => 's3cr3t'
+          }
+        end
+        let(:auth) {
+          {
+            'provider' => 'ldap',
+            ldap: {
+              'host' => host,
+              'base' => base,
+              'user_object' => user_object,
+              'service_account_hash' => service_account_hash
+            }
+          }
+        }
+        it 'should attempt ldap authentication' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash)
 
-        expect(subject.authenticate(auth, username_str, password_str)).to be false
+          subject.authenticate(auth, username_str, password_str)
+        end
+
+        it 'should return true when authentication is successful' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(true)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be true
+        end
+
+        it 'should return false when authentication fails' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base, username_str, password_str, service_account_hash).and_return(false)
+
+          expect(subject.authenticate(auth, username_str, password_str)).to be false
+        end
+      end
+
+      context 'with an alternate ssl_version' do
+        let(:secure_encryption) do
+          {
+            :method => :start_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the alternate ssl_version when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
       end
 
       context 'with an alternate port' do
@@ -289,7 +355,27 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should specify the alternate port when authenticating' do
-          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, user_object, base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(alternate_port, host, default_encryption, user_object, base, username_str, password_str, nil)
+
+          subject.authenticate(auth, username_str, password_str)
+        end
+      end
+
+      context 'with simple_tls and port 636' do
+        let(:secure_port) { 636 }
+        let(:secure_encryption) do
+          {
+            :method => :simple_tls,
+            :tls_options => { :ssl_version => 'TLSv1_2' }
+          }
+        end
+        before(:each) do
+          auth[:ldap]['port'] = secure_port
+          auth[:ldap]['encryption'] = secure_encryption
+        end
+
+        it 'should specify the secure port and encryption options when authenticating' do
+          expect(subject).to receive(:authenticate_ldap).with(secure_port, host, secure_encryption, user_object, base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
@@ -307,36 +393,36 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should attempt to bind with each base' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should not search the second base when the first binds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(true)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the second base when the first bind fails' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when any bind succeeds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when all bind attempts fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object, base[1], username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object, base[1], username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
@@ -354,36 +440,36 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should attempt to bind with each user object' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base, username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should not search the second user object when the first binds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base, username_str, password_str).and_return(true)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object[1], base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the second user object when the first bind fails' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base, username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base, username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when any bind succeeds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base, username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base, username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when all bind attempts fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base, username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base, username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base, username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base, username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
@@ -408,64 +494,64 @@ describe Vmpooler::API::Helpers do
         end
 
         it 'should attempt to bind with each user object and base' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should not continue searching when the first combination binds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(true)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str)
-          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(true)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to_not receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the remaining combinations when the first bind fails' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the remaining combinations when the first two binds fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should search the remaining combination when the first three binds fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil)
 
           subject.authenticate(auth, username_str, password_str)
         end
 
         it 'should return true when any bind succeeds' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str).and_return(true)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(true)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be true
         end
 
         it 'should return false when all bind attempts fail' do
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[0], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[0], base[1], username_str, password_str).and_return(false)
-          expect(subject).to receive(:authenticate_ldap).with(default_port, host, user_object[1], base[1], username_str, password_str).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[0], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[0], base[1], username_str, password_str, nil).and_return(false)
+          expect(subject).to receive(:authenticate_ldap).with(default_port, host, default_encryption, user_object[1], base[1], username_str, password_str, nil).and_return(false)
 
           expect(subject.authenticate(auth, username_str, password_str)).to be false
         end
@@ -493,16 +579,25 @@ describe Vmpooler::API::Helpers do
     let(:base) { 'ou=users,dc=example,dc=com' }
     let(:username_str) { 'admin' }
     let(:password_str) { 's3cr3t' }
+    let(:encryption) do
+      {
+        :method => :start_tls,
+        :tls_options => { :ssl_version => 'TLSv1' }
+      }
+    end
+    let(:service_account_hash) do
+      {
+        :user_dn => 'cn=Service Account,ou=users,dc=example,dc=com',
+        :password => 's3cr3t'
+      }
+    end
     let(:ldap) { double('ldap') }
     it 'should create a new ldap connection' do
       allow(ldap).to receive(:bind)
       expect(Net::LDAP).to receive(:new).with(
         :host => host,
         :port => port,
-        :encryption => {
-          :method => :start_tls,
-          :tls_options => { :ssl_version => 'TLSv1' }
-        },
+        :encryption => encryption,
         :base => base,
         :auth => {
           :method => :simple,
@@ -511,21 +606,35 @@ describe Vmpooler::API::Helpers do
         }
       ).and_return(ldap)
 
-      subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)
+      subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)
     end
 
     it 'should return true when a bind is successful' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(true)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be true
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be true
     end
 
     it 'should return false when a bind fails' do
       expect(Net::LDAP).to receive(:new).and_return(ldap)
       expect(ldap).to receive(:bind).and_return(false)
 
-      expect(subject.authenticate_ldap(port, host, user_object, base, username_str, password_str)).to be false
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str)).to be false
+    end
+
+    it 'should return true when a bind_as is successful' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(true)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be true
+    end
+
+    it 'should return false when a bind_as fails' do
+      expect(Net::LDAP).to receive(:new).and_return(ldap)
+      expect(ldap).to receive(:bind_as).and_return(false)
+
+      expect(subject.authenticate_ldap(port, host, encryption, user_object, base, username_str, password_str, service_account_hash)).to be false
     end
   end
 
diff --git a/spec/unit/api/input_validator_spec.rb b/spec/unit/api/input_validator_spec.rb
new file mode 100644
index 0000000..24982ed
--- /dev/null
+++ b/spec/unit/api/input_validator_spec.rb
@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rack/test'
+require 'vmpooler/api/input_validator'
+
+describe Vmpooler::API::InputValidator do
+  let(:test_class) do
+    Class.new do
+      include Vmpooler::API::InputValidator
+    end
+  end
+  let(:validator) { test_class.new }
+
+  describe '#validate_hostname' do
+    it 'accepts valid hostnames' do
+      expect(validator.validate_hostname('test-host.example.com')).to be true
+      expect(validator.validate_hostname('host123')).to be true
+    end
+
+    it 'rejects invalid hostnames' do
+      result = validator.validate_hostname('invalid_host!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid hostname format')
+    end
+
+    it 'rejects hostnames that are too long' do
+      long_hostname = 'a' * 300
+      result = validator.validate_hostname(long_hostname)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+
+    it 'rejects empty hostnames' do
+      result = validator.validate_hostname('')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('required')
+    end
+  end
+
+  describe '#validate_pool_name' do
+    it 'accepts valid pool names' do
+      expect(validator.validate_pool_name('centos-7-x86_64')).to be true
+      expect(validator.validate_pool_name('ubuntu-2204')).to be true
+    end
+
+    it 'rejects invalid pool names' do
+      result = validator.validate_pool_name('invalid pool!')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid pool name format')
+    end
+
+    it 'rejects pool names that are too long' do
+      result = validator.validate_pool_name('a' * 150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_tag' do
+    it 'accepts valid tags' do
+      expect(validator.validate_tag('project', 'test-123')).to be true
+      expect(validator.validate_tag('owner', 'user@example.com')).to be true
+    end
+
+    it 'rejects tags with invalid keys' do
+      result = validator.validate_tag('invalid key!', 'value')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid tag key format')
+    end
+
+    it 'rejects tags with invalid characters in value' do
+      result = validator.validate_tag('key', 'value<script>')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('invalid characters')
+    end
+
+    it 'rejects tags that are too long' do
+      result = validator.validate_tag('key', 'a' * 300)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too long')
+    end
+  end
+
+  describe '#validate_vm_count' do
+    it 'accepts valid VM counts' do
+      expect(validator.validate_vm_count(5)).to eq(5)
+      expect(validator.validate_vm_count('10')).to eq(10)
+    end
+
+    it 'rejects counts less than 1' do
+      result = validator.validate_vm_count(0)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at least 1')
+    end
+
+    it 'rejects counts greater than 100' do
+      result = validator.validate_vm_count(150)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 100')
+    end
+
+    it 'rejects non-integer values' do
+      result = validator.validate_vm_count('abc')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('valid integer')
+    end
+  end
+
+  describe '#validate_disk_size' do
+    it 'accepts valid disk sizes' do
+      expect(validator.validate_disk_size(50)).to eq(50)
+      expect(validator.validate_disk_size('100')).to eq(100)
+    end
+
+    it 'rejects sizes less than 1' do
+      result = validator.validate_disk_size(0)
+      expect(result['ok']).to be false
+    end
+
+    it 'rejects sizes greater than 2048' do
+      result = validator.validate_disk_size(3000)
+      expect(result['ok']).to be false
+    end
+  end
+
+  describe '#validate_lifetime' do
+    it 'accepts valid lifetimes' do
+      expect(validator.validate_lifetime(24)).to eq(24)
+      expect(validator.validate_lifetime('48')).to eq(48)
+    end
+
+    it 'rejects lifetimes greater than 168 hours (1 week)' do
+      result = validator.validate_lifetime(200)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('at most 168')
+    end
+  end
+
+  describe '#sanitize_json_body' do
+    it 'parses valid JSON' do
+      result = validator.sanitize_json_body('{"key": "value"}')
+      expect(result).to eq('key' => 'value')
+    end
+
+    it 'rejects invalid JSON' do
+      result = validator.sanitize_json_body('{invalid}')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('Invalid JSON')
+    end
+
+    it 'rejects non-object JSON' do
+      result = validator.sanitize_json_body('["array"]')
+      expect(result['ok']).to be false
+      expect(result['error']).to include('must be a JSON object')
+    end
+
+    it 'rejects deeply nested JSON' do
+      deep_json = '{"a":{"b":{"c":{"d":{"e":{"f":"too deep"}}}}}}'
+      result = validator.sanitize_json_body(deep_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too complex')
+    end
+
+    it 'rejects bodies that are too large' do
+      large_json = '{"data":"' + ('a' * 20000) + '"}'
+      result = validator.sanitize_json_body(large_json)
+      expect(result['ok']).to be false
+      expect(result['error']).to include('too large')
+    end
+  end
+
+  describe '#validation_error?' do
+    it 'returns true for error responses' do
+      error = { 'ok' => false, 'error' => 'test error' }
+      expect(validator.validation_error?(error)).to be true
+    end
+
+    it 'returns false for successful responses' do
+      expect(validator.validation_error?(true)).to be false
+      expect(validator.validation_error?(5)).to be false
+    end
+  end
+end
diff --git a/spec/unit/collector_middleware_spec.rb b/spec/unit/collector_middleware_spec.rb
index 5244c25..704ac5a 100644
--- a/spec/unit/collector_middleware_spec.rb
+++ b/spec/unit/collector_middleware_spec.rb
@@ -72,14 +72,14 @@ describe Vmpooler::Metrics::Promstats::CollectorMiddleware do
   it 'normalizes paths containing /ondemandvm by ' do
     expect(Benchmark).to receive(:realtime).and_yield.and_return(0.3)
 
-    get '/foo/ondemand/bar/fatman'
+    get '/foo/ondemandvm/bar/fatman'
 
     metric = :http_server_requests_total
-    labels = { method: 'get', path: '/foo/ondemand', code: '200' }
+    labels = { method: 'get', path: '/foo/ondemandvm', code: '200' }
     expect(registry.get(metric).get(labels: labels)).to eql(1.0)
 
     metric = :http_server_request_duration_seconds
-    labels = { method: 'get', path: '/foo/ondemand' }
+    labels = { method: 'get', path: '/foo/ondemandvm' }
     expect(registry.get(metric).get(labels: labels)).to include("0.1" => 0, "0.5" => 1)
   end
 
diff --git a/spec/unit/dns/base_spec.rb b/spec/unit/dns/base_spec.rb
new file mode 100644
index 0000000..9fb656b
--- /dev/null
+++ b/spec/unit/dns/base_spec.rb
@@ -0,0 +1,172 @@
+require 'spec_helper'
+require 'vmpooler/dns/base'
+
+# This spec does not really exercise code paths but is merely used
+# to enforce that certain methods are defined in the base classes
+
+describe 'Vmpooler::PoolManager::Dns::Base' do
+  let(:logger) { MockLogger.new }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { {} }
+  let(:dns_plugin_name) { 'base' }
+  let(:dns_options) { { 'param' => 'value' } }
+
+  let(:fake_vm) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
+  }
+
+  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
+    metrics: metrics,
+    connpool_type: 'redis_connection_pool',
+    connpool_provider: 'testprovider',
+    size: 1,
+    timeout: 5
+  ) { MockRedis.new }
+  }
+
+  subject { Vmpooler::PoolManager::Dns::Base.new(config, logger, metrics, redis_connection_pool, dns_plugin_name, dns_options) }
+
+  # Helper attr_reader methods
+  describe '#logger' do
+    it 'should come from the provider initialization' do
+      expect(subject.logger).to be(logger)
+    end
+  end
+
+  describe '#metrics' do
+    it 'should come from the provider initialization' do
+      expect(subject.metrics).to be(metrics)
+    end
+  end
+
+  describe '#dns_options' do
+    it 'should come from the provider initialization' do
+      expect(subject.dns_options).to be(dns_options)
+    end
+  end
+
+  describe '#pool_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: '#{poolname}'
+    alias: [ 'mockpool' ]
+    template: 'Templates/pool1'
+    folder: 'Pooler/pool1'
+    datastore: 'datastore0'
+    size: 5
+    timeout: 10
+    ready_ttl: 1440
+    clone_target: 'cluster1'
+EOT
+      )
+    }
+    context 'Given a pool that does not exist' do
+      it 'should return nil' do
+        expect(subject.pool_config('missing_pool')).to be_nil
+      end
+    end
+
+    context 'Given a pool that does exist' do
+      it 'should return the pool\'s configuration' do
+        result = subject.pool_config(poolname)
+        expect(result['name']).to eq(poolname)
+      end
+    end
+  end
+
+  describe '#dns_config' do
+    let(:poolname) { 'pool1' }
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :#{dns_plugin_name}:
+    option1: 'value1'
+EOT
+      )
+    }
+
+    context 'Given a dns plugin with no configuration' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :bad_dns:
+    option1: 'value1'
+    option2: 'value1'
+EOT
+      )
+    }
+      it 'should return nil' do
+        expect(subject.dns_config).to be_nil
+      end
+    end
+
+    context 'Given a correct dns config name' do
+      it 'should return the dns\'s configuration' do
+        result = subject.dns_config
+        expect(result['option1']).to eq('value1')
+      end
+    end
+  end
+
+  describe '#global_config' do
+    it 'should come from the dns initialization' do
+      expect(subject.global_config).to be(config)
+    end
+  end
+
+  describe '#name' do
+    it "should come from the dns initialization" do
+      expect(subject.name).to eq(dns_plugin_name)
+    end
+  end
+
+  describe '#get_ip' do
+    it 'calls redis hget with vm name and ip' do
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:hget).with("vmpooler__vm__vm1", 'ip')
+      end
+      subject.get_ip(fake_vm['name'])
+    end
+  end
+
+  describe '#provided_pools' do
+    let(:config) { YAML.load(<<-EOT
+---
+:pools:
+  - name: 'pool1'
+    dns_config: 'base'
+  - name: 'pool2'
+    dns_config: 'base'
+  - name: 'otherpool'
+    dns_config: 'other provider'
+  - name: 'no name'
+EOT
+      )
+    }
+
+    it "should return pools serviced by this provider" do
+      expect(subject.provided_pools).to eq(['pool1','pool2'])
+    end
+  end
+
+  describe '#create_or_replace_record' do
+    it 'should raise error' do
+      expect{subject.create_or_replace_record('pool')}.to raise_error(/does not implement create_or_replace_record/)
+    end
+  end
+
+  describe '#delete_record' do
+    it 'should raise error' do
+      expect{subject.delete_record('pool')}.to raise_error(/does not implement delete_record/)
+    end
+  end
+end
diff --git a/spec/unit/dns_spec.rb b/spec/unit/dns_spec.rb
new file mode 100644
index 0000000..3e02704
--- /dev/null
+++ b/spec/unit/dns_spec.rb
@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe 'Vmpooler::Dns' do
+  let(:dns_class) { 'mock-dnsservice' }
+  let(:dns_config_name) { 'mock' }
+  let(:pool) { 'pool1' }
+  let(:config) { YAML.load(<<~EOT
+  ---
+  :dns_configs:
+    :mock:
+      dns_class: 'mock'
+      domain: 'example.com'
+  :pools:
+    - name: 'pool1'
+      dns_plugin: 'mock'
+EOT
+  )}
+  subject { Vmpooler::Dns.new }
+
+  describe '.get_dns_plugin_class_by_name' do
+    it 'returns the plugin class for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_class_by_name(config, dns_config_name)
+      expect(result).to eq('mock')
+    end
+  end
+
+  describe '.get_domain_for_pool' do
+    it 'returns the domain for the specified pool' do
+      result = Vmpooler::Dns.get_domain_for_pool(config, pool)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_domain_by_name' do
+    it 'returns the domain for the specified config' do
+      result = Vmpooler::Dns.get_dns_plugin_domain_by_name(config, dns_config_name)
+      expect(result).to eq('example.com')
+    end
+  end
+
+  describe '.get_dns_plugin_config_classes' do
+    it 'returns the list of dns plugin classes' do
+      result = Vmpooler::Dns.get_dns_plugin_config_classes(config)
+      expect(result).to eq(['mock'])
+    end
+  end
+
+  describe '#load_from_gems' do
+    let(:gem_name) { 'mock-dnsservice' }
+    let(:translated_gem_name) { 'mock/dnsservice' }
+
+    before(:each) do
+      allow(subject).to receive(:require).with(gem_name).and_return(true)
+    end
+
+    it 'loads the specified gem' do
+      expect(subject).to receive(:require).with("vmpooler/dns/#{translated_gem_name}")
+      result = subject.load_from_gems(gem_name)
+      expect(result).to eq("vmpooler/dns/#{translated_gem_name}")
+    end
+  end
+end
diff --git a/spec/unit/env_config.rb b/spec/unit/env_config.rb
index 0eba2fb..1529cf7 100644
--- a/spec/unit/env_config.rb
+++ b/spec/unit/env_config.rb
@@ -22,13 +22,12 @@ describe 'Vmpooler' do
         ['prefix', test_string, ""],
         ['logfile', test_string, nil],
         ['site_name', test_string, nil],
-        ['domain', test_string, nil],
         ['clone_target', test_string, nil],
         ['create_folders', test_bool, nil],
         ['create_template_delta_disks', test_bool, nil],
         ['create_linked_clones', test_bool, nil],
         ['experimental_features', test_bool, nil],
-        ['purge_unconfigured_folders', test_bool, nil],
+        ['purge_unconfigured_resources', test_bool, nil],
         ['usage_stats', test_bool, nil],
         ['request_logger', test_bool, nil],
         ['extra_config', test_string, nil],
diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb
index 3fb44da..1b2ccef 100644
--- a/spec/unit/pool_manager_spec.rb
+++ b/spec/unit/pool_manager_spec.rb
@@ -14,12 +14,14 @@ describe 'Pool Manager' do
   let(:pool) { 'pool1' }
   let(:vm) { 'vm1' }
   let(:timeout) { 5 }
+  let(:timeout_notification) { 2 }
   let(:host) { double('host') }
   let(:token) { 'token1234' }
   let(:request_id) { '1234' }
   let(:current_time) { Time.now }
 
   let(:provider_options) { {} }
+  let(:dns_options) { {} }
   let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
     metrics: metrics,
     connpool_type: 'redis_connection_pool',
@@ -30,16 +32,19 @@ describe 'Pool Manager' do
   }
 
   let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'mock_provider', provider_options) }
-
+  let(:dns_plugin) { MockPoolManagerDnsBase.new }
   let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
+:dns_configs: {}
 :providers:
   :mock:
+    dns_class: base
 :redis: {}
 :pools:
   - name: '#{pool}'
     size: 1
+    dns_plugin: 'mock'
 EOT
     )
   }
@@ -70,14 +75,112 @@ EOT
     )
     }
     it do
-      files = ["#{project_root_dir}/lib/vmpooler/providers/vsphere.rb",
-               "#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
+      files = ["#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
       expect(subject.load_used_providers).to eq(files)
     end
   end
 
+  describe '#load_used_dns_plugins' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :base:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+    )
+    }
+    it do
+      files = ['vmpooler/dns/base']
+      expect(subject.load_used_dns_plugins).to eq(files)
+    end
+
+
+  end
+
+  describe '#used_providers' do
+    context 'with no named providers' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+    context 'with one named provider without a provider_class key' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+
+    context 'with one named provider with a provider_class key' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+    provider_class: 'mock_mock'
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock_mock", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+
+    context 'with one named provider with a provider_class key and one without' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+    provider_class: 'mock_mock'
+  :foo:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+      )
+      }
+      it do
+        result = ["mock_mock", "foo", "dummy"]
+        expect(subject.used_providers).to eq(result)
+      end
+    end
+  end
+
   it '#default_providers' do
-    expect(subject.default_providers).to eq(['vsphere', 'dummy'])
+    expect(subject.default_providers).to eq(['dummy'])
   end
 
   describe '#check_pending_vm' do
@@ -87,9 +190,9 @@ EOT
 
     it 'calls _check_pending_vm' do
       expect(Thread).to receive(:new).and_yield
-      expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,provider)
+      expect(subject).to receive(:_check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)
 
-      subject.check_pending_vm(vm, pool, timeout, provider)
+      subject.check_pending_vm(vm, pool, timeout, timeout_notification, provider)
     end
   end
 
@@ -101,21 +204,21 @@ EOT
     context 'host is in pool' do
 
       it 'calls move_pending_vm_to_ready if host is ready' do
-        expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(true)
         redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(true)
           expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool, redis, nil)
         end
 
-        subject._check_pending_vm(vm, pool, timeout, provider)
+        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       end
 
       it 'calls fail_pending_vm if host is not ready' do
-        expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(false)
         redis_connection_pool.with do |redis|
-          expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, redis)
+          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(false)
+          expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, timeout_notification, redis)
         end
 
-        subject._check_pending_vm(vm, pool, timeout, provider)
+        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       end
     end
 
@@ -128,13 +231,15 @@ EOT
       it 'should return' do
         expect(subject).to receive(:vm_mutex).and_return(mutex)
 
-        expect(subject._check_pending_vm(vm, pool, timeout, provider)).to be_nil
+        expect(subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)).to be_nil
       end
     end
   end
 
   describe '#remove_nonexistent_vm' do
     before do
+      expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      expect(dns_plugin).to receive(:delete_record)
       expect(subject).not_to be_nil
     end
 
@@ -170,14 +275,16 @@ EOT
 
     it 'takes no action if VM is not cloning' do
       redis_connection_pool.with do |redis|
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis)).to eq(true)
+        expect(logger).to_not receive(:log)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
       end
     end
 
     it 'takes no action if VM is within timeout' do
       redis_connection_pool.with do |redis|
         redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis)).to eq(true)
+        expect(logger).to_not receive(:log)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
         expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
       end
     end
@@ -185,7 +292,7 @@ EOT
     it 'moves VM to completed queue if VM has exceeded timeout and exists' do
       redis_connection_pool.with do |redis|
         redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis, true)).to eq(true)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
         expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
         expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
       end
@@ -194,23 +301,24 @@ EOT
     it 'logs message if VM has exceeded timeout and exists' do
       redis_connection_pool.with do |redis|
         redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
-        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis, true)).to eq(true)
+        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: ")
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
       end
     end
 
     it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do
       redis_connection_pool.with do |redis|
         redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
+        expect(logger).to_not receive(:log)
         expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool, redis)
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis, false)).to eq(true)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: false)).to eq(true)
       end
     end
 
     it 'swallows error if an error is raised' do
       redis_connection_pool.with do |redis|
         redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
-        expect(subject.fail_pending_vm(vm, pool, timeout, redis, true)).to eq(false)
+        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(false)
       end
     end
 
@@ -219,7 +327,7 @@ EOT
         redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
         expect(logger).to receive(:log).with('d', String)
 
-        subject.fail_pending_vm(vm, pool, timeout, redis, true)
+        subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
       end
     end
 
@@ -230,13 +338,130 @@ EOT
           redis.hset("vmpooler__vm__#{vm}", 'clone',(Time.now - 900).to_s)
           redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
           redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
-          subject.fail_pending_vm(vm, pool, timeout, redis, true)
+          subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
           expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to eq(["#{pool}:#{pool}:1:#{request_id}"])
         end
       end
     end
   end
 
+  describe '#handle_timed_out_vm' do
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    before(:each) do
+      redis_connection_pool.with do |redis|
+        create_pending_vm(pool, vm, redis)
+        config[:config]['max_vm_retries'] = 3
+      end
+    end
+
+    context 'without request_id' do
+      it 'moves VM to completed queue and returns error' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'open_socket_error', 'connection failed')
+          result = subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
+          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
+          expect(result).to eq('connection failed')
+        end
+      end
+    end
+
+    context 'with request_id and transient error' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'network timeout')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'Timeout::Error')
+        end
+      end
+
+      it 'retries on first failure' do
+        redis_connection_pool.with do |redis|
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count')).to eq('1')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+
+      it 'marks as failed after max retries' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__odrequest__#{request_id}", 'retry_count', '3')
+          
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to eq('Max retry attempts exceeded')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+    end
+
+    context 'with request_id and permanent error' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'template not found')
+          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'RuntimeError')
+        end
+      end
+
+      it 'immediately marks as failed without retrying' do
+        redis_connection_pool.with do |redis|
+          subject.handle_timed_out_vm(vm, pool, redis)
+          
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
+          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to include('Configuration error')
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
+        end
+      end
+    end
+  end
+
+  describe '#permanent_error?' do
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    it 'identifies template not found errors as permanent' do
+      expect(subject.permanent_error?('template not found', 'RuntimeError')).to be(true)
+    end
+
+    it 'identifies invalid path errors as permanent' do
+      expect(subject.permanent_error?('invalid path specified', 'ArgumentError')).to be(true)
+    end
+
+    it 'identifies permission denied errors as permanent' do
+      expect(subject.permanent_error?('permission denied', 'SecurityError')).to be(true)
+    end
+
+    it 'identifies ArgumentError class as permanent' do
+      expect(subject.permanent_error?('some argument error', 'ArgumentError')).to be(true)
+    end
+
+    it 'identifies network errors as transient' do
+      expect(subject.permanent_error?('connection timeout', 'Timeout::Error')).to be(false)
+    end
+
+    it 'identifies socket errors as transient' do
+      expect(subject.permanent_error?('connection refused', 'Errno::ECONNREFUSED')).to be(false)
+    end
+
+    it 'returns false for nil inputs' do
+      expect(subject.permanent_error?(nil, nil)).to be(false)
+      expect(subject.permanent_error?('error', nil)).to be(false)
+      expect(subject.permanent_error?(nil, 'Error')).to be(false)
+    end
+  end
+
   describe '#move_pending_vm_to_ready' do
     let(:host) { { 'hostname' => vm }}
 
@@ -401,10 +626,10 @@ EOT
         it 'should specify auth data on the vm' do
           redis_connection_pool.with do |redis|
             allow(redis).to receive(:hset)
-            expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'token:token', token)
-            expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'token:user', user)
 
             subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'token:token')).to eq(token)
+            expect(redis.hget("vmpooler__vm__#{vm}", 'token:user')).to eq(user)
           end
         end
       end
@@ -524,7 +749,7 @@ EOT
         end
 
         it 'should log messages about being unreachable' do
-          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
+          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' removed from 'ready' queue. vm unreachable with error: ")
 
           subject._check_ready_vm(vm, pool, ttl, provider)
         end
@@ -684,6 +909,25 @@ EOT
           expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
         end
       end
+
+      it 'should try to tag if it has not been done' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          expect(provider).to receive(:tag_vm_user)
+          subject._check_running_vm(vm, pool, 0, provider)
+        end
+      end
+
+      it 'should not try to tag if it has been done' do
+        redis_connection_pool.with do |redis|
+          expect(provider).to receive(:vm_ready?).and_return(true)
+          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
+          redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true')
+          expect(provider).not_to receive(:tag_vm_user)
+          subject._check_running_vm(vm, pool, 0, provider)
+        end
+      end
     end
 
     context 'with a locked vm mutex' do
@@ -759,17 +1003,78 @@ EOT
     end
 
     it 'calls _clone_vm' do
-      expect(subject).to receive(:_clone_vm).with(pool_object,provider,nil,nil)
+      expect(subject).to receive(:_clone_vm).with(pool_object,provider,dns_plugin,nil,nil)
 
-      subject.clone_vm(pool_object,provider)
+      subject.clone_vm(pool_object,provider,dns_plugin)
     end
 
     it 'logs a message if an error is raised' do
       allow(logger).to receive(:log)
       expect(logger).to receive(:log).with('s',"[!] [#{pool}] failed while cloning VM with an error: MockError")
-      expect(subject).to receive(:_clone_vm).with(pool,provider,nil,nil).and_raise('MockError')
+      expect(subject).to receive(:_clone_vm).with(pool,provider,dns_plugin,nil,nil).and_raise('MockError')
 
-      expect{subject.clone_vm(pool,provider)}.to raise_error(/MockError/)
+      expect{subject.clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
+    end
+  end
+
+  describe '#find_unique_hostname' do
+    # it should return the hostname
+    before do
+      allow(subject).to receive(:generate_and_check_hostname).and_return(["spicy-proton", true])
+      allow(subject).to receive(:get_provider_for_pool).and_return(provider)
+    end
+
+    context 'with a setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:providers:
+  :mock_provider:
+    skip_dns_check_before_creating_vm: true
+:pools:
+  - name: '#{pool}'
+    size: 1
+    dns_config: 'mock'
+        EOT
+        )
+      }
+      it 'should skip the dns check' do
+        #method is skipped
+        expect(subject).not_to receive(:check_dns_available)
+        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+    end
+    context 'without the setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:providers:
+  :mock_provider:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    dns_config: 'mock'
+        EOT
+        )
+      }
+      it 'should run the dns check and pass' do
+        #method is skipped
+        expect(subject).to receive(:check_dns_available).and_return(["1.1.1.1",true])
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+
+      it 'should run the dns check and fail' do
+        #method is skipped
+        allow(subject).to receive(:check_dns_available).and_return(["1.1.1.1",false]).exactly(3).times
+        expect{subject.find_unique_hostname(pool)}.to raise_error(RuntimeError)
+      end
     end
   end
 
@@ -784,10 +1089,17 @@ EOT
     let(:config) {
       YAML.load(<<-EOT
 ---
+:dns_configs:
+  :mock:
+    dns_class: base
 :config:
   prefix: "prefix"
 :redis:
   ttl: #{redis_ttl}
+:pools:
+  - name: 'pool1'
+    size: 1
+    dns_config: 'mock'
 EOT
       )
     }
@@ -795,8 +1107,12 @@ EOT
     context 'with no errors during cloning' do
       before(:each) do
         allow(metrics).to receive(:timing)
-        expect(metrics).to receive(:timing).with(/clone\./,/0/)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
         expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).and_return(1)
+        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
         allow(logger).to receive(:log)
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
@@ -807,7 +1123,7 @@ EOT
         redis_connection_pool.with do |redis|
           expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
 
-          subject._clone_vm(pool,provider)
+          subject._clone_vm(pool,provider,dns_plugin)
 
           expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1)
           expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil
@@ -822,7 +1138,7 @@ EOT
           redis.incr('vmpooler__tasks__clone')
           redis.incr('vmpooler__tasks__clone')
           expect(redis.get('vmpooler__tasks__clone')).to eq('2')
-          subject._clone_vm(pool,provider)
+          subject._clone_vm(pool,provider,dns_plugin)
           expect(redis.get('vmpooler__tasks__clone')).to eq('1')
         end
       end
@@ -830,19 +1146,32 @@ EOT
       it 'should log a message that is being cloned from a template' do
         expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] Starting to clone '(.+)'/)
 
-        subject._clone_vm(pool,provider)
+        subject._clone_vm(pool,provider,dns_plugin)
       end
 
       it 'should log a message that it completed being cloned' do
         expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned in [0-9.]+ seconds/)
 
-        subject._clone_vm(pool,provider)
+        subject._clone_vm(pool,provider,dns_plugin)
+      end
+    end
+
+    context 'with a failure to get ip address after cloning' do
+      it 'should log a message that it completed being cloned' do
+        allow(metrics).to receive(:timing)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
+        expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).and_return(nil)
+
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(StandardError)
       end
     end
 
     context 'with an error during cloning' do
       before(:each) do
         expect(provider).to receive(:create_vm).with(pool, String).and_raise('MockError')
+        allow(subject).to receive(:get_domain_for_pool)
         allow(logger).to receive(:log)
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
@@ -853,7 +1182,7 @@ EOT
         redis_connection_pool.with do |redis|
           expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
 
-          expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
 
           expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
           # Get the new VM Name from the pending pool queue as it should be the only entry
@@ -867,20 +1196,22 @@ EOT
           redis.incr('vmpooler__tasks__clone')
           redis.incr('vmpooler__tasks__clone')
           expect(redis.get('vmpooler__tasks__clone')).to eq('2')
-          expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
           expect(redis.get('vmpooler__tasks__clone')).to eq('1')
         end
       end
 
       it 'should expire the vm metadata' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:expire)
-          expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire)
+          end
+          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
         end
       end
 
       it 'should raise the error' do
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
       end
 
     end
@@ -888,8 +1219,12 @@ EOT
     context 'with request_id' do
       before(:each) do
         allow(metrics).to receive(:timing)
-        expect(metrics).to receive(:timing).with(/clone\./,/0/)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
         expect(provider).to receive(:create_vm).with(pool, String)
+        allow(provider).to receive(:get_vm_ip_address).with(vm,pool).and_return(1)
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
+        expect(dns_plugin).to receive(:create_or_replace_record)
         allow(logger).to receive(:log)
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
@@ -898,7 +1233,7 @@ EOT
 
       it 'should set request_id and pool_alias on the vm data' do
         redis_connection_pool.with do |redis|
-          subject._clone_vm(pool,provider,request_id,pool)
+          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
           expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool)
           expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id)
         end
@@ -907,7 +1242,7 @@ EOT
       it 'should reduce the clone count' do
         redis_connection_pool.with do |redis|
           expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone')
-          subject._clone_vm(pool,provider,request_id,pool)
+          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
         end
       end
     end
@@ -915,22 +1250,25 @@ EOT
     context 'with #check_dns_available' do
       before(:each) do
         allow(logger).to receive(:log)
+        allow(provider).to receive(:get_vm_ip_address).and_return(true)
+        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
       end
       it 'should error out if DNS already exists' do
         vm_name = "foo"
         resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
         expect(subject).to receive(:generate_and_check_hostname).exactly(3).times.and_return([vm_name, true]) #skip this, make it available all times
         expect(resolv).to receive(:getaddress).exactly(3).times.and_return("1.2.3.4")
-        expect(metrics).to receive(:increment).with("errors.staledns.#{pool}").exactly(3).times
-        expect{subject._clone_vm(pool,provider)}.to raise_error(/Unable to generate a unique hostname after/)
+        expect(metrics).to receive(:increment).with("vmpooler_errors.staledns.#{pool}").exactly(3).times
+        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/Unable to generate a unique hostname after/)
       end
       it 'should be successful if DNS does not exist' do
         vm_name = "foo"
         resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
         expect(subject).to receive(:generate_and_check_hostname).and_return([vm_name, true])
         expect(resolv).to receive(:getaddress).exactly(1).times.and_raise(Resolv::ResolvError)
+        expect(provider).to receive(:get_vm_ip_address)
         expect(provider).to receive(:create_vm).with(pool, String)
-        subject._clone_vm(pool,provider)
+        subject._clone_vm(pool,provider,dns_plugin)
       end
     end
   end
@@ -942,17 +1280,17 @@ EOT
     end
 
     it 'calls _destroy_vm' do
-      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider)
+      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin)
 
-      subject.destroy_vm(vm,pool,provider)
+      subject.destroy_vm(vm,pool,provider,dns_plugin)
     end
 
     it 'logs a message if an error is raised' do
       allow(logger).to receive(:log)
       expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: MockError")
-      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider).and_raise('MockError')
+      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin).and_raise('MockError')
 
-      expect{subject.destroy_vm(vm,pool,provider)}.to raise_error(/MockError/)
+      expect{subject.destroy_vm(vm,pool,provider,dns_plugin)}.to raise_error(/MockError/)
     end
   end
 
@@ -978,8 +1316,10 @@ EOT
 
       it 'should call redis expire with 0' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:expire).with("vmpooler__vm__#{vm}", 0)
-          subject._destroy_vm(vm,pool,provider)
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__vm__#{vm}", 0)
+          end
+          subject._destroy_vm(vm,pool,provider,dns_plugin)
         end
       end
     end
@@ -990,7 +1330,7 @@ EOT
       end
 
       it 'should raise an error' do
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(NoMethodError)
       end
     end
 
@@ -1001,7 +1341,7 @@ EOT
       end
 
       it 'should not raise an error' do
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
     end
 
@@ -1010,29 +1350,22 @@ EOT
         expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/)
         allow(logger).to receive(:log)
 
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
 
       it 'should emit a timing metric' do
         allow(subject).to receive(:get_vm_usage_labels)
         allow(metrics).to receive(:timing)
-        expect(metrics).to receive(:timing).with("destroy.#{pool}", String)
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with("vmpooler_destroy.#{pool}", String)
 
-        subject._destroy_vm(vm,pool,provider)
-      end
-
-      it 'should check usage labels' do
-        redis_connection_pool.with do |redis|
-          expect(subject).to receive(:get_vm_usage_labels).with(vm, redis)
-        end
-
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
 
       it 'should dereference the mutex' do
         expect(subject).to receive(:dereference_mutex)
 
-        subject._destroy_vm(vm,pool,provider)
+        subject._destroy_vm(vm,pool,provider,dns_plugin)
       end
     end
 
@@ -1046,13 +1379,13 @@ EOT
         expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/).exactly(0).times
         allow(logger).to receive(:log)
 
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
       end
 
       it 'should not emit a timing metric' do
         expect(metrics).to receive(:timing).with("destroy.#{pool}", String).exactly(0).times
 
-        expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/)
+        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
       end
     end
 
@@ -1065,287 +1398,12 @@ EOT
       it 'should return' do
         expect(subject).to receive(:vm_mutex).with(vm).and_return(mutex)
 
-        expect(subject._destroy_vm(vm,pool,provider)).to eq(nil)
+        expect(subject._destroy_vm(vm,pool,provider,dns_plugin)).to eq(nil)
       end
     end
   end
 
-  describe '#get_vm_usage_labels' do
-
-    let(:template) { 'pool1' }
-    let(:user) { 'vmpuser' }
-    let(:vm) { 'vm1' }
-
-    context 'when label evaluation is disabled' do
-      it 'should do nothing' do
-        redis_connection_pool.with do |redis|
-          subject.get_vm_usage_labels(vm, redis)
-        end
-      end
-    end
-
-    context 'when label evaluation is enabled' do
-
-      before(:each) do
-        config[:config]['usage_stats'] = true
-      end
-
-      context 'when a VM has not been checked out' do
-        before(:each) do
-          redis_connection_pool.with do |redis|
-            create_ready_vm(template, vm, redis)
-          end
-        end
-
-        it 'should return' do
-          expect(subject).to receive(:get_vm_usage_labels).and_return(nil)
-
-          redis_connection_pool.with do |redis|
-            subject.get_vm_usage_labels(vm, redis)
-          end
-        end
-      end
-
-      context 'when a VM has been checked out' do
-
-        context 'without auth' do
-
-          before(:each) do
-            redis_connection_pool.with do |redis|
-              create_running_vm(template, vm, redis)
-            end
-          end
-
-          it 'should emit a metric' do
-            redis_connection_pool.with do |redis|
-              expect(metrics).to receive(:increment).with("user.unauthenticated.#{template}")
-
-              subject.get_vm_usage_labels(vm, redis)
-            end
-          end
-        end
-
-        context 'with auth' do
-
-          before(:each) do
-            redis_connection_pool.with do |redis|
-              create_running_vm(template, vm, redis, token, user)
-            end
-          end
-
-          it 'should emit a metric' do
-            expect(metrics).to receive(:increment).with("user.#{user}.#{template}")
-            expect(metrics).not_to receive(:increment)
-
-            redis_connection_pool.with do |redis|
-              subject.get_vm_usage_labels(vm, redis)
-            end
-          end
-
-          context 'with a user with period in name' do
-            let(:user) { 'test.user'.gsub('.', '_') }
-            let(:metric_string) { "user.#{user}.#{template}" }
-            let(:metric_nodes) { metric_string.split('.') }
-
-            before(:each) do
-              redis_connection_pool.with do |redis|
-                create_running_vm(template, vm, redis)
-              end
-            end
-
-            it 'should emit a metric with the character replaced' do
-              expect(metrics).to receive(:increment).with(metric_string)
-              expect(metrics).not_to receive(:increment)
-
-              redis_connection_pool.with do |redis|
-                subject.get_vm_usage_labels(vm, redis)
-              end
-            end
-
-            it 'should include three nodes' do
-              expect(metric_nodes.count).to eq(3)
-            end
-          end
-
-          context 'with a jenkins_build_url label' do
-            let(:jenkins_build_url) { 'https://jenkins.example.com/job/enterprise_pe-acceptance-tests_integration-system_pe_full-agent-upgrade_weekend_2018.1.x/LAYOUT=centos6-64mcd-ubuntu1404-32f-64f,LEGACY_AGENT_VERSION=NONE,PLATFORM=NOTUSED,SCM_BRANCH=2018.1.x,UPGRADE_FROM=2018.1.0,UPGRADE_TO_VERSION=NONE,label=beaker/222/' }
-            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-            let(:instance) { url_parts[0] }
-            let(:value_stream_parts) { url_parts[2].split('_') }
-            let(:value_stream) { value_stream_parts.shift }
-            let(:branch) { value_stream_parts.pop }
-            let(:project) { value_stream_parts.shift }
-            let(:job_name) { value_stream_parts.join('_') }
-
-            let(:metric_string_1) { "user.#{user}.#{template}" }
-            let(:metric_string_2) { "usage_jenkins_instance.#{instance.gsub('.', '_')}.#{value_stream.gsub('.', '_')}.#{template}" }
-            let(:metric_string_3) { "usage_branch_project.#{branch.gsub('.', '_')}.#{project.gsub('.', '_')}.#{template}" }
-            let(:metric_string_4) { "usage_job_component.#{job_name.gsub('.', '_')}.none.#{template}" }
-
-            before(:each) do
-              redis_connection_pool.with do |redis|
-                create_tag(vm, 'jenkins_build_url', jenkins_build_url, redis)
-              end
-            end
-
-            it 'should emit 4 metric withs information from the URL' do
-              expect(metrics).to receive(:increment).with(metric_string_1)
-              expect(metrics).to receive(:increment).with(metric_string_2)
-              expect(metrics).to receive(:increment).with(metric_string_3)
-              expect(metrics).to receive(:increment).with(metric_string_4)
-              expect(metrics).not_to receive(:increment)
-
-              redis_connection_pool.with do |redis|
-                subject.get_vm_usage_labels(vm, redis)
-              end
-            end
-          end
-
-          context 'with a jenkins_build_url that contains RMM_COMPONENT_TO_TEST_NAME' do
-            let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/' }
-            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-            let(:instance) { url_parts[0].gsub('.', '_') }
-            let(:value_stream_parts) { url_parts[2].split('_') }
-            let(:value_stream) { value_stream_parts.shift }
-            let(:branch) { value_stream_parts.pop }
-            let(:project) { value_stream_parts.shift }
-            let(:job_name) { value_stream_parts.join('_') }
-            let(:build_metadata) { url_parts[3] }
-            let(:build_component) { subject.component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata) }
-            let(:expected_string) { "usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{build_component}.#{template}" }
-            let(:metric_nodes) { expected_string.split('.') }
-
-            let(:metric_string_1) { "user.#{user}.#{template}" }
-            let(:metric_string_2) { "usage_jenkins_instance.#{instance.gsub('.', '_')}.#{value_stream.gsub('.', '_')}.#{template}" }
-            let(:metric_string_3) { "usage_branch_project.#{branch.gsub('.', '_')}.#{project.gsub('.', '_')}.#{template}" }
-            let(:metric_string_4) { "usage_job_component.#{job_name.gsub('.', '_')}.#{build_component}.#{template}" }
-
-            before(:each) do
-              redis_connection_pool.with do |redis|
-                create_tag(vm, 'jenkins_build_url', jenkins_build_url, redis)
-              end
-            end
-
-            it 'should emit 4 metrics with information from the URL' do
-              expect(metrics).to receive(:increment).with(metric_string_1)
-              expect(metrics).to receive(:increment).with(metric_string_2)
-              expect(metrics).to receive(:increment).with(metric_string_3)
-              expect(metrics).to receive(:increment).with(metric_string_4)
-              expect(metrics).not_to receive(:increment)
-
-              redis_connection_pool.with do |redis|
-                subject.get_vm_usage_labels(vm, redis)
-              end
-            end
-
-            it 'should contain exactly nine nodes' do
-              expect(metric_nodes.count).to eq(9)
-            end
-
-            context 'when there is no matrix job information' do
-
-              let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/824/' }
-              let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
-              let(:instance) { url_parts[0].gsub('.', '_') }
-              let(:value_stream_parts) { url_parts[2].split('_') }
-              let(:value_stream) { value_stream_parts.shift }
-              let(:branch) { value_stream_parts.pop }
-              let(:project) { value_stream_parts.shift }
-              let(:job_name) { value_stream_parts.join('_') }
-
-              let(:metric_string_1) { "user.#{user}.#{template}" }
-              let(:metric_string_2) { "usage_jenkins_instance.#{instance.gsub('.', '_')}.#{value_stream.gsub('.', '_')}.#{template}" }
-              let(:metric_string_3) { "usage_branch_project.#{branch.gsub('.', '_')}.#{project.gsub('.', '_')}.#{template}" }
-              let(:metric_string_4) { "usage_job_component.#{job_name.gsub('.', '_')}.none.#{template}" }
-  
-              before(:each) do
-                redis_connection_pool.with do |redis|
-                  create_tag(vm, 'jenkins_build_url', jenkins_build_url, redis)
-                end
-              end
-
-              it 'should emit 4 metrics with information from the URL without a build_component' do
-                expect(metrics).to receive(:increment).with(metric_string_1)
-                expect(metrics).to receive(:increment).with(metric_string_2)
-                expect(metrics).to receive(:increment).with(metric_string_3)
-                expect(metrics).to receive(:increment).with(metric_string_4)
-                expect(metrics).not_to receive(:increment)
-  
-                redis_connection_pool.with do |redis|
-                  subject.get_vm_usage_labels(vm, redis)
-                end
-              end
-            end
-
-
-          end
-
-          context 'with a litmus job' do
-            let(:jenkins_build_url) { 'https://litmus_manual' }
-
-            let(:metric_string_1) { "user.#{user}.#{template}" }
-            let(:metric_string_2) { "usage_litmus.#{user}.#{template}" }
-
-            before(:each) do
-              redis_connection_pool.with do |redis|
-                create_tag(vm, 'jenkins_build_url', jenkins_build_url, redis)
-              end
-            end
-
-            it 'should emit 2 metrics with the second indicating a litmus job' do
-              expect(metrics).to receive(:increment).with(metric_string_1)
-              expect(metrics).to receive(:increment).with(metric_string_2)
-              expect(metrics).not_to receive(:increment)
-
-              redis_connection_pool.with do |redis|
-                subject.get_vm_usage_labels(vm, redis)
-              end
-            end
-          end
-
-        end
-      end
-    end
-  end
-
-  describe '#component_to_test' do
-    let(:matching_key) { 'LABEL_ONE' }
-    let(:matching_value) { 'test' }
-    let(:labels_string) { "#{matching_key}=#{matching_value},LABEL_TWO=test2,LABEL_THREE=test3" }
-    let(:nonmatrix_string) { 'test,stuff,and,things' }
-
-    context 'when string contains a matching key' do
-      it 'should print the corresponding value' do
-        expect(subject.component_to_test(matching_key, labels_string)).to eq(matching_value)
-      end
-
-      context 'when match contains no value' do
-        it 'should return none' do
-          expect(subject.component_to_test(matching_key, matching_key)).to eq('none')
-        end
-      end
-    end
-
-    context 'when string contains no key value pairs' do
-      it 'should return' do
-        expect(subject.component_to_test(matching_key, nonmatrix_string)).to eq('none')
-      end
-    end
-
-    context 'when labels_string is a job number' do
-      it 'should return nil' do
-        expect(subject.component_to_test(matching_key, '25')).to eq('none')
-      end
-    end
-
-    context 'when labels_string is nil' do
-      it 'should return nil' do
-        expect(subject.component_to_test(matching_key, nil)).to be nil
-      end
-    end
-  end
-
-  describe '#purge_unused_vms_and_folders' do
+  describe '#purge_unused_vms_and_resources' do
     let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
@@ -1359,104 +1417,51 @@ EOT
     }
 
     it 'should return when purging is not enabled' do
-      expect(subject.purge_unused_vms_and_folders).to be_nil
+      expect(subject.purge_unused_vms_and_resources).to be_nil
     end
 
     context 'with purging enabled globally' do
       before(:each) do
-        config[:config]['purge_unconfigured_folders'] = true
+        config[:config]['purge_unconfigured_resources'] = true
         expect(Thread).to receive(:new).and_yield
       end
 
       it 'should run a purge for each provider' do
-        expect(subject).to receive(:purge_vms_and_folders)
+        expect(subject).to receive(:purge_vms_and_resources)
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
 
       it 'should log when purging fails' do
-        expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
+        expect(subject).to receive(:purge_vms_and_resources).and_raise(RuntimeError,'MockError')
         expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
     end
 
     context 'with purging enabled on the provider' do
       before(:each) do
-        config[:providers][:mock]['purge_unconfigured_folders'] = true
+        config[:providers][:mock]['purge_unconfigured_resources'] = true
         expect(Thread).to receive(:new).and_yield
       end
 
       it 'should run a purge for the provider' do
-        expect(subject).to receive(:purge_vms_and_folders)
+        expect(subject).to receive(:purge_vms_and_resources)
 
-        subject.purge_unused_vms_and_folders
+        subject.purge_unused_vms_and_resources
       end
     end
   end
 
-  describe '#pool_folders' do
-    let(:folder_name) { 'myinstance' }
-    let(:folder_base) { 'vmpooler' }
-    let(:folder) { [folder_base,folder_name].join('/') }
-    let(:datacenter) { 'dc1' }
-    let(:provider_name) { 'mock_provider' }
-    let(:expected_response) {
-      {
-        folder_name => "#{datacenter}/vm/#{folder_base}"
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:providers:
-  :mock:
-:pools:
-  - name: '#{pool}'
-    folder: '#{folder}'
-    size: 1
-    datacenter: '#{datacenter}'
-    provider: '#{provider_name}'
-  - name: '#{pool}2'
-    folder: '#{folder}'
-    size: 1
-    datacenter: '#{datacenter}'
-    provider: '#{provider_name}2'
-EOT
-      )
-    }
-
-    context 'when evaluating pool folders' do
-      before do
-        expect(subject).not_to be_nil
-        # Inject mock provider into global variable - Note this is a code smell
-        $providers = { provider_name => provider }
-      end
-
-      it 'should return a list of pool folders' do
-        expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
-
-        expect(subject.pool_folders(provider_name)).to eq(expected_response)
-      end
-
-      it 'should raise an error when the provider fails to get the datacenter' do
-        expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
-
-        expect{ subject.pool_folders(provider_name) }.to raise_error(RuntimeError, 'mockerror')
-      end
-    end
-  end
-
-  describe '#purge_vms_and_folders' do
+  describe '#purge_vms_and_resources' do
     let(:folder_name) { 'myinstance' }
     let(:folder_base) { 'vmpooler' }
     let(:datacenter) { 'dc1' }
     let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
-    let(:configured_folders) { { folder_name => full_folder_path } }
-    let(:base_folders) { [ full_folder_path ] }
     let(:folder) { [folder_base,folder_name].join('/') }
     let(:provider_name) { 'mock_provider' }
-    let(:whitelist) { nil }
+    let(:allowlist) { nil }
     let(:config) { YAML.load(<<-EOT
 ---
 :config: {}
@@ -1479,20 +1484,18 @@ EOT
         $providers = { provider_name => provider }
       end
 
-      it 'should run purge_unconfigured_folders' do
-        expect(subject).to receive(:pool_folders).and_return(configured_folders)
-        expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
-        expect(provider).to receive(:provider_config).and_return({})
+      it 'should run purge_unconfigured_resources' do
+        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist)
+        expect(provider).to receive(:provider_config).and_return({}).twice
 
-        subject.purge_vms_and_folders(provider_name)
+        subject.purge_vms_and_resources(provider_name)
       end
 
       it 'should raise any errors' do
-        expect(subject).to receive(:pool_folders).and_return(configured_folders)
-        expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
-        expect(provider).to receive(:provider_config).and_return({})
+        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist).and_raise('mockerror')
+        expect(provider).to receive(:provider_config).and_return({}).twice
 
-        expect{ subject.purge_vms_and_folders(provider_name) }.to raise_error(RuntimeError, 'mockerror')
+        expect{ subject.purge_vms_and_resources(provider_name) }.to raise_error(RuntimeError, 'mockerror')
       end
     end
   end
@@ -1910,6 +1913,51 @@ EOT
     end
   end
 
+  describe '#get_dns_plugin_class_name_for_pool' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+
+    )}
+    before(:each) do
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+    end
+
+    it 'calls Vmpooler::Dns.get_dns_plugin_class_by_name' do
+      expect(Vmpooler::Dns).to receive(:get_dns_plugin_class_by_name).with(config, 'mock')
+
+      subject.get_dns_plugin_class_name_for_pool(pool)
+    end
+  end
+
+  describe '#get_dns_plugin_domain_for_pool' do
+    let(:config) { YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+  )}
+    before(:each) do
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+    end
+
+    it 'calls Vmpooler::Dns.get_dns_plugin_domain_for_pool' do
+      expect(Vmpooler::Dns).to receive(:get_dns_plugin_domain_by_name).with(config, 'mock')
+
+      subject.get_dns_plugin_domain_for_pool(pool)
+    end
+  end
+
   describe '#check_disk_queue' do
     let(:threads) {[]}
 
@@ -2062,6 +2110,7 @@ EOT
 
     before(:each) do
       expect(Thread).to receive(:new).and_yield
+      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
       allow(subject).to receive(:_check_snapshot_queue).with(no_args)
     end
 
@@ -2107,6 +2156,10 @@ EOT
       let(:maxloop) { 5 }
       # Note a maxloop of zero can not be tested as it never terminates
 
+      before(:each) do
+        allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
+      end
+
       after(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = nil
@@ -2946,6 +2999,7 @@ EOT
       allow(subject).to receive(:check_snapshot_queue)
       allow(subject).to receive(:check_pool)
       allow(subject).to receive(:check_ondemand_requests)
+      allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
 
       allow(logger).to receive(:log)
     end
@@ -2956,6 +3010,23 @@ EOT
     end
 
     context 'on startup' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+        )}
+
+      before(:each) do
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+      end
+
       it 'should log a message that VMPooler has started' do
         expect(logger).to receive(:log).with('d', 'starting vmpooler')
 
@@ -3002,6 +3073,46 @@ EOT
         subject.execute!(1,0)
       end
 
+      context 'creating Dns plugins' do
+        let(:mock_dns_plugin) { double('mock_dns_plugin') }
+        let(:config) {
+        YAML.load(<<-EOT
+---
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+  - name: 'dummy'
+    dns_plugin: 'mock'
+  - name: 'dummy2'
+    dns_plugin: 'mock'
+EOT
+        )}
+
+        it 'should call create_dns_object idempotently' do
+          # Even though there are two pools using the mock dns plugin, it should only
+          # create the dns object once.
+          expect(subject).to receive(:create_dns_object).and_return(mock_dns_plugin)
+
+          subject.execute!(1,0)
+        end
+
+        it 'should raise an error if the dns plugin cannot be created' do
+          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")
+
+          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
+        end
+
+        it 'should log a message if the dns plugin cannot be created' do
+          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")
+          expect(logger).to receive(:log).with('s',"Error while creating dns plugin for pool #{pool}: MockError")
+
+          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
+        end
+      end
+
       context 'creating Providers' do
         let(:vsphere_provider) { double('vsphere_provider') }
         let(:config) {
@@ -3009,15 +3120,25 @@ EOT
 ---
 :providers:
   :vsphere: {}
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    provider: 'vsphere'
+    dns_plugin: 'mock'
   - name: 'dummy'
     provider: 'vsphere'
+    dns_plugin: 'mock'
 EOT
         )}
 
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
+
         it 'should call create_provider_object idempotently' do
-          # Even though there are two pools using the vsphere provider (the default), it should only
+          # Even though there are two pools using the vsphere provider, it should only
           # create the provider object once.
           expect(subject).to receive(:create_provider_object).with(Object, Object, Object, redis_connection_pool, 'vsphere', 'vsphere', Object).and_return(vsphere_provider)
 
@@ -3039,62 +3160,25 @@ EOT
       end
     end
 
-    context 'creating multiple vsphere Providers' do
-      let(:vsphere_provider) { double('vsphere_provider') }
-      let(:vsphere_provider2) { double('vsphere_provider') }
-      let(:provider1) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'vsphere', provider_options) }
-      let(:provider2) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'secondvsphere', provider_options) }
-      let(:config) {
-        YAML.load(<<-EOT
----
-:providers:
-  :vsphere:
-    server: 'blah1'
-    provider_class: 'vsphere'
-  :secondvsphere:
-    server: 'blah2'
-    provider_class: 'vsphere'
-:pools:
-  - name: #{pool}
-    provider: 'vsphere'
-  - name: 'secondpool'
-    provider: 'secondvsphere'
-EOT
-        )}
-
-      it 'should call create_provider_object twice' do
-        # The two pools use a different provider name, but each provider_class is vsphere
-        expect(subject).to receive(:create_provider_object).with(Object, Object, Object, redis_connection_pool, "vsphere", "vsphere", Object).and_return(vsphere_provider)
-        expect(subject).to receive(:create_provider_object).with(Object, Object, Object, redis_connection_pool, "vsphere", "secondvsphere", Object).and_return(vsphere_provider2)
-        subject.execute!(1,0)
-      end
-
-      it 'should have vsphere providers with different servers' do
-        allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider1)
-        result = subject.get_provider_for_pool(pool)
-        allow(provider1).to receive(:provider_config).and_call_original
-        expect(result.provider_config['server']).to eq('blah1')
-
-        allow(subject).to receive(:get_provider_for_pool).with('secondpool').and_return(provider2)
-        result = subject.get_provider_for_pool('secondpool')
-        allow(provider1).to receive(:provider_config).and_call_original
-        expect(result.provider_config['server']).to eq('blah2')
-        subject.execute!(1,0)
-      end
-    end
-
     context 'modify configuration on startup' do
       context 'move vSphere configuration to providers location' do
         let(:config) {
         YAML.load(<<-EOT
+:dns_configs:
+  :mock:
+    dns_class: base
 :vsphere:
   server: 'vsphere.example.com'
   username: 'vmpooler'
   password: 'password'
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
 EOT
         )}
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
 
         it 'should set providers with no provider to vsphere' do
           expect(subject.config[:providers]).to be nil
@@ -3113,29 +3197,38 @@ EOT
       end
 
 
-      context 'default to the vphere provider' do
+      context 'default to the dummy provider' do
         let(:config) {
         YAML.load(<<-EOT
 ---
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
   - name: 'dummy'
     provider: 'dummy'
+    dns_plugin: 'mock'
 EOT
         )}
 
-        it 'should set providers with no provider to vsphere' do
+        before(:each) do
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+        end
+
+        it 'should set providers with no provider to dummy' do
           expect(subject.config[:pools][0]['provider']).to be_nil
           expect(subject.config[:pools][1]['provider']).to eq('dummy')
 
           subject.execute!(1,0)
 
-          expect(subject.config[:pools][0]['provider']).to eq('vsphere')
+          expect(subject.config[:pools][0]['provider']).to eq('dummy')
           expect(subject.config[:pools][1]['provider']).to eq('dummy')
         end
 
         it 'should log a message' do
-          expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'vsphere' as default")
+          expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'dummy' as default")
 
           subject.execute!(1,0)
         end
@@ -3144,11 +3237,31 @@ EOT
 
     context 'with dead disk_manager thread' do
       let(:disk_manager_thread) { double('thread', :alive? => false) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
 
       before(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = {}
         $threads['disk_manager'] = disk_manager_thread
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'should run the check_disk_queue method and log a message' do
@@ -3161,10 +3274,30 @@ EOT
 
     context 'with dead snapshot_manager thread' do
       let(:snapshot_manager_thread) { double('thread', :alive? => false) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       before(:each) do
         # Reset the global variable - Note this is a code smell
         $threads = {}
         $threads['snapshot_manager'] = snapshot_manager_thread
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'should run the check_snapshot_queue method and log a message' do
@@ -3182,10 +3315,27 @@ EOT
         let(:default_check_loop_delay_min) { 5 }
         let(:default_check_loop_delay_max) { 60 }
         let(:default_check_loop_delay_decay) { 2.0 }
+        let(:check_loop_delay_min) { 7 }
+        let(:check_loop_delay_max) { 20 }
+        let(:check_loop_delay_decay) { 2.1 }
+        let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
         before(:each) do
           # Reset the global variable - Note this is a code smell
           $threads = {}
           $threads[pool] = pool_thread
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
         end
 
         it 'should run the check_pool method and log a message' do
@@ -3204,6 +3354,19 @@ EOT
         let(:default_check_loop_delay_min) { 5 }
         let(:default_check_loop_delay_max) { 60 }
         let(:default_check_loop_delay_decay) { 2.0 }
+        let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
         before(:each) do
           # Reset the global variable - Note this is a code smell
           $threads = {}
@@ -3230,8 +3393,12 @@ EOT
   check_loop_delay_min: #{check_loop_delay_min}
   check_loop_delay_max: #{check_loop_delay_max}
   check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
 :pools:
   - name: #{pool}
+    dns_plugin: 'mock'
 EOT
           )
         }
@@ -3239,6 +3406,8 @@ EOT
           # Reset the global variable - Note this is a code smell
           $threads = {}
           $threads[pool] = pool_thread
+          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
+          # allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
         end
 
         it 'should run the check_pool method and log a message' do
@@ -3256,6 +3425,26 @@ EOT
     context 'delays between loops' do
       let(:maxloop) { 2 }
       let(:loop_delay) { 1 }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
+      let(:maxloop) { 5 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       # Note a maxloop of zero can not be tested as it never terminates
       before(:each) do
 
@@ -3263,6 +3452,7 @@ EOT
         allow(subject).to receive(:check_snapshot_queue)
         allow(subject).to receive(:check_pool)
         allow(subject).to receive(:check_ondemand_requests)
+        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
       end
 
       it 'when a non-default loop delay is specified' do
@@ -3274,10 +3464,31 @@ EOT
 
     context 'loops specified number of times (5)' do
       let(:alive_thread) { double('thread', :alive? => true) }
+      let(:check_loop_delay_min) { 7 }
+      let(:check_loop_delay_max) { 20 }
+      let(:check_loop_delay_decay) { 2.1 }
       let(:maxloop) { 5 }
+      let(:config) {
+      YAML.load(<<-EOT
+---
+:config:
+  check_loop_delay_min: #{check_loop_delay_min}
+  check_loop_delay_max: #{check_loop_delay_max}
+  check_loop_delay_decay: #{check_loop_delay_decay}
+:dns_configs:
+  :mock:
+    dns_class: base
+:pools:
+  - name: #{pool}
+    dns_plugin: 'mock'
+EOT
+          )
+        }
       # Note a maxloop of zero can not be tested as it never terminates
       before(:each) do
-        end
+        allow(subject).to receive(:get_domain_for_pool)
+        allow(subject).to receive(:get_dns_plugin_config_classes)
+      end
 
       it 'should run startup tasks only once' do
         redis_connection_pool.with do |redis|
@@ -3442,6 +3653,54 @@ EOT
       end
     end
 
+    describe 'with the undo_override wakeup option' do
+      let(:wakeup_option) {{
+        :undo_override => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a undoing a template override is requested' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.sadd('vmpooler__pool__undo_template_override', pool)
+            allow(redis).to receive(:hget)
+          end
+        end
+
+        it 'should sleep until the undo override request is detected' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).at_least(2).times
+            expect(subject).to receive(:sleep).at_most(3).times
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,true)
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+
+      context 'when a undoing a size override is requested' do
+        before(:each) do
+          redis_connection_pool.with do |redis|
+            redis.sadd('vmpooler__pool__undo_size_override', pool)
+            allow(redis).to receive(:hget)
+          end
+        end
+
+        it 'should sleep until the undo override request is detected' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,false)
+            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false,true)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
+
     describe 'with the pending_vm wakeup option' do
       let(:wakeup_option) {{
         :pending_vm => true,
@@ -3471,8 +3730,9 @@ EOT
       it 'should sleep until the provisioning request is detected' do
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:sleep).exactly(3).times
-          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
-          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[1,0,0])
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,1)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
         end
 
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
@@ -3481,17 +3741,20 @@ EOT
       it 'should sleep until provisioning processing is detected' do
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:sleep).exactly(3).times
-          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
-          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,1,0])
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,1)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
         end
+
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
       end
 
       it 'should sleep until ondemand creation task is detected' do
         redis_connection_pool.with do |redis|
           expect(subject).to receive(:sleep).exactly(3).times
-          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
-          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,0,1])
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
+          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,1)
         end
 
         subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
@@ -3760,6 +4023,54 @@ EOT
     end
   end
 
+  describe 'undo_override' do
+    let(:mutex) { Mutex.new }
+    let(:original_template) { 'templates/template1' }
+    let(:override_template) { 'templates/template2' }
+    let(:original_size) { 2 }
+    let(:override_size) { 10 }
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+  task_limit: 5
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: #{override_size}
+    template: '#{override_template}'
+:pool_index:
+  '#{pool}': 0
+:pools_at_startup:
+  - name: '#{pool}'
+    size: #{original_size}
+    template: '#{original_template}'
+EOT
+      )
+    }
+
+    before(:each) do
+      redis_connection_pool.with do |redis|
+        redis.sadd('vmpooler__pool__undo_template_override', pool)
+        redis.sadd('vmpooler__pool__undo_size_override', pool)
+        # allow(redis).to receive(:hget)
+      end
+    end
+
+    it 'should revert to the original template and pool size' do
+      redis_connection_pool.with do |redis|
+        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(true)
+        expect(redis).to receive(:srem).with('vmpooler__pool__undo_template_override', pool).and_return(true)
+        expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, original_template, override_template, redis)
+
+        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(true)
+        expect(redis).to receive(:srem).with('vmpooler__pool__undo_size_override', pool).and_return(true)
+      end
+
+      subject.undo_override(config[:pools][0], provider)
+    end
+  end
+
   describe '#create_inventory' do
 
     it 'should log an error if one occurs' do
@@ -3950,10 +4261,10 @@ EOT
 
       it 'should call fail_pending_vm' do
         redis_connection_pool.with do |redis|
-          expect(subject).to receive(:fail_pending_vm).with(vm,pool,Integer,redis,false)
+          expect(subject).to receive(:fail_pending_vm).with(vm, pool, Integer, Integer, redis, exists: false)
         end
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
     end
 
@@ -3971,7 +4282,7 @@ EOT
 
       it 'should return the number of checked pending VMs' do
         allow(subject).to receive(:check_pending_vm)
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
 
         expect(pool_check_response[:checked_pending_vms]).to be(1)
       end
@@ -3980,31 +4291,31 @@ EOT
         expect(subject).to receive(:check_pending_vm).and_raise(RuntimeError,'MockError')
         expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating pending VMs: MockError")
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
 
       it 'should use the pool timeout if set' do
         big_lifetime = 2000
 
         config[:pools][0]['timeout'] = big_lifetime
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,big_lifetime,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm,pool, big_lifetime, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
       end
 
       it 'should use the configuration setting if the pool timeout is not set' do
         big_lifetime = 2000
 
         config[:config]['timeout'] = big_lifetime
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,big_lifetime,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm, pool, big_lifetime, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
       end
 
       it 'should use a pool timeout of 15 if nothing is set' do
-        expect(subject).to receive(:check_pending_vm).with(vm,pool,timeout,provider)
+        expect(subject).to receive(:check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)
 
-        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout)
+        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
       end
     end
   end
@@ -4252,6 +4563,10 @@ EOT
       }
     }
 
+    before(:each) do
+      allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+    end
+
     it 'should not call clone_vm when number of VMs is equal to the pool size' do
       expect(subject).to receive(:clone_vm).exactly(0).times
 
@@ -4306,6 +4621,9 @@ EOT
     end
 
     context 'when pool is marked as empty' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
 
       before(:each) do
         redis_connection_pool.with do |redis|
@@ -4331,6 +4649,9 @@ EOT
     end
 
     context 'when number of VMs is less than the pool size' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
 
       it 'should return the number of cloned VMs' do
         pool_size = 5
@@ -4387,6 +4708,10 @@ EOT
     end
 
     context 'export metrics' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       it 'increments metrics for ready queue' do
         redis_connection_pool.with do |redis|
           create_ready_vm(pool,'vm1',redis)
@@ -4477,6 +4802,7 @@ EOT
         allow(subject).to receive(:destroy_vm)
         allow(subject).to receive(:clone_vm)
         allow(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response)
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'calls inventory correctly' do
@@ -4581,6 +4907,10 @@ EOT
 
     # RUNNING
     context 'when checking running VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -4603,6 +4933,10 @@ EOT
 
     # READY
     context 'when checking ready VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -4625,6 +4959,10 @@ EOT
 
     # PENDING
     context 'when checking ready VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -4639,7 +4977,7 @@ EOT
 
       it 'should call #check_ready_pool_vms' do
         allow(subject).to receive(:create_inventory).and_return({})
-        expect(subject).to receive(:check_pending_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['timeout'])
+        expect(subject).to receive(:check_pending_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['timeout'], pool_object['timeout_notification'])
 
         subject._check_pool(pool_object,provider)
       end
@@ -4648,6 +4986,10 @@ EOT
 
     # COMPLETED
     context 'when checking completed VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
             discovered_vms: 0,
@@ -4670,6 +5012,10 @@ EOT
 
     # DISCOVERED
     context 'when checking discovered VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
             discovered_vms: 0,
@@ -4692,6 +5038,10 @@ EOT
 
     # MIGRATIONS
     context 'when checking migrating VMs' do
+      before(:each) do
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
+      end
+
       let(:pool_check_response) {
         {
           discovered_vms: 0,
@@ -4721,6 +5071,7 @@ EOT
         redis_connection_pool.with do |redis|
           redis.hset('vmpooler__config__poolsize', pool, newpoolsize)
         end
+        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'should change the pool size configuration' do
@@ -4762,6 +5113,7 @@ EOT
           expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
           expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
         end
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
       end
 
       it 'should call remove_excess_vms' do
@@ -4826,6 +5178,44 @@ EOT
       end
     end
 
+    context 'when request is already marked as failed' do
+      let(:request_string) { "#{pool}:#{pool}:1" }
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+          set_ondemand_request_status(request_id, 'failed', redis)
+        end
+      end
+
+      it 'logs that the request is already failed' do
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "Request '#{request_id}' already marked as failed, skipping VM creation")
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+
+      it 'removes the request from provisioning__request queue' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zscore('vmpooler__provisioning__request', request_id)).to be_nil
+        end
+      end
+
+      it 'does not create VM tasks' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zcard('vmpooler__odcreate__task')).to eq(0)
+        end
+      end
+
+      it 'does not add to provisioning__processing queue' do
+        redis_connection_pool.with do |redis|
+          subject.create_ondemand_vms(request_id, redis)
+          expect(redis.zscore('vmpooler__provisioning__processing', request_id)).to be_nil
+        end
+      end
+    end
+
     context 'with a request that has data' do
       let(:request_string) { "#{pool}:#{pool}:1" }
       before(:each) do
@@ -4837,16 +5227,20 @@ EOT
 
       it 'creates tasks for instances to be provisioned' do
         redis_connection_pool.with do |redis|
-          allow(redis).to receive(:zadd)
-          expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}")
+          redis.pipelined do |pipe|
+            allow(pipe).to receive(:zadd)
+            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}")
+          end
           subject.create_ondemand_vms(request_id, redis)
         end
       end
 
       it 'adds a member to provisioning__processing' do
         redis_connection_pool.with do |redis|
-          allow(redis).to receive(:zadd)
-          expect(redis).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id)
+          redis.pipelined do |pipe|
+            allow(pipe).to receive(:zadd)
+            expect(pipe).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id)
+          end
           subject.create_ondemand_vms(request_id, redis)
         end
       end
@@ -4867,6 +5261,7 @@ EOT
       before(:each) do
         config[:config]['ondemand_clone_limit'] = 10
         expect(subject).to receive(:get_provider_for_pool).and_return(provider)
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
         redis_connection_pool.with do |redis|
           create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
         end
@@ -4874,7 +5269,7 @@ EOT
 
       it 'creates the vm' do
         redis_connection_pool.with do |redis|
-          expect(subject).to receive(:clone_vm).with(pool, provider, request_id, pool_alias)
+          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias)
           subject.process_ondemand_vms(redis)
         end
       end
@@ -4887,6 +5282,7 @@ EOT
       before(:each) do
         config[:config]['ondemand_clone_limit'] = 3
         expect(subject).to receive(:get_provider_for_pool).and_return(provider)
+        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
         redis_connection_pool.with do |redis|
           create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
         end
@@ -4894,14 +5290,16 @@ EOT
 
       it 'should create the maximum number of vms' do
         redis_connection_pool.with do |redis|
-          expect(subject).to receive(:clone_vm).with(pool, provider, request_id, pool_alias).exactly(3).times
+          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias).exactly(3).times
           subject.process_ondemand_vms(redis)
         end
       end
 
       it 'should add the remaining number back as a new create task with the same score' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string_remaining}:#{request_id}")
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string_remaining}:#{request_id}")
+          end
           subject.process_ondemand_vms(redis)
         end
       end
@@ -5111,21 +5509,27 @@ EOT
 
       it 'removes the request from processing requests' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
+          end
           subject.request_expired?(request_id, expired_time, redis)
         end
       end
 
       it 'sets the status as failed on the request hash' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+          end
           subject.request_expired?(request_id, expired_time, redis)
         end
       end
 
       it 'marks the request hash for expiration' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{request_id}", expiration_ttl * 60 * 60)
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__odrequest__#{request_id}", expiration_ttl * 60 * 60)
+          end
           subject.request_expired?(request_id, expired_time, redis)
         end
       end
@@ -5164,14 +5568,18 @@ EOT
 
       it 'should remove the associated vms' do
         redis_connection_pool.with do |redis|
-          expect(subject).to receive(:move_vm_queue).with(pool, String, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time").twice
+          redis.pipelined do |pipe|
+            expect(subject).to receive(:move_vm_queue).with(pool, String, 'running', 'completed', pipe, "moved to completed queue. '#{request_id}' could not be filled in time").twice
+          end
           subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
         end
       end
 
       it 'should mark the ready set for expiration' do
         redis_connection_pool.with do |redis|
-          expect(redis).to receive(:expire).with("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+          redis.pipelined do |pipe|
+            expect(pipe).to receive(:expire).with("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+          end
           subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
         end
       end
diff --git a/spec/unit/promstats_spec.rb b/spec/unit/promstats_spec.rb
index dca2e12..9075536 100644
--- a/spec/unit/promstats_spec.rb
+++ b/spec/unit/promstats_spec.rb
@@ -198,46 +198,51 @@ describe 'prometheus' do
           po.get(labels: metric[:labels])
         }.by(1)
       end
-      it 'Increments user.#{user}.#{poolname}' do
+      it 'Increments user.#{user}.#{operation}.#{poolname}' do
         user = 'myuser'
+        operation = 'allocate'
         poolname = 'test-pool'
-        expect { subject.increment("user.#{user}.#{poolname}") }.to change {
-          metric, po = subject.get("user.#{user}.#{poolname}")
+        expect { subject.increment("user.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("user.#{user}.#{operation}.#{poolname}")
           po.get(labels: metric[:labels])
         }.by(1)
       end
-      it 'Increments usage_litmus.#{user}.#{poolname}' do
+      it 'Increments usage_litmus.#{user}.#{operation}.#{poolname}' do
         user = 'myuser'
+        operation = 'allocate'
         poolname = 'test-pool'
-        expect { subject.increment("usage_litmus.#{user}.#{poolname}") }.to change {
-          metric, po = subject.get("usage_litmus.#{user}.#{poolname}")
+        expect { subject.increment("usage_litmus.#{user}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_litmus.#{user}.#{operation}.#{poolname}")
           po.get(labels: metric[:labels])
         }.by(1)
       end
-      it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}' do
+      it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}' do
         jenkins_instance = 'jenkins_test_instance'
         value_stream = 'notional_value'
+        operation = 'allocate'
         poolname = 'test-pool'
-        expect { subject.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}") }.to change {
-          metric, po = subject.get("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}")
+        expect { subject.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
           po.get(labels: metric[:labels])
         }.by(1)
       end
-      it 'Increments label usage_branch_project.#{branch}.#{project}.#{poolname}' do
+      it 'Increments label usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}' do
         branch = 'treetop'
         project = 'test-project'
+        operation = 'allocate'
         poolname = 'test-pool'
-        expect { subject.increment("usage_branch_project.#{branch}.#{project}.#{poolname}") }.to change {
-          metric, po = subject.get("usage_branch_project.#{branch}.#{project}.#{poolname}")
+        expect { subject.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
           po.get(labels: metric[:labels])
         }.by(1)
       end
-      it 'Increments label usage_job_component.#{job_name}.#{component_to_test}.#{poolname}' do
+      it 'Increments label usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}' do
         job_name = 'a-job'
         component_to_test = 'component-name'
+        operation = 'allocate'
         poolname = 'test-pool'
-        expect { subject.increment("usage_job_component.#{job_name}.#{component_to_test}.#{poolname}") }.to change {
-          metric, po = subject.get("usage_job_component.#{job_name}.#{component_to_test}.#{poolname}")
+        expect { subject.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") }.to change {
+          metric, po = subject.get("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
           po.get(labels: metric[:labels])
         }.by(1)
       end
diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb
deleted file mode 100644
index a2970ac..0000000
--- a/spec/unit/providers/vsphere_spec.rb
+++ /dev/null
@@ -1,3558 +0,0 @@
-require 'spec_helper'
-require 'mock_redis'
-require 'vmpooler/providers/vsphere'
-
-RSpec::Matchers.define :relocation_spec_with_host do |value|
-  match { |actual| actual[:spec].host == value }
-end
-
-RSpec::Matchers.define :create_virtual_disk_with_size do |value|
-  match { |actual| actual[:spec].capacityKb == value * 1024 * 1024 }
-end
-
-RSpec::Matchers.define :create_vm_spec do |new_name,target_folder_name,datastore|
-  match { |actual|
-    # Should have the correct new name
-    actual[:name] == new_name &&
-    # Should be in the new folder
-    actual[:folder].name == target_folder_name &&
-    # Should be poweredOn after clone
-    actual[:spec].powerOn == true &&
-    # Should be on the correct datastore
-    actual[:spec][:location].datastore.name == datastore &&
-    # Should contain annotation data
-    actual[:spec][:config].annotation != '' &&
-    # Should contain VIC information
-    actual[:spec][:config].extraConfig[0][:key] == 'guestinfo.hostname' &&
-    actual[:spec][:config].extraConfig[0][:value] == new_name
-  }
-end
-
-RSpec::Matchers.define :create_snapshot_spec do |new_snapshot_name|
-  match { |actual|
-    # Should have the correct new name
-    actual[:name] == new_snapshot_name &&
-    # Should snapshot the memory too
-    actual[:memory] == true &&
-    # Should quiesce the disk
-    actual[:quiesce] == true
-  }
-end
-
-describe 'Vmpooler::PoolManager::Provider::VSphere' do
-  let(:logger) { MockLogger.new }
-  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
-  let(:poolname) { 'pool1'}
-  let(:provider_options) { { 'param' => 'value' } }
-  let(:datacenter_name) { 'MockDC' }
-  let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_tries: 3
-  retry_factor: 10
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    insecure: true
-    # Drop the connection pool timeout way down for spec tests so they fail fast
-    connection_pool_timeout: 1
-    datacenter: MockDC
-    create_linked_clones: true
-:pools:
-  - name: '#{poolname}'
-    alias: [ 'mockpool' ]
-    template: 'Templates/pool1'
-    folder: 'Pooler/pool1'
-    datastore: 'datastore0'
-    size: 5
-    timeout: 10
-    ready_ttl: 1440
-    clone_target: 'cluster1'
-EOT
-    )
-  }
-
-  let(:connection_options) {{}}
-  let(:connection) { mock_RbVmomi_VIM_Connection(connection_options) }
-  let(:vmname) { 'vm1' }
-  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
-    metrics: metrics,
-    connpool_type: 'redis_connection_pool',
-    connpool_provider: 'testprovider',
-    size: 1,
-    timeout: 5
-  ) { MockRedis.new }
-  }
-
-  subject { Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, redis_connection_pool, 'vsphere', provider_options) }
-
-  before(:each) do
-    allow(subject).to receive(:vsphere_connection_ok?).and_return(true)
-  end
-
-  describe '#name' do
-    it 'should be vsphere' do
-      expect(subject.name).to eq('vsphere')
-    end
-  end
-
-  describe '#folder_configured?' do
-    let(:folder_title) { 'folder1' }
-    let(:other_folder) { 'folder2' }
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:configured_folders) { { folder_title => base_folder } }
-    let(:whitelist) { nil }
-    it 'should return true when configured_folders includes the folder_title' do
-      expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true
-    end
-
-    it 'should return false when title is not in configured_folders' do
-      expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false
-    end
-
-    context 'with another base folder' do
-      let(:base_folder) { 'dc2/vm/base' }
-      let(:configured_folders) { { folder_title => 'dc1/vm/base' } }
-      it 'should return false' do
-        expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false
-      end
-    end
-
-    context 'with a whitelist set' do
-      let(:whitelist) { [ other_folder ] }
-      it 'should return true' do
-        expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true
-      end
-    end
-
-    context 'with string whitelist value' do
-      let(:whitelist) { 'whitelist' }
-      it 'should raise an error' do
-        expect(whitelist).to receive(:include?).and_raise('mockerror')
-
-        expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror')
-      end
-    end
-  end
-
-  describe '#destroy_vm_and_log' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-        :powerstate => 'poweredOn',
-      })
-    }
-    let(:pool) { 'pool1' }
-    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-    let(:data_ttl) { 1 }
-    let(:finish) { '0.00' }
-    let(:now) { Time.now }
-
-    context 'when destroying a vm' do
-      before(:each) do
-        allow(power_off_task).to receive(:wait_for_completion)
-        allow(destroy_task).to receive(:wait_for_completion)
-        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-      end
-
-      it 'should log a message that the vm is destroyed' do
-        # Ensure Time returns a consistent value so finish is predictable
-        # Otherwise finish occasionally increases to 0.01 and causes a failure
-        allow(Time).to receive(:now).and_return(Time.now)
-        expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds")
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-
-      it 'should record metrics' do
-        expect(metrics).to receive(:timing).with('connection_waited.redis_connection_pool.testprovider', 0)
-        expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-
-      it 'should power off and destroy the vm' do
-        allow(destroy_task).to receive(:wait_for_completion)
-        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-    end
-
-    context 'with a powered off vm' do
-      before(:each) do
-        vm_object.runtime.powerState = 'poweredOff'
-      end
-
-      it 'should destroy the vm without attempting to power it off' do
-        allow(destroy_task).to receive(:wait_for_completion)
-        expect(vm_object).to_not receive(:PowerOffVM_Task)
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
-      end
-    end
-
-    context 'with a folder object' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) }
-
-      it 'should log that a folder object was received' do
-        expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying")
-
-        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
-      end
-    end
-
-    context 'with an error that is not a RuntimeError' do
-      it 'should retry three times' do
-        expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times
-
-        expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/)
-      end
-    end
-  end
-
-  describe '#destroy_folder_and_children' do
-    let(:data_ttl) { 1 }
-    let(:config) {
-      {
-        redis: {
-          'data_ttl' => data_ttl
-        }
-      }
-    }
-    let(:foldername) { 'pool1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-
-    before(:each) do
-      $config = config
-    end
-
-    context 'with an empty folder' do
-      it 'should destroy the folder' do
-        expect(subject).to_not receive(:destroy_vm_and_log)
-        expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil)
-
-        subject.destroy_folder_and_children(folder_object)
-      end
-    end
-
-    context 'with a folder containing vms' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-      before(:each) do
-        folder_object.childEntity << vm_object
-      end
-
-      it 'should destroy the vms' do
-        allow(subject).to receive(:destroy_vm_and_log).and_return(nil)
-        allow(subject).to receive(:destroy_folder).and_return(nil)
-        expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl)
-
-        subject.destroy_folder_and_children(folder_object)
-      end
-    end
-
-    it 'should raise any errors' do
-      expect(subject).to receive(:destroy_folder).and_throw('mockerror')
-
-      expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#destroy_folder' do
-    let(:foldername) { 'pool1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
-      allow(destroy_task).to receive(:wait_for_completion)
-    end
-
-    it 'should destroy the folder' do
-      expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-      subject.destroy_folder(folder_object)
-    end
-
-    it 'should log that the folder is being destroyed' do
-      expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder")
-
-      subject.destroy_folder(folder_object)
-    end
-
-    it 'should retry three times when failing' do
-      expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times
-
-      expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#purge_unconfigured_folders' do
-    let(:folder_title) { 'folder1' }
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
-    let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) }
-    let(:whitelist) { nil }
-    let(:base_folders) { [ base_folder ] }
-    let(:configured_folders) { { folder_title => base_folder } }
-    let(:folder_children) { [ folder_title => child_folder ] }
-    let(:empty_list) { [] }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'with an empty folder' do
-      it 'should not attempt to destroy any folders' do
-        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list)
-        expect(subject).to_not receive(:destroy_folder_and_children)
-
-        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-      end
-    end
-
-    it 'should retrieve the folder children' do
-      expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
-      allow(subject).to receive(:folder_configured?).and_return(true)
-
-      subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-    end
-
-    context 'with a folder that is not configured' do
-      before(:each) do
-        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
-        allow(subject).to receive(:folder_configured?).and_return(false)
-      end
-
-      it 'should destroy the folder and children' do
-        expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil)
-
-        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
-      end
-    end
-
-    it 'should raise any errors' do
-      expect(subject).to receive(:get_folder_children).and_throw('mockerror')
-
-      expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#get_folder_children' do
-    let(:base_folder) { 'dc1/vm/base' }
-    let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
-    let(:foldername) { 'folder1' }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
-    let(:folder_return) { [ { foldername => folder_object } ] }
-
-    before(:each) do
-      base_folder_object.childEntity << folder_object
-    end
-
-    it 'should return an array of configured folder hashes' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object)
-
-      result = subject.get_folder_children(foldername, connection)
-
-      expect(result).to eq(folder_return)
-    end
-
-    it 'should raise any errors' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror')
-
-      expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/)
-    end
-  end
-
-  describe '#vms_in_pool' do
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-    let(:pool_config) { config[:pools][0] }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'Given a pool folder that is missing' do
-      before(:each) do
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(nil)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should return an empty array' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq([])
-      end
-    end
-
-    context 'Given an empty pool folder' do
-      before(:each) do
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should return an empty array' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq([])
-      end
-    end
-
-    context 'Given a pool folder with many VMs' do
-      let(:expected_vm_list) {[
-        { 'name' => 'vm1'},
-        { 'name' => 'vm2'},
-        { 'name' => 'vm3'}
-      ]}
-      before(:each) do
-        expected_vm_list.each do |vm_hash|
-          mock_vm = mock_RbVmomi_VIM_VirtualMachine({ :name => vm_hash['name'] })
-          # Add the mocked VM to the folder
-          folder_object.childEntity << mock_vm
-        end
-
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should get a connection' do
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection)
-
-        subject.vms_in_pool(poolname)
-      end
-
-      it 'should list all VMs in the VM folder for the pool' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq(expected_vm_list)
-      end
-    end
-
-    context 'given a pool folder with a folder and vms' do
-      let(:expected_vm_list) {[
-        { 'name' => 'vm1'},
-        { 'name' => 'vm2'},
-        { 'name' => 'vm3'}
-      ]}
-      let(:folder_object2) { mock_RbVmomi_VIM_Folder({ :name => 'pool2'}) }
-      before(:each) do
-        expected_vm_list.each do |vm_hash|
-          mock_vm = mock_RbVmomi_VIM_VirtualMachine({ :name => vm_hash['name'] })
-          # Add the mocked VM to the folder
-          folder_object.childEntity << mock_vm
-        end
-        folder_object.childEntity << folder_object2
-        expect(subject).to receive(:find_vm_folder).with(poolname,connection).and_return(folder_object)
-      end
-
-      it 'should return the vms without the folder' do
-        result = subject.vms_in_pool(poolname)
-
-        expect(result).to eq(expected_vm_list)
-      end
-    end
-  end
-
-  describe '#get_vm' do
-    let(:vm_object) { nil }
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      expect(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      it 'should return nil' do
-        expect(subject.get_vm(poolname,vmname)).to be_nil
-      end
-    end
-
-    context 'when VM exists but is missing information' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-        })
-      }
-
-      it 'should return a hash' do
-        expect(subject.get_vm(poolname,vmname)).to be_kind_of(Hash)
-      end
-
-      it 'should return the VM name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-
-      ['hostname','boottime','powerstate'].each do |testcase|
-        it "should return nil for #{testcase}" do
-          result = subject.get_vm(poolname,vmname)
-
-          expect(result[testcase]).to be_nil
-        end
-      end
-    end
-
-    context 'when VM exists and contains all information' do
-      let(:vm_hostname) { "#{vmname}.demo.local" }
-      let(:boot_time) { Time.now }
-      let(:power_state) { 'MockPowerState' }
-
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :hostname => vm_hostname,
-          :powerstate => power_state,
-          :boottime => boot_time,
-          # This path should match the folder in the mocked pool in the config above
-          :path => [
-            { :type => 'folder',     :name => 'Datacenters' },
-            { :type => 'datacenter', :name => 'DC01' },
-            { :type => 'folder',     :name => 'vm' },
-            { :type => 'folder',     :name => 'Pooler' },
-            { :type => 'folder',     :name => 'pool1'},
-          ]
-        })
-      }
-      let(:pool_info) { config[:pools][0]}
-
-      it 'should return a hash' do
-        expect(subject.get_vm(poolname,vmname)).to be_kind_of(Hash)
-      end
-
-      it 'should return the VM name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-
-      it 'should return the VM hostname' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['hostname']).to eq(vm_hostname)
-      end
-
-      it 'should return the template name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['template']).to eq(pool_info['template'])
-      end
-
-      it 'should return the pool name' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['poolname']).to eq(pool_info['name'])
-      end
-
-      it 'should return the boot time' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['boottime']).to eq(boot_time)
-      end
-
-      it 'should return the powerstate' do
-        result = subject.get_vm(poolname,vmname)
-
-        expect(result['powerstate']).to eq(power_state)
-      end
-
-    end
-  end
-
-  describe '#create_vm' do
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-      :datastores => ['datastore0'],
-      :vmfolder_tree => {
-        'Templates' => { :children => {
-          'pool1' => { :object_type => 'vm', :name => 'pool1' },
-        }},
-        'Pooler' => { :children => {
-          'pool1' => nil,
-        }},
-      },
-      :hostfolder_tree => {
-        'cluster1' =>  {:object_type => 'compute_resource'},
-      }
-    })
-    }
-
-    let(:clone_vm_task) { mock_RbVmomi_VIM_Task() }
-    let(:new_vm_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-    let(:new_template_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
-    context 'Given an invalid pool name' do
-      it 'should raise an error' do
-        expect{ subject.create_vm('missing_pool', vmname) }.to raise_error(/missing_pool does not exist/)
-      end
-    end
-
-    context 'Given an invalid template path in the pool config' do
-      before(:each) do
-        config[:pools][0]['template'] = 'bad_template'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template that starts with /' do
-      before(:each) do
-        config[:pools][0]['template'] = '/bad_template'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template that ends with /' do
-      before(:each) do
-        config[:pools][0]['template'] = 'bad_template/'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/did not specify a full path for the template/)
-      end
-    end
-
-    context 'Given a template VM that does not exist' do
-      before(:each) do
-        config[:pools][0]['template'] = 'Templates/missing_template'
-        expect(subject).to receive(:find_template_vm).and_raise("specifies a template VM of #{vmname} which does not exist")
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/specifies a template VM of .+ which does not exist/)
-      end
-    end
-
-    context 'when create_vm_folder returns nil' do
-      before(:each) do
-        allow(subject).to receive(:find_template_vm).and_return(new_template_object)
-        expect(subject).to receive(:find_vm_folder).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(ArgumentError)
-      end
-    end
-
-    context 'Given a successful creation' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-      before(:each) do
-        template_vm = new_template_object
-        allow(subject).to receive(:find_template_vm).and_return(new_template_object)
-        allow(template_vm).to receive(:CloneVM_Task).and_return(clone_vm_task)
-        allow(clone_vm_task).to receive(:wait_for_completion).and_return(new_vm_object)
-        allow(subject).to receive(:find_vm_folder).and_return(folder_object)
-      end
-
-      it 'should return a hash' do
-        result = subject.create_vm(poolname, vmname)
-
-        expect(result.is_a?(Hash)).to be true
-      end
-
-      it 'should use the appropriate Create_VM spec' do
-        template_vm = new_template_object
-        expect(template_vm).to receive(:CloneVM_Task)
-          .with(create_vm_spec(vmname,'pool1','datastore0'))
-          .and_return(clone_vm_task)
-
-        subject.create_vm(poolname, vmname)
-      end
-
-      it 'should have the new VM name' do
-        result = subject.create_vm(poolname, vmname)
-
-        expect(result['name']).to eq(vmname)
-      end
-    end
-  end
-
-  describe '#set_network_device' do
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-      :datastores => ['datastore0'],
-      :networks => ['network0'],
-    })
-    }
-    let(:template_vm_network_device)  { mock_RbVmomi_VIM_VirtualVmxnet3() }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
-    context 'Given an invalid network name' do
-      network_name = "invalid_network"
-
-      it 'should raise an error' do
-        expect { subject.set_network_device("datacenter_name", template_vm_network_device, network_name, connection)}.to raise_error(/Cannot find network/)
-      end
-    end
-
-    context 'Given a valid network name' do
-      network_name = "network0"
-      it 'should return the network device' do
-        result = subject.set_network_device("datacenter_name", template_vm_network_device, network_name, connection)
-        expect(result).to be_instance_of(RbVmomi::VIM::VirtualVmxnet3)
-        expect(result.deviceInfo).to be_instance_of(RbVmomi::VIM::Description)
-        expect(result.deviceInfo.summary).to eq('network0')
-        expect(result.backing).to be_instance_of(RbVmomi::VIM::VirtualEthernetCardNetworkBackingInfo)
-        expect(result.backing.network.is_a?(RbVmomi::VIM::Network)).to be true
-        expect(result.backing.network.name).to eq('network0')
-        expect(result.connectable).to be_instance_of(RbVmomi::VIM::VirtualDeviceConnectInfo)
-        expect(result.addressType).to eq('assigned')
-      end
-    end
-  end
-
-  describe '#create_disk' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
-    let(:datastorename) { 'datastore0' }
-    let(:disk_size) { 10 }
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-    end
-
-    context 'Given an invalid pool name' do
-      it 'should raise an error' do
-        expect{ subject.create_disk('missing_pool',vmname,disk_size) }.to raise_error(/missing_pool does not exist/)
-      end
-    end
-
-    context 'Given a missing datastore in the pool config' do
-      before(:each) do
-        config[:pools][0]['datastore'] = nil
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/does not have a datastore defined/)
-      end
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when adding the disk raises an error' do
-      before(:each) do
-        expect(subject).to receive(:add_disk).and_raise(RuntimeError,'Mock Disk Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_disk(poolname,vmname,disk_size) }.to raise_error(/Mock Disk Error/)
-      end
-    end
-
-    context 'when adding the disk succeeds' do
-      before(:each) do
-        expect(subject).to receive(:add_disk).with(vm_object, disk_size, datastorename, connection, datacenter_name)
-      end
-
-      it 'should return true' do
-        expect(subject.create_disk(poolname,vmname,disk_size)).to be true
-      end
-    end
-  end
-
-  describe '#create_snapshot' do
-    let(:snapshot_task) { mock_RbVmomi_VIM_Task() }
-    let(:snapshot_name) { 'snapshot' }
-    let(:snapshot_tree) {{}}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname, vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when snapshot already exists' do
-      let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-      let(:snapshot_tree) { { snapshot_name => { :ref => snapshot_object } } }
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Snapshot #{snapshot_name} .+ already exists /)
-      end
-    end
-
-    context 'when snapshot raises an error' do
-      before(:each) do
-        expect(vm_object).to receive(:CreateSnapshot_Task).and_raise(RuntimeError,'Mock Snapshot Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Mock Snapshot Error/)
-      end
-    end
-
-    context 'when snapshot succeeds' do
-      before(:each) do
-        expect(vm_object).to receive(:CreateSnapshot_Task)
-          .with(create_snapshot_spec(snapshot_name))
-          .and_return(snapshot_task)
-        expect(snapshot_task).to receive(:wait_for_completion).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.create_snapshot(poolname,vmname,snapshot_name)).to be true
-      end
-    end
-  end
-
-  describe '#revert_snapshot' do
-    let(:snapshot_task) { mock_RbVmomi_VIM_Task() }
-    let(:snapshot_name) { 'snapshot' }
-    let(:snapshot_tree) { { snapshot_name => { :ref => snapshot_object } } }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(vm_object)
-    end
-
-    context 'when VM does not exist' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).with(poolname,vmname,connection).and_return(nil)
-      end
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/VM #{vmname} .+ does not exist/)
-      end
-    end
-
-    context 'when snapshot does not exist' do
-      let(:snapshot_tree) {{}}
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Snapshot #{snapshot_name} .+ does not exist /)
-      end
-    end
-
-    context 'when revert to snapshot raises an error' do
-      before(:each) do
-        expect(snapshot_object).to receive(:RevertToSnapshot_Task).and_raise(RuntimeError,'Mock Snapshot Error')
-      end
-
-      it 'should raise an error' do
-        expect{ subject.revert_snapshot(poolname,vmname,snapshot_name) }.to raise_error(/Mock Snapshot Error/)
-      end
-    end
-
-    context 'when revert to snapshot succeeds' do
-      before(:each) do
-        expect(snapshot_object).to receive(:RevertToSnapshot_Task).and_return(snapshot_task)
-        expect(snapshot_task).to receive(:wait_for_completion).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.revert_snapshot(poolname,vmname,snapshot_name)).to be true
-      end
-    end
-  end
-
-  describe '#destroy_vm' do
-    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
-    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'Given a missing VM name' do
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(nil)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, 'missing_vm')).to be true
-      end
-    end
-
-    context 'Given a powered on VM' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :powerstate => 'poweredOn',
-        })
-      }
-
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(vm_object)
-        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        allow(power_off_task).to receive(:wait_for_completion)
-        allow(destroy_task).to receive(:wait_for_completion)
-      end
-
-      it 'should call PowerOffVM_Task on the VM' do
-        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should call Destroy_Task on the VM' do
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, vmname)).to be true
-      end
-    end
-
-    context 'Given a powered off VM' do
-      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-          :powerstate => 'poweredOff',
-        })
-      }
-
-      before(:each) do
-        expect(subject).to receive(:find_vm).and_return(vm_object)
-        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        allow(destroy_task).to receive(:wait_for_completion)
-      end
-
-      it 'should not call PowerOffVM_Task on the VM' do
-        expect(vm_object).to receive(:PowerOffVM_Task).exactly(0).times
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should call Destroy_Task on the VM' do
-        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-
-        subject.destroy_vm(poolname, vmname)
-      end
-
-      it 'should return true' do
-        expect(subject.destroy_vm(poolname, vmname)).to be true
-      end
-    end
-  end
-
-  describe '#vm_ready?' do
-    let(:domain) { nil }
-    context 'When a VM is ready' do
-      before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname, domain)
-      end
-
-      it 'should return true' do
-        expect(subject.vm_ready?(poolname,vmname)).to be true
-      end
-    end
-
-    context 'When a VM is ready but the pool does not exist' do
-      # TODO not sure how to handle a VM that is passed in but
-      # not located in the pool.  Is that ready or not?
-      before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname, domain)
-      end
-
-      it 'should return true' do
-        expect(subject.vm_ready?('missing_pool',vmname)).to be true
-      end
-    end
-
-    context 'When an error occurs connecting to the VM' do
-      # TODO not sure how to handle a VM that is passed in but
-      # not located in the pool.  Is that ready or not?
-      before(:each) do
-        expect(subject).to receive(:open_socket).and_raise(RuntimeError,'MockError')
-      end
-
-      it 'should return false' do
-        expect(subject.vm_ready?(poolname,vmname)).to be false
-      end
-    end
-  end
-
-  describe '#vm_exists?' do
-    it 'should return true when get_vm returns an object' do
-      allow(subject).to receive(:get_vm).with(poolname,vmname).and_return(mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }))
-
-      expect(subject.vm_exists?(poolname,vmname)).to eq(true)
-    end
-
-    it 'should return false when get_vm returns nil' do
-      allow(subject).to receive(:get_vm).with(poolname,vmname).and_return(nil)
-
-      expect(subject.vm_exists?(poolname,vmname)).to eq(false)
-    end
-  end
-
-  # vSphere helper methods
-  describe '#get_target_datacenter_from_config' do
-    let(:pool_dc) { 'PoolDC'}
-    let(:provider_dc) { 'ProvDC'}
-
-    context 'when not specified' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-:pools:
-  - name: '#{poolname}'
-EOT
-        )
-      }
-      it 'returns nil' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to be_nil
-      end
-    end
-
-    context 'when specified only in the pool' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{pool_dc}'
-EOT
-        )
-      }
-      it 'returns the pool datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
-      end
-    end
-
-    context 'when specified only in the provider' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    datacenter: '#{provider_dc}'
-:pools:
-  - name: '#{poolname}'
-EOT
-        )
-      }
-      it 'returns the provider datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(provider_dc)
-      end
-    end
-
-    context 'when specified in the provider and pool' do
-      let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    server: "vcenter.domain.local"
-    username: "vcenter_user"
-    password: "vcenter_password"
-    datacenter: '#{provider_dc}'
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{pool_dc}'
-EOT
-        )
-      }
-      it 'returns the pool datacenter' do
-        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
-      end
-    end
-  end
-
-  # vSphere helper methods
-  describe '#ensured_vsphere_connection' do
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-:providers:
-  :vsphere:
-    # Drop the connection pool timeout way down for spec tests so they fail fast
-    connection_pool_timeout: 1
-    connection_pool_size: 1
-:pools:
-EOT
-      )
-    }
-    let(:connection1) { mock_RbVmomi_VIM_Connection(connection_options) }
-    let(:connection2) { mock_RbVmomi_VIM_Connection(connection_options) }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection1)
-    end
-
-    # This is to ensure that the pool_size of 1 is in effect
-    it 'should return the same connection object when calling the pool multiple times' do
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-      subject.connection_pool.with_metrics do |pool_object|
-        expect(pool_object[:connection]).to be(connection1)
-      end
-    end
-
-    context 'when the connection breaks' do
-      before(:each) do
-        # Emulate the connection state being good, then bad, then good again
-        expect(subject).to receive(:vsphere_connection_ok?).and_return(true, false, true)
-        expect(subject).to receive(:connect_to_vsphere).and_return(connection1, connection2)
-      end
-
-      it 'should restore the connection' do
-        subject.connection_pool.with_metrics do |pool_object|
-          # This line needs to be added to all instances of the connection_pool allocation
-          connection = subject.ensured_vsphere_connection(pool_object)
-
-          expect(connection).to be(connection1)
-        end
-
-        subject.connection_pool.with_metrics do |pool_object|
-          connection = subject.ensured_vsphere_connection(pool_object)
-          # The second connection would have failed.  This test ensures that a
-          # new connection object was created.
-          expect(connection).to be(connection2)
-        end
-
-        subject.connection_pool.with_metrics do |pool_object|
-          connection = subject.ensured_vsphere_connection(pool_object)
-          expect(connection).to be(connection2)
-        end
-      end
-    end
-  end
-
-  describe '#connect_to_vsphere' do
-    before(:each) do
-      allow(RbVmomi::VIM).to receive(:connect).and_return(connection)
-    end
-
-    let (:credentials) { config[:providers][:vsphere] }
-
-    context 'successful connection' do
-      it 'should use the supplied credentials' do
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => credentials['insecure']
-        }).and_return(connection)
-        subject.connect_to_vsphere
-      end
-
-      it 'should honor the insecure setting' do
-        config[:providers][:vsphere][:insecure] = true
-
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => true,
-        }).and_return(connection)
-        subject.connect_to_vsphere
-      end
-
-      it 'should default to an insecure connection' do
-        config[:providers][:vsphere][:insecure] = nil
-
-        expect(RbVmomi::VIM).to receive(:connect).with({
-          :host     => credentials['server'],
-          :user     => credentials['username'],
-          :password => credentials['password'],
-          :insecure => true
-        }).and_return(connection)
-
-        subject.connect_to_vsphere
-      end
-
-      it 'should return the connection object' do
-        result = subject.connect_to_vsphere
-
-        expect(result).to be(connection)
-      end
-
-      it 'should increment the connect.open counter' do
-        expect(metrics).to receive(:increment).with('connect.open')
-        subject.connect_to_vsphere
-      end
-    end
-
-    context 'connection is initially unsuccessful' do
-      before(:each) do
-        # Simulate a failure and then success
-        expect(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError').ordered
-        expect(RbVmomi::VIM).to receive(:connect).and_return(connection).ordered
-
-        allow(subject).to receive(:sleep)
-      end
-
-      it 'should return the connection object' do
-        result = subject.connect_to_vsphere
-
-        expect(result).to be(connection)
-      end
-
-      it 'should increment the connect.fail and then connect.open counter' do
-        expect(metrics).to receive(:increment).with('connect.fail').exactly(1).times
-        expect(metrics).to receive(:increment).with('connect.open').exactly(1).times
-        subject.connect_to_vsphere
-      end
-    end
-
-    context 'connection is always unsuccessful' do
-      before(:each) do
-        allow(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError')
-        allow(subject).to receive(:sleep)
-      end
-
-      it 'should retry the connection attempt config.max_tries times' do
-        expect(RbVmomi::VIM).to receive(:connect).exactly(config[:config]['max_tries']).times
-
-        begin
-          # Swallow any errors
-          subject.connect_to_vsphere
-        rescue
-        end
-      end
-
-      it 'should increment the connect.fail counter config.max_tries times' do
-        expect(metrics).to receive(:increment).with('connect.fail').exactly(config[:config]['max_tries']).times
-
-        begin
-          # Swallow any errors
-          subject.connect_to_vsphere
-        rescue
-        end
-      end
-
-      [{:max_tries => 5, :retry_factor => 1},
-      {:max_tries => 8, :retry_factor => 5},
-      ].each do |testcase|
-        context "Configuration set for max_tries of #{testcase[:max_tries]} and retry_facter of #{testcase[:retry_factor]}" do
-          it "should sleep #{testcase[:max_tries] - 1} times between attempts with increasing timeout" do
-            config[:config]['max_tries'] = testcase[:max_tries]
-            config[:config]['retry_factor'] = testcase[:retry_factor]
-
-            (1..testcase[:max_tries] - 1).each do |try|
-              expect(subject).to receive(:sleep).with(testcase[:retry_factor] * try).ordered
-            end
-
-            begin
-              # Swallow any errors
-              subject.connect_to_vsphere
-            rescue
-            end
-          end
-        end
-      end
-    end
-  end
-
-  describe '#open_socket' do
-    let(:TCPSocket) { double('tcpsocket') }
-    let(:socket) { double('tcpsocket') }
-    let(:hostname) { 'host' }
-    let(:domain) { 'domain.local'}
-    let(:default_socket) { 22 }
-
-    before do
-      expect(subject).not_to be_nil
-      allow(socket).to receive(:close)
-    end
-
-    it 'opens socket with defaults' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-
-      expect(subject.open_socket(hostname)).to eq(nil)
-    end
-
-    it 'yields the socket if a block is given' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-
-      expect{ |socket| subject.open_socket(hostname,nil,nil,default_socket,&socket) }.to yield_control.exactly(1).times 
-    end
-
-    it 'closes the opened socket' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket)
-      expect(socket).to receive(:close)
-
-      expect(subject.open_socket(hostname)).to eq(nil)
-    end
-
-    it 'opens a specific socket' do
-      expect(TCPSocket).to receive(:new).with(hostname,80).and_return(socket)
-
-      expect(subject.open_socket(hostname,nil,nil,80)).to eq(nil)
-    end
-
-    it 'uses a specific domain with the hostname' do
-      expect(TCPSocket).to receive(:new).with("#{hostname}.#{domain}",default_socket).and_return(socket)
-
-      expect(subject.open_socket(hostname,domain)).to eq(nil)
-    end
-
-    it 'raises error if host is not resolvable' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'getaddrinfo: No such host is known')
-
-      expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError)
-    end
-
-    it 'raises error if socket is not listening' do
-      expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'No connection could be made because the target machine actively refused it')
-
-      expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError)
-    end
-  end
-
-  describe '#get_vm_folder_path' do
-    [
-      { :path_description => 'Datacenters/DC01/vm/Pooler/pool1/vm1',
-        :expected_path => 'Pooler/pool1',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'Pooler' },
-          { :type => 'folder',     :name => 'pool1'},
-        ],
-      },
-      { :path_description => 'Datacenters/DC01/vm/something/subfolder/pool/vm1',
-        :expected_path => 'something/subfolder/pool',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'something' },
-          { :type => 'folder',     :name => 'subfolder' },
-          { :type => 'folder',     :name => 'pool'},
-        ],
-      },
-      { :path_description => 'Datacenters/DC01/vm/vm1',
-        :expected_path => '',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-          { :type => 'folder',     :name => 'vm' },
-        ],
-      },
-    ].each do |testcase|
-      context "given a path of #{testcase[:path_description]}" do
-        it "should return '#{testcase[:expected_path]}'" do
-          vm_object = mock_RbVmomi_VIM_VirtualMachine({
-            :name => 'vm1',
-            :path => testcase[:vm_object_path],
-          })
-          expect(subject.get_vm_folder_path(vm_object)).to eq(testcase[:expected_path])
-        end
-      end
-    end
-
-    [
-      { :path_description => 'a path missing a Datacenter',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'folder',     :name => 'vm' },
-          { :type => 'folder',     :name => 'Pooler' },
-          { :type => 'folder',     :name => 'pool1'},
-        ],
-      },
-      { :path_description => 'a path missing root VM folder',
-        :vm_object_path => [
-          { :type => 'folder',     :name => 'Datacenters' },
-          { :type => 'datacenter', :name => 'DC01' },
-        ],
-      },
-    ].each do |testcase|
-      context "given #{testcase[:path_description]}" do
-        it "should return nil" do
-          vm_object = mock_RbVmomi_VIM_VirtualMachine({
-            :name => 'vm1',
-            :path => testcase[:vm_object_path],
-          })
-          expect(subject.get_vm_folder_path(vm_object)).to be_nil
-        end
-      end
-    end
-  end
-
-  describe '#add_disk' do
-    let(:datastorename) { 'datastore' }
-    let(:disk_size) { 30 }
-    let(:collectMultiple_response) { {} }
-
-    let(:vm_scsi_controller) { mock_RbVmomi_VIM_VirtualSCSIController() }
-
-    # Require at least one SCSI Controller
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-      })
-      mock_vm.config.hardware.device << vm_scsi_controller
-
-      mock_vm
-    }
-
-    # Require at least one DC with the required datastore
-    let(:connection_options) {{
-      :serviceContent => {
-        :datacenters => [
-          { :name => datacenter_name, :datastores => [datastorename] }
-        ]
-      }
-    }}
-
-    let(:create_virtual_disk_task) { mock_RbVmomi_VIM_Task() }
-    let(:reconfig_vm_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      # Mocking for find_vmdks
-      allow(connection.serviceContent.propertyCollector).to receive(:collectMultiple).and_return(collectMultiple_response)
-
-      # Mocking for creating the disk
-      allow(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task) do |options|
-        if options[:datacenter][:name] == datacenter_name
-          create_virtual_disk_task
-        end
-      end
-      allow(create_virtual_disk_task).to receive(:wait_for_completion).and_return(true)
-
-      # Mocking for adding disk to the VM
-      allow(vm_object).to receive(:ReconfigVM_Task).and_return(reconfig_vm_task)
-      allow(reconfig_vm_task).to receive(:wait_for_completion).and_return(true)
-    end
-
-    context 'Successfully adding disk' do
-      it 'should return true' do
-        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
-      end
-
-      it 'should request a disk of appropriate size' do
-        expect(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task)
-          .with(create_virtual_disk_with_size(disk_size))
-          .and_return(create_virtual_disk_task)
-
-
-        subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)
-      end
-    end
-
-    context 'Requested disk size is 0' do
-      it 'should raise an error' do
-        expect(subject.add_disk(vm_object,0,datastorename,connection,datacenter_name)).to be false
-      end
-    end
-
-    context 'No datastores or datastore missing' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => ['missing_datastore'] }
-          ]
-        }
-      }}
-
-      it 'should raise error' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(/does not exist/)
-      end
-    end
-
-    context 'Multiple datacenters with multiple datastores' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC', :datastores => ['dc1','dc2'] },
-            { :name => datacenter_name, :datastores => ['dc3',datastorename,'dc4'] },
-          ]
-        }
-      }}
-
-      it 'should return true' do
-        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
-      end
-    end
-
-    context 'VM does not have a SCSI Controller' do
-      let(:vm_object) {
-        mock_vm = mock_RbVmomi_VIM_VirtualMachine({
-          :name => vmname,
-        })
-
-        mock_vm
-      }
-
-      it 'should raise an error' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(NoMethodError)
-      end
-    end
-  end
-
-  describe '#create_clone_spec' do
-    let(:relocate_spec) { mock_RbVmomi_VIM_VirtualMachineRelocateSpec({
-      :datastore => 'datastore0',
-      :diskMoveType => :moveChildMostDiskBacking,
-      :pool => 'pool0'
-    })
-    }
-
-    let(:config_spec) { mock_RbVmomi_VIM_VirtualMachineConfigSpec()}
-
-    it 'should return the configured clone spec' do
-      result = subject.create_clone_spec(relocate_spec, config_spec)
-      expect(result.location.pool.name).to eq('pool0')
-      expect(result.location.datastore.name).to eq('datastore0')
-      expect(result.location.diskMoveType).to eq(:moveChildMostDiskBacking)
-      expect(result.config.deviceChange.first[:operation]).to eq(:edit)
-      expect(result.config.deviceChange.first[:device].is_a?(RbVmomi::VIM::VirtualVmxnet3)).to be true
-    end
-  end
-
-  describe '#find_datastore' do
-    let(:datastorename) { 'datastore' }
-    let(:datastore_list) { [] }
-
-    context 'No datastores in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => [] }
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        expect(result).to be_nil
-      end
-    end
-
-    context 'Many datastores in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name, :datastores => ['ds1','ds2',datastorename,'ds3'] }
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore('missing_datastore',connection,datacenter_name)
-        expect(result).to be_nil
-      end
-
-      it 'should find the datastore in the datacenter' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        
-        expect(result).to_not be_nil
-        expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
-        expect(result.name).to eq(datastorename)
-      end
-    end
-
-    context 'Many datastores in many datacenters' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC', :datastores => ['ds1','ds2','ds3'] },
-            { :name => datacenter_name, :datastores => ['ds3','ds4',datastorename,'ds5'] },
-          ]
-        }
-      }}
-
-      it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore(datastorename,connection,'AnotherDC')
-        expect(result).to be_nil
-      end
-
-      it 'should find the datastore in the datacenter' do
-        result = subject.find_datastore(datastorename,connection,datacenter_name)
-        
-        expect(result).to_not be_nil
-        expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
-        expect(result.name).to eq(datastorename)
-      end
-    end
-  end
-
-  describe '#find_device' do
-    let(:devicename) { 'device1' }
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-      mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      mock_vm
-    }
-
-    it 'should return a device if the device name matches' do
-      result = subject.find_device(vm_object,devicename)
-
-      expect(result.deviceInfo.label).to eq(devicename)
-    end
-
-    it 'should return nil if the device name does not match' do
-      result = subject.find_device(vm_object,'missing_device')
-
-      expect(result).to be_nil
-    end
-  end
-
-  describe '#find_disk_controller' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-
-    it 'should return nil when there are no devices' do
-      result = subject.find_disk_controller(vm_object)
-
-      expect(result).to be_nil
-    end
-
-    [0,1,14].each do |testcase|
-      it "should return a device for a single VirtualSCSIController with #{testcase} attached disks" do
-        mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-        vm_object.config.hardware.device << mock_scsi
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-        # Add the disks
-        (1..testcase).each do
-          vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key })
-        end
-
-        result = subject.find_disk_controller(vm_object)
-
-        expect(result).to eq(mock_scsi)
-      end
-    end
-
-    [15].each do |testcase|
-      it "should return nil for a single VirtualSCSIController with #{testcase} attached disks" do
-        mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-        vm_object.config.hardware.device << mock_scsi
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-        vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-        # Add the disks
-        (1..testcase).each do
-          vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key })
-        end
-
-        result = subject.find_disk_controller(vm_object)
-
-        expect(result).to be_nil
-      end
-    end
-
-    it 'should raise if a VirtualDisk is missing a controller' do
-      # Note - Typically this is not possible as a VirtualDisk requires a controller (SCSI, PVSCSI or IDE)
-      mock_scsi = mock_RbVmomi_VIM_VirtualDisk()
-      vm_object.config.hardware.device << mock_scsi
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      expect{subject.find_disk_controller(vm_object)}.to raise_error(NoMethodError)
-    end
-  end
-
-  describe '#find_disk_devices' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-
-    it 'should return empty hash when there are no devices' do
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result).to eq({})
-    end
-
-    it 'should return empty hash when there are no VirtualSCSIController or VirtualDisk devices' do
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result).to eq({})
-    end
-
-    it 'should return a device for a VirtualSCSIController device with no children' do
-      mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController()
-      vm_object.config.hardware.device << mock_scsi
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(1)
-      expect(result[mock_scsi.key]).to_not be_nil
-      expect(result[mock_scsi.key]['children']).to eq([])
-      expect(result[mock_scsi.key]['device']).to eq(mock_scsi)
-    end
-
-    it 'should return a device for a VirtualDisk device' do
-      mock_disk = mock_RbVmomi_VIM_VirtualDisk()
-      vm_object.config.hardware.device << mock_disk
-      vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'})
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(1)
-      expect(result[mock_disk.controllerKey]).to_not be_nil
-      expect(result[mock_disk.controllerKey]['children'][0]).to eq(mock_disk)
-    end
-
-    it 'should return one device for many VirtualDisk devices on the same controller' do
-      controller1Key = rand(2000)
-      controller2Key = controller1Key + 1
-      mock_disk1 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key})
-      mock_disk2 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key})
-      mock_disk3 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller2Key})
-
-      vm_object.config.hardware.device << mock_disk2
-      vm_object.config.hardware.device << mock_disk1
-      vm_object.config.hardware.device << mock_disk3
-
-      result = subject.find_disk_devices(vm_object)
-
-      expect(result.count).to eq(2)
-
-      expect(result[controller1Key]).to_not be_nil
-      expect(result[controller2Key]).to_not be_nil
-
-      expect(result[controller1Key]['children']).to contain_exactly(mock_disk1,mock_disk2)
-      expect(result[controller2Key]['children']).to contain_exactly(mock_disk3)
-    end
-  end
-
-  describe '#find_disk_unit_number' do
-    let(:vm_object) {
-      mock_vm = mock_RbVmomi_VIM_VirtualMachine()
-
-      mock_vm
-    }
-    let(:controller) { mock_RbVmomi_VIM_VirtualSCSIController() }
-
-    it 'should return 0 when there are no devices' do
-      result = subject.find_disk_unit_number(vm_object,controller)
-
-      expect(result).to eq(0)
-    end
-
-    context 'with a single SCSI Controller' do
-      before(:each) do
-        vm_object.config.hardware.device << controller
-      end
-
-      it 'should return 1 when the host bus controller is at 0' do
-        controller.scsiCtlrUnitNumber = 0
-
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(1)
-      end
-
-      it 'should return the next lowest id when disks are attached' do
-        expected_id = 9
-        controller.scsiCtlrUnitNumber = 0
-
-        (1..expected_id-1).each do |disk_id|
-          mock_disk = mock_RbVmomi_VIM_VirtualDisk({
-            :controllerKey => controller.key,
-            :unitNumber => disk_id,
-          })
-          vm_object.config.hardware.device << mock_disk
-        end
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(expected_id)
-      end
-
-      it 'should return nil when there are no spare units' do
-        controller.scsiCtlrUnitNumber = 0
-
-        (1..15).each do |disk_id|
-          mock_disk = mock_RbVmomi_VIM_VirtualDisk({
-            :controllerKey => controller.key,
-            :unitNumber => disk_id,
-          })
-          vm_object.config.hardware.device << mock_disk
-        end
-        result = subject.find_disk_unit_number(vm_object,controller)
-
-        expect(result).to eq(nil)
-      end
-    end
-  end
-
-  describe '#find_vm_folder' do
-    let(:foldername) { 'folder'}
-
-    context 'with no folder hierarchy' do
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy in many datacenters' do
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-
-    context 'with a multi layer folder hierarchy' do
-      let(:foldername) { 'folder2/folder4/folder' }
-      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername}) }
-
-      it 'should return the folder when found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(folder_object)
-        allow(folder_object).to receive(:class).and_return(RbVmomi::VIM::Folder)
-        result = subject.find_vm_folder(poolname,connection)
-        expect(result.name).to eq(foldername)
-      end
-
-      it 'should return nil if the folder is not found' do
-        allow(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-        expect(subject.find_vm_folder(poolname,connection)).to be_nil
-      end
-    end
-  end
-
-  describe '#get_host_utilization' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:limit) { 75 }
-    let(:default_limit) { 90 }
-
-    context "host with a different model" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem() }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,different_model,limit)).to be_nil
-      end
-    end
-
-    context "host in maintenance mode" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :maintenance_mode => true,
-        })
-      }
-      it 'should return nil' do
-        host.runtime.inMaintenanceMode = true
-
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host with status of not green" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_status => 'purple_alert',
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host with configuration issue" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :config_issue => 'No quickstats',
-      })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    # CPU utilization
-    context "host which exceeds limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 100,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host which exceeds default limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => default_limit + 1.0,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model)).to be_nil
-      end
-    end
-
-    context "host which does not exceed default limit in CPU utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => default_limit,
-        :overall_memory_usage => 1,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should not return nil' do
-        expect(subject.get_host_utilization(host,model)).to_not be_nil
-      end
-    end
-
-    # Memory utilization
-    context "host which exceeds limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => 100,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        # Set the Memory Usage to 100%
-        expect(subject.get_host_utilization(host,model,limit)).to be_nil
-      end
-    end
-
-    context "host which exceeds default limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => default_limit + 1.0,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return nil' do
-        expect(subject.get_host_utilization(host,model)).to be_nil
-      end
-    end
-
-    context "host which does not exceed default limit in Memory utilization" do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 1,
-        :overall_memory_usage => default_limit,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should not return nil' do
-        expect(subject.get_host_utilization(host,model)).to_not be_nil
-      end
-    end
-
-    context "host which does not exceed limits" do
-      # Set CPU to 10%
-      # Set Memory to 20%
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :overall_cpu_usage => 10,
-        :overall_memory_usage => 20,
-        :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-        })
-      }
-      it 'should return the sum of CPU and Memory utilization' do
-        expect(subject.get_host_utilization(host,model,limit)[0]).to eq(10)
-      end
-
-      it 'should return the host' do
-        expect(subject.get_host_utilization(host,model,limit)[1]).to eq(host)
-      end
-    end
-
-    context 'host with no quickstats' do
-      let(:host) { mock_RbVmomi_VIM_HostSystem({
-        :cpu_speed => 100,
-        :num_cores_per_cpu => 1,
-        :num_cpu => 1,
-        :memory_size => 100.0 * 1024 * 1024
-        })
-      }
-      before(:each) do
-        host.summary.quickStats.overallCpuUsage = nil
-      end
-
-      it 'should return nil' do
-        result = subject.get_host_utilization(host,model,limit)
-        expect(result).to be nil
-      end
-    end
-  end
-
-  describe '#host_has_cpu_model?' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:host) { mock_RbVmomi_VIM_HostSystem({
-      :cpu_model => cpu_model,
-      })
-    }
-
-    it 'should return true if the model matches' do
-      expect(subject.host_has_cpu_model?(host,model)).to eq(true)
-    end
-
-    it 'should return false if the model is different' do
-      expect(subject.host_has_cpu_model?(host,different_model)).to eq(false)
-    end
-  end
-
-  describe '#get_host_cpu_arch_version' do
-    let(:cpu_model) { 'vendor line type sku v4 speed' }
-    let(:model) { 'v4' }
-    let(:different_model) { 'different_model' }
-    let(:host) { mock_RbVmomi_VIM_HostSystem({
-      :cpu_model => cpu_model,
-      :num_cpu => 2,
-      })
-    }
-
-    it 'should return the fifth element in the string delimited by spaces' do
-      expect(subject.get_host_cpu_arch_version(host)).to eq(model)
-    end
-
-    it 'should use the description of the first CPU' do
-      host.hardware.cpuPkg[0].description = 'vendor line type sku v6 speed'
-      expect(subject.get_host_cpu_arch_version(host)).to eq('v6')
-    end
-  end
-
-  describe '#cpu_utilization_for' do
-    [{ :cpu_usage => 10.0,
-      :core_speed => 10.0,
-      :num_cores => 2,
-      :expected_value => 50.0,
-    },
-    { :cpu_usage => 10.0,
-      :core_speed => 10.0,
-      :num_cores => 4,
-      :expected_value => 25.0,
-    },
-    { :cpu_usage => 14.0,
-      :core_speed => 12.0,
-      :num_cores => 5,
-      :expected_value => 23.0 + 1.0/3.0,
-    },
-    ].each do |testcase|
-      context "CPU Usage of #{testcase[:cpu_usage]}MHz with #{testcase[:num_cores]} x #{testcase[:core_speed]}MHz cores" do
-        it "should be #{testcase[:expected_value]}%" do
-          host = mock_RbVmomi_VIM_HostSystem({
-            :num_cores_per_cpu => testcase[:num_cores],
-            :cpu_speed         => testcase[:core_speed],
-            :overall_cpu_usage => testcase[:cpu_usage],
-          })
-
-          expect(subject.cpu_utilization_for(host)).to eq(testcase[:expected_value])
-        end
-      end
-    end
-  end
-
-  describe '#memory_utilization_for' do
-    [{ :memory_usage_gigbytes => 10.0,
-      :memory_size_bytes => 10.0 * 1024 * 1024,
-      :expected_value => 100.0,
-    },
-    { :memory_usage_gigbytes => 15.0,
-      :memory_size_bytes => 25.0 * 1024 * 1024,
-      :expected_value => 60.0,
-    },
-    { :memory_usage_gigbytes => 9.0,
-      :memory_size_bytes => 31.0 * 1024 * 1024,
-      :expected_value => 29.03225806451613,
-    },
-    ].each do |testcase|
-      context "Memory Usage of #{testcase[:memory_usage_gigbytes]}GBytes with #{testcase[:memory_size_bytes]}Bytes of total memory" do
-        it "should be #{testcase[:expected_value]}%" do
-          host = mock_RbVmomi_VIM_HostSystem({
-            :memory_size          => testcase[:memory_size_bytes],
-            :overall_memory_usage => testcase[:memory_usage_gigbytes],
-          })
-
-          expect(subject.memory_utilization_for(host)).to eq(testcase[:expected_value])
-        end
-      end
-    end
-  end
-
-  describe '#select_target_hosts' do
-    let(:target) { {} }
-    let(:cluster) { 'cluster1' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:hosts_hash) {
-      {
-        'hosts' => [host],
-        'architectures' => {
-          architecture => [host]
-        }
-      }
-    }
-
-    it 'returns a hash of the least used hosts by cluster and architecture' do
-      expect(subject).to receive(:find_least_used_hosts).and_return(hosts_hash)
-
-      subject.select_target_hosts(target, cluster, datacenter)
-      expect(target["#{datacenter}_#{cluster}"]).to eq(hosts_hash)
-    end
-
-    context 'with a cluster specified that does not exist' do
-      it 'raises an error' do
-        expect(subject).to receive(:find_least_used_hosts).with(missing_cluster_name, datacenter, 100).and_raise("Cluster #{cluster} cannot be found")
-        expect{subject.select_target_hosts(target, missing_cluster_name, datacenter)}.to raise_error(RuntimeError,/Cluster #{cluster} cannot be found/)
-      end
-    end
-  end
-
-  describe '#get_average_cluster_utilization' do
-    let(:hosts) {
-      [
-        [60, 'host1'],
-        [100, 'host2'],
-        [200, 'host3']
-      ]
-    }
-    it 'returns the average utilization for a given set of host utilizations assuming the first member of the list for each host is the utilization value' do
-      expect(subject.get_average_cluster_utilization(hosts)).to eq(120)
-    end
-  end
-
-  describe '#build_compatible_hosts_lists' do
-    let(:host1) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST1' })}
-    let(:host2) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST2' })}
-    let(:host3) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST3' })}
-    let(:architecture) { 'v4' }
-    let(:percentage) { 100 }
-    let(:hosts) {
-      [
-        [60, host1],
-        [100, host2],
-        [200, host3]
-      ]
-    }
-    let(:result) {
-      {
-          architecture => ['HOST1','HOST2']
-      }
-    }
-
-    it 'returns a hash of target host architecture versions containing lists of target hosts' do
-
-      expect(subject.build_compatible_hosts_lists(hosts, percentage)).to eq(result)
-    end
-  end
-
-  describe '#select_least_used_hosts' do
-    let(:percentage) { 100 }
-    let(:host1) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST1' })}
-    let(:host2) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST2' })}
-    let(:host3) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST3' })}
-    let(:hosts) {
-      [
-        [60, host1],
-        [100, host2],
-        [200, host3]
-      ]
-    }
-    let(:result) { ['HOST1','HOST2'] }
-    it 'returns the percentage specified of the least used hosts in the cluster determined by selecting from less than or equal to average cluster utilization' do
-      expect(subject.select_least_used_hosts(hosts, percentage)).to eq(result)
-    end
-
-    context 'when selecting 20 percent of hosts below average' do
-      let(:percentage) { 20 }
-      let(:result) { ['HOST1'] }
-
-      it 'should return the result' do
-        expect(subject.select_least_used_hosts(hosts, percentage)).to eq(result)
-      end
-    end
-
-    it 'should raise' do
-      expect{subject.select_least_used_hosts([], percentage)}.to raise_error(RuntimeError,/Provided hosts list to select_least_used_hosts is empty/)
-    end
-  end
-
-  describe '#run_select_hosts' do
-    let(:target) { {} }
-    let(:cluster) { 'cluster1' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:loop_delay) { 5 }
-    let(:max_age) { 60 }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:hosts_hash) {
-      {
-       dc => {
-          'hosts' => [host],
-          'architectures' => {
-            architecture => [host]
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_age: 60
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'target does not have key dc' do
-      let(:hosts_hash) { { } }
-      it 'should run select_target_hosts' do
-        expect(subject).to receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when check_time_finished is greater than max_age' do
-      before(:each) do
-        hosts_hash[dc]['check_time_finished'] = Time.now - (max_age + 10)
-      end
-
-      it 'should run select_target_hosts' do
-        expect(subject).to receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when check_time_finished is not greater than max_age' do
-      before(:each) do
-        hosts_hash[dc]['check_time_finished'] = Time.now - (max_age / 2)
-      end
-
-      it 'should not run select_target_hosts' do
-        expect(subject).to_not receive(:select_target_hosts).with(hosts_hash, cluster, datacenter)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-    end
-
-    context 'when hosts are being selected' do
-      before(:each) do
-        hosts_hash[dc]['checking'] = true
-      end
-
-      it 'should run wait_for_host_selection' do
-        expect(subject).to receive(:wait_for_host_selection).with(dc, hosts_hash, loop_delay, max_age)
-        subject.run_select_hosts(poolname, hosts_hash)
-      end
-
-    end
-
-    context 'with no clone_target' do
-      before(:each) do
-        config[:pools][0].delete('clone_target')
-      end
-
-      it 'should raise an error' do
-        expect{subject.run_select_hosts(poolname, hosts_hash)}.to raise_error(/cluster for pool #{poolname} cannot be identified/)
-      end
-    end
-
-    context 'with no datacenter' do
-      before(:each) do
-        config[:pools][0].delete('datacenter')
-      end
-
-      it 'should raise an error' do
-        expect{subject.run_select_hosts(poolname, hosts_hash)}.to raise_error(/datacenter for pool #{poolname} cannot be identified/)
-      end
-    end
-  end
-
-  describe '#wait_for_host_selection' do
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:maxloop) { 1 }
-    let(:loop_delay) { 1 }
-    let(:max_age) { 60 }
-    let(:target) {
-      {
-        dc => { }
-      }
-    }
-
-    context 'when target does not have key check_time_finished' do
-      it 'should sleep for loop_delay maxloop times' do
-        expect(subject).to receive(:sleep).with(loop_delay).once
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-
-    context 'when target has dc and check_time_finished is greater than max_age' do
-      before(:each) do
-        target[dc]['check_time_finished'] = Time.now - (max_age + 10)
-      end
-
-      it 'should sleep for loop_delay maxloop times' do
-        expect(subject).to receive(:sleep).with(loop_delay).once
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-
-    context 'when target has dc and check_time_finished difference from now is less than max_age' do
-      before(:each) do
-        target[dc]['check_time_finished'] = Time.now - (max_age / 2)
-      end
-
-      it 'should not wait' do
-        expect(subject).to_not receive(:sleep).with(loop_delay)
-
-        subject.wait_for_host_selection(dc, target, maxloop, loop_delay, max_age)
-      end
-    end
-  end
-
-  describe '#select_next_host' do
-    let(:cluster) { 'cluster1' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v3' }
-    let(:host) { 'host1' }
-    let(:loop_delay) { 5 }
-    let(:max_age) { 60 }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:hosts_hash) {
-      {
-       dc => {
-          'hosts' => [host, 'host2'],
-          'architectures' => {
-            architecture => ['host3', 'host4']
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  max_age: 60
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    context 'when host is requested' do
-      it 'should return the first host' do
-        result = subject.select_next_host(poolname, hosts_hash)
-        expect(result).to eq(host)
-      end
-
-      it 'should return the second host if called twice' do
-        subject.select_next_host(poolname, hosts_hash)
-        result2 = subject.select_next_host(poolname, hosts_hash)
-        expect(result2).to eq('host2')
-      end
-    end
-
-    context 'when host and architecture are requested' do
-      it 'should return the first host' do
-        result = subject.select_next_host(poolname, hosts_hash, architecture)
-        expect(result).to eq('host3')
-      end
-
-      it 'should return the second host if called twice' do
-        subject.select_next_host(poolname, hosts_hash, architecture)
-        result2 = subject.select_next_host(poolname, hosts_hash, architecture)
-        expect(result2).to eq('host4')
-      end
-    end
-
-    context 'with no hosts available' do
-      before(:each) do
-        hosts_hash[dc].delete('hosts')
-        hosts_hash[dc].delete('architectures')
-      end
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-
-      it 'should raise an error when selecting with architecture' do
-        expect{subject.select_next_host(poolname, hosts_hash, architecture)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-    end
-
-    context 'with no datacenter set for pool' do
-      before(:each) do
-        config[:pools][0].delete('datacenter')
-      end
-
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("datacenter for pool #{poolname} cannot be identified")
-      end
-    end
-
-    context 'with no clone_target set for pool' do
-      before(:each) do
-        config[:pools][0].delete('clone_target')
-      end
-
-      it 'should raise an error' do
-        expect{subject.select_next_host(poolname, hosts_hash)}.to raise_error("cluster for pool #{poolname} cannot be identified")
-      end
-    end
-  end
-
-  describe '#vm_in_target?' do
-    let(:parent_host) { 'host1' }
-    let(:architecture) { 'v3' }
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:maxloop) { 1 }
-    let(:loop_delay) { 1 }
-    let(:max_age) { 60 }
-    let(:target) {
-      {
-        dc => {
-          'hosts' => [parent_host],
-          'architectures' => {
-            architecture => [parent_host]
-          }
-        }
-      }
-    }
-    let(:config) { YAML.load(<<-EOT
----
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-
-    it 'returns true when parent_host is in hosts' do
-      result = subject.vm_in_target?(poolname, parent_host, architecture, target)
-      expect(result).to be true
-    end
-
-    context 'with parent_host only in architectures' do
-      before(:each) do
-        target[dc]['hosts'] = ['host2']
-      end
-
-      it 'returns true when parent_host is in architectures' do
-        result = subject.vm_in_target?(poolname, parent_host, architecture, target)
-        expect(result).to be true
-      end
-    end
-
-    it 'returns false when parent_host is not in hosts or architectures' do
-      result = subject.vm_in_target?(poolname, 'host3', architecture, target)
-      expect(result).to be false
-    end
-
-    context 'with no hosts key' do
-      before(:each) do
-        target[dc].delete('hosts')
-      end
-
-      it 'should raise an error' do
-        expect{subject.vm_in_target?(poolname, parent_host, architecture, target)}.to raise_error("there is no candidate in vcenter that meets all the required conditions, that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory")
-      end
-    end
-  end
-
-  describe '#get_vm_details' do
-    let(:parent_host) { 'host1' }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:architecture) { 'v4' }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it 'returns nil when vm_object is not found' do
-      expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(nil)
-      result = subject.get_vm_details(poolname, vmname, connection)
-      expect(result).to be nil
-    end
-
-    it 'raises an error when unable to determine parent_host' do
-      expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-      expect{subject.get_vm_details(poolname, vmname, connection)}.to raise_error('Unable to determine which host the VM is running on')
-    end
-
-    context 'when it can find parent host' do
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'returns vm details hash' do
-        expect(subject).to receive(:find_vm).with(poolname, vmname, connection).and_return(vm_object)
-        result = subject.get_vm_details(poolname, vmname, connection)
-        expect(result).to eq(vm_details)
-      end
-    end
-  end
-
-  describe '#migrate_vm' do
-    let(:parent_host) { 'host1' }
-    let(:new_host) { 'host2' }
-    let(:datacenter) { 'dc1' }
-    let(:architecture) { 'v4' }
-    let(:cluster) { 'cluster1' }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:new_host_object) { mock_RbVmomi_VIM_HostSystem({ :name => new_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  migration_limit: 5
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-    let(:dc) { "#{datacenter}_#{cluster}" }
-    let(:provider_hosts) {
-      {
-        dc => {
-          'hosts' => [new_host],
-          'architectures' => {
-            architecture => [new_host]
-          },
-          'check_time_finished' => Time.now
-        }
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'when vm should be migrated' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to receive(:run_select_hosts).with(poolname, {})
-          expect(subject).to receive(:vm_in_target?).and_return false
-          expect(subject).to receive(:migration_enabled?).and_return true
-          redis_connection_pool.with do |redis|
-            redis.hset("vmpooler__vm__#{vmname}", 'checkout', Time.now)
-          end
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'logs a message' do
-        expect(subject).to receive(:select_next_host).and_return(new_host)
-        expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
-        expect(subject).to receive(:migrate_vm_host).with(vm_object, new_host_object)
-        expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from #{parent_host} to #{new_host} in 0.00 seconds")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-
-      it 'migrates a vm' do
-        expect(subject).to receive(:migrate_vm_to_new_host).with(poolname, vmname, vm_details, connection)
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'current host is in the list of target hosts' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to receive(:run_select_hosts).with(poolname, {})
-          expect(subject).to receive(:vm_in_target?).and_return true
-          expect(subject).to receive(:migration_enabled?).and_return true
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'logs a message that no migration is required' do
-        expect(logger).to receive(:log).with('s', "[ ] [#{poolname}] No migration required for '#{vmname}' running on #{parent_host}")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'with migration limit exceeded' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to_not receive(:run_select_hosts)
-          expect(subject).to_not receive(:vm_in_target?)
-          expect(subject).to receive(:migration_enabled?).and_return true
-          redis_connection_pool.with do |redis|
-            expect(redis).to receive(:scard).with('vmpooler__migration').and_return(5)
-          end
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-
-      it 'should log the current host' do
-        expect(logger).to receive(:log).with('s', "[ ] [#{poolname}] '#{vmname}' is running on #{parent_host}. No migration will be evaluated since the migration_limit has been reached")
-
-        subject.migrate_vm(poolname, vmname)
-      end
-    end
-
-    context 'when an error occurs' do
-      before(:each) do
-        subject.connection_pool.with_metrics do |pool_object|
-          expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect(subject).to receive(:run_select_hosts)
-          expect(subject).to receive(:vm_in_target?).and_return false
-          expect(subject).to receive(:migration_enabled?).and_return true
-          expect(subject).to receive(:select_next_host).and_raise(RuntimeError,'Mock migration error')
-        end
-        vm_object.summary.runtime.host = host_object
-      end
-      it 'should log the current host' do
-        expect(logger).to receive(:log).with('s', "[!] [#{poolname}] '#{vmname}' is running on #{parent_host}")
-
-        expect{subject.migrate_vm(poolname, vmname)}.to raise_error(RuntimeError, 'Mock migration error')
-      end
-    end
-  end
-
-  describe '#migrate_vm_to_new_host' do
-    let(:parent_host) { 'host1' }
-    let(:new_host) { 'host2' }
-    let(:datacenter) { 'dc1' }
-    let(:cluster) { 'cluster1' }
-    let(:architecture) { 'v4' }
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => parent_host })}
-    let(:new_host_object) { mock_RbVmomi_VIM_HostSystem({ :name => new_host })}
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:config) { YAML.load(<<-EOT
----
-:config:
-  migration_limit: 5
-:pools:
-  - name: '#{poolname}'
-    datacenter: '#{datacenter}'
-    clone_target: '#{cluster}'
-EOT
-      )
-    }
-    let(:vm_details) {
-      {
-        'host_name' => parent_host,
-        'object' => vm_object,
-        'architecture' => architecture
-      }
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it' migrates a vm' do
-      expect(subject).to receive(:select_next_host).and_return(new_host)
-      expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
-      expect(subject).to receive(:migrate_vm_and_record_timing).and_return(format('%.2f', (Time.now - (Time.now - 15))))
-      expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from host1 to host2 in 15.00 seconds")
-      subject.migrate_vm_to_new_host(poolname, vmname, vm_details, connection)
-    end
-  end
-
-  describe '#create_folder' do
-    let(:datacenter) { 'dc1' }
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
-    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
-    let(:new_folder) { poolname }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    it 'creates a folder' do
-      expect(connection.serviceInstance).to receive(:find_datacenter).with(datacenter).and_return(datacenter_object)
-      expect(datacenter_object.vmFolder).to receive(:traverse).with(new_folder, RbVmomi::VIM::Folder, true).and_return(folder_object)
-
-      result = subject.create_folder(connection, new_folder, datacenter)
-      expect(result).to eq(folder_object)
-    end
-
-    context 'with folder_object returning nil' do
-      it 'shoud raise an error' do
-        expect(connection.serviceInstance).to receive(:find_datacenter).with(datacenter).and_return(datacenter_object)
-        expect(datacenter_object.vmFolder).to receive(:traverse).with(new_folder, RbVmomi::VIM::Folder, true).and_return(nil)
-
-        expect{subject.create_folder(connection, new_folder, datacenter)}.to raise_error("Cannot create folder #{new_folder}")
-      end
-    end
-  end
-
-  describe '#migration_enabled?' do
-    let(:config) {
-      { :config => { 'migration_limit' => 5 } }
-    }
-    it 'returns true if migration is enabled' do
-      result = subject.migration_enabled?(config)
-      expect(result).to be true
-    end
-
-    context 'with migration disabled' do
-      let(:config) {
-        { :config => { } }
-      }
-      it 'should return false' do
-        result = subject.migration_enabled?(config)
-        expect(result).to be false
-      end
-    end
-
-    context 'with non-integer value for migration limit' do
-      let(:config) {
-        { :config => { 'migration_limit' => 1.0 } }
-      }
-      it 'should return false' do
-        result = subject.migration_enabled?(config)
-        expect(result).to be false
-      end
-    end
-  end
-
-  describe '#find_least_used_hosts' do
-    let(:cluster_name) { 'cluster' }
-    let(:missing_cluster_name) { 'missing_cluster' }
-    let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
-    let(:percentage) { 100 }
-
-    before(:each) do
-      # This mocking is a little fragile but hard to do without a real vCenter instance
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-      datacenter_object.hostFolder.childEntity = [cluster_object]
-    end
-
-    context 'missing cluster' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-      }]})}
-      let(:expected_host) { cluster_object.host[0] }
-#,datacenter_name
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-      }]})}
-      let(:expected_host) { cluster_object.host[0][:name] }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'standalone host outside the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [{
-          :name => cluster_name,
-          :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024,
-      }]})}
-      let(:expected_host) { cluster_object.host[0] }
-
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'cluster of 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1].name }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      it 'should raise an error' do
-        expect{subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)}.to raise_error(RuntimeError,/Cluster #{missing_cluster_name} cannot be found/)
-      end
-    end
-
-    context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1].name }
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_hosts(cluster_name,datacenter_name,percentage)
-
-        expect(result['hosts'][0]).to be(expected_host)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({
-        :name => cluster_name,
-        :hosts => [
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-          { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      it 'should return a host' do
-        pending('https://github.com/puppetlabs/vmpooler/issues/206')
-        result = subject.find_least_used_hosts(missing_cluster_name,datacenter_name,percentage)
-        expect(result).to_not be_nil
-      end
-    end
-  end
-
-  describe '#find_cluster' do
-    let(:cluster) {'cluster'}
-    let(:host) { 'host' }
-    let(:missing_cluster) {'missing_cluster'}
-
-    context 'no clusters in the datacenter' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name }
-          ]
-        }
-      }}
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'cluster_compute_resource'},
-                'cluster2' => {:object_type => 'cluster_compute_resource'},
-                cluster => {:object_type => 'cluster_compute_resource'},
-                'cluster3' => {:object_type => 'cluster_compute_resource'},
-                host => {:object_type => 'compute_resource'},
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return the single host when found' do
-        result = subject.find_cluster(host,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(host)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-
-    context 'with a single layer folder hierarchy with multiple datacenters' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => 'AnotherDC',
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'cluster_compute_resource'},
-                'cluster2' => {:object_type => 'cluster_compute_resource'},
-              }
-            },
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                cluster => {:object_type => 'cluster_compute_resource'},
-                'cluster3' => {:object_type => 'cluster_compute_resource'},
-                host => {:object_type => 'compute_resource'}
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return the single host when found' do
-        result = subject.find_cluster(host,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(host)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,'AnotherDC')).to be_nil
-      end
-    end
-
-    context 'with a multi layer folder hierarchy' do
-      let(:connection_options) {{
-        :serviceContent => {
-          :datacenters => [
-            { :name => datacenter_name,
-              :hostfolder_tree => {
-                'cluster1' =>  {:object_type => 'cluster_compute_resource'},
-                'folder2' => {
-                  :children => {
-                    cluster => {:object_type => 'cluster_compute_resource'},
-                  }
-                },
-                'cluster3' => {:object_type => 'cluster_compute_resource'},
-                'folder4' => {
-                  :children => {
-                    host => {:object_type => 'compute_resource'},
-                  }
-                }
-              }
-            }
-          ]
-        }
-      }}
-
-      it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(cluster)
-      end
-
-      it 'should return the host when found' do
-        result = subject.find_cluster(host,connection,datacenter_name)
-
-        expect(result).to_not be_nil
-        expect(result.name).to eq(host)
-      end
-
-      it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
-      end
-    end
-  end
-
-  describe '#get_cluster_host_utilization' do
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) }
-      
-      it 'should return array with one element' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(1)
-      end
-    end
-
-    context 'standalone host which is out the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 0 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(0)
-      end
-    end
-
-    context 'cluster with 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 3 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(3)
-      end
-    end
-
-    context 'cluster with 5 hosts of which 3 within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 3 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(3)
-      end
-    end
-
-    context 'cluster with 3 hosts of which none are within the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      
-      it 'should return array with 0 elements' do
-        result = subject.get_cluster_host_utilization(cluster_object)
-        expect(result).to_not be_nil
-        expect(result.count).to eq(0)
-      end
-    end
-  end
-
-  describe '#find_least_used_vpshere_compatible_host' do
-    let(:vm) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    context 'standalone host within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) }
-      let(:standalone_host) { cluster_object.host[0] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = standalone_host
-      end
-
-      it 'should return the standalone host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(standalone_host)
-        expect(result[1]).to eq(standalone_host.name)
-      end
-    end
-
-    context 'standalone host outside of limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:standalone_host) { cluster_object.host[0] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = standalone_host
-      end
-
-      it 'should raise error' do
-        expect{subject.find_least_used_vpshere_compatible_host(vm)}.to raise_error(/There is no host candidate in vcenter that meets all the required conditions/)
-      end
-    end
-
-    context 'cluster of 3 hosts within limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return the least used host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(expected_host)
-        expect(result[1]).to eq(expected_host.name)
-      end
-    end
-
-    context 'cluster of 3 hosts all outside of the limits' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should raise error' do
-        expect{subject.find_least_used_vpshere_compatible_host(vm)}.to raise_error(/There is no host candidate in vcenter that meets all the required conditions/)
-      end
-    end
-
-    context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[2] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return the least used host' do
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-        expect(result[0]).to be(expected_host)
-        expect(result[1]).to eq(expected_host.name)
-      end
-    end
-
-    context 'cluster of 3 hosts all with the same utilisation' do
-      let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-        { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 },
-      ]}) }
-      let(:expected_host) { cluster_object.host[1] }
-
-      before(:each) do
-        # This mocking is a little fragile but hard to do without a real vCenter instance
-        vm.summary.runtime.host = expected_host
-      end
-
-      it 'should return a host' do
-        pending('https://github.com/puppetlabs/vmpooler/issues/206 is fixed')
-        result = subject.find_least_used_vpshere_compatible_host(vm)
-
-        expect(result).to_not be_nil
-      end
-    end
-  end
-
-  describe '#find_snapshot' do
-    let(:snapshot_name) {'snapshot'}
-    let(:missing_snapshot_name) {'missing_snapshot'}
-    let(:vm) { mock_RbVmomi_VIM_VirtualMachine(mock_options) }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachineSnapshot() }
-
-    context 'VM with no snapshots' do
-      let(:mock_options) {{ :snapshot_tree => nil }}
-      it 'should return nil' do
-        expect(subject.find_snapshot(vm,snapshot_name)).to be_nil
-      end
-    end
-
-    context 'VM with a single layer of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => nil,
-          'snapshot4' => nil,
-          snapshot_name => { :ref => snapshot_object},
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.find_snapshot(vm,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-
-      it 'should return nil which no matches are found' do
-        result = subject.find_snapshot(vm,missing_snapshot_name)
-        expect(result).to be_nil
-      end
-    end
-
-    context 'VM with a nested layers of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => { :children => {
-            'snapshot4' => nil,
-            'snapshot5' => { :children => {
-              snapshot_name => { :ref => snapshot_object},
-            }},
-          }},
-          'snapshot6' => nil,
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.find_snapshot(vm,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-
-      it 'should return nil which no matches are found' do
-        result = subject.find_snapshot(vm,missing_snapshot_name)
-        expect(result).to be_nil
-      end
-    end
-  end
-
-  describe '#find_vm' do
-    let(:missing_vm) { 'missing_vm' }
-    let(:folder) { 'Pooler/pool1' }
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    before(:each) do
-      allow(connection.searchIndex).to receive(:FindByInventoryPath)
-    end
-
-    it 'should call FindByInventoryPath with the correct parameters' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath)
-
-      subject.find_vm(poolname,vmname,connection)
-    end
-
-    it 'should return the VM object when found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(vm_object)
-
-      expect(subject.find_vm(poolname,vmname,connection)).to be(vm_object)
-    end
-
-    it 'should return nil if the VM is not found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-
-      expect(subject.find_vm(poolname,missing_vm,connection)).to be_nil
-    end
-  end
-
-  describe '#get_base_vm_container_from' do
-    it 'should return a recursive view of type VirtualMachine' do
-      result = subject.get_base_vm_container_from(connection)
-
-      expect(result.recursive).to be true
-      expect(result.type).to eq(['VirtualMachine'])
-    end
-  end
-
-  describe '#get_snapshot_list' do
-    let(:snapshot_name) {'snapshot'}
-    let(:snapshot_tree) { mock_RbVmomi_VIM_VirtualMachine(mock_options).snapshot.rootSnapshotList }
-    let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    it 'should raise if the snapshot tree is nil' do
-      expect{ subject.get_snapshot_list(nil,snapshot_name)}.to raise_error(NoMethodError)
-    end
-
-    context 'VM with a single layer of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => nil,
-          'snapshot4' => nil,
-          snapshot_name => { :ref => snapshot_object},
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.get_snapshot_list(snapshot_tree,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-    end
-
-    context 'VM with a nested layers of snapshots' do
-      let(:mock_options) {{
-        :snapshot_tree => {
-          'snapshot1' => nil,
-          'snapshot2' => nil,
-          'snapshot3'  => { :children => {
-            'snapshot4' => nil,
-            'snapshot5' => { :children => {
-              snapshot_name => { :ref => snapshot_object},
-            }},
-          }},
-          'snapshot6' => nil,
-        }
-      }}
-
-      it 'should return snapshot which matches the name' do
-        result = subject.get_snapshot_list(snapshot_tree,snapshot_name)
-        expect(result).to be(snapshot_object)
-      end
-    end
-  end
-
-  describe '#migrate_vm_host' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })}
-    let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST' })}
-    let(:relocate_task) { mock_RbVmomi_VIM_Task() }
-
-    before(:each) do
-      allow(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task)
-      allow(relocate_task).to receive(:wait_for_completion)
-    end
-
-    it 'should call RelocateVM_Task' do
-      expect(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task)
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should use a Relocation Spec object with correct host' do
-      expect(vm_object).to receive(:RelocateVM_Task).with(relocation_spec_with_host(host_object))
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should wait for the relocation to complete' do
-      expect(relocate_task).to receive(:wait_for_completion)
-
-      subject.migrate_vm_host(vm_object,host_object)
-    end
-
-    it 'should return the result of the relocation' do
-      expect(relocate_task).to receive(:wait_for_completion).and_return('RELOCATE_RESULT')
-
-      expect(subject.migrate_vm_host(vm_object,host_object)).to eq('RELOCATE_RESULT')
-    end
-  end
-
-  describe 'find_template_vm' do
-    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine() }
-
-    before(:each) do
-      allow(connection.searchIndex).to receive(:FindByInventoryPath)
-    end
-    it 'should raise an error when the datacenter cannot be found' do
-      config[:providers][:vsphere]['datacenter'] = nil
-
-      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error('cannot find datacenter')
-    end
-
-    it 'should raise an error when the template specified cannot be found' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
-
-      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error("Pool #{poolname} specifies a template VM of #{config[:pools][0]['template']} which does not exist for the provider vsphere")
-    end
-
-    it 'should return the vm object' do
-      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(vm_object)
-
-      subject.find_template_vm(config[:pools][0],connection)
-    end
-  end
-
-  describe 'valid_template_path?' do
-
-    it 'should return true with a valid template path' do
-      expect(subject.valid_template_path?('test/template')).to eq(true)
-    end
-
-    it 'should return false when no / is found' do
-      expect(subject.valid_template_path?('testtemplate')).to eq(false)
-    end
-
-    it 'should return false when template path begins with /' do
-      expect(subject.valid_template_path?('/testtemplate')).to eq(false)
-    end
-
-    it 'should return false when template path ends with /' do
-      expect(subject.valid_template_path?('testtemplate/')).to eq(false)
-    end
-  end
-
-  describe 'create_template_delta_disks' do
-    let(:template_object) { mock_RbVmomi_VIM_VirtualMachine({
-        :name => vmname,
-      })
-    }
-
-    before(:each) do
-      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
-    end
-
-    context 'with a template VM found' do
-
-      before(:each) do
-        expect(subject).to receive(:find_template_vm).and_return(template_object)
-      end
-
-      it 'should reconfigure the VM creating delta disks' do
-        expect(template_object).to receive(:add_delta_disk_layer_on_all_disks)
-
-        subject.create_template_delta_disks(config[:pools][0])
-      end
-    end
-  end
-
-  describe 'get_disk_backing' do
-
-    it 'should return moveChildMostDiskBacking when linked clone enabled' do
-      expect( subject.get_disk_backing({'create_linked_clone' => true}) ).to eq(:moveChildMostDiskBacking)
-    end
-
-    it 'should return moveAllDiskBackingsAndConsolidate when no preference is specified' do
-      expect( subject.get_disk_backing({})).to eq(:moveAllDiskBackingsAndConsolidate)
-    end
-
-    it 'should return moveAllDiskBackingsAndConsolidate when linked clone is false' do
-      expect( subject.get_disk_backing({create_linked_clone: false})).to eq(:moveAllDiskBackingsAndConsolidate)
-    end
-  end
-
-  describe 'linked_clone?' do
-    it 'should return true when linked clone is enabled on the pool' do
-      expect( subject.linked_clone?({'create_linked_clone' => true}) ).to be true
-    end
-
-    it 'should return nil when linked clone is not enabled on the pool' do
-      expect( subject.linked_clone?({}) ).to be nil
-    end
-  end
-end
diff --git a/spec/unit/providers_spec.rb b/spec/unit/providers_spec.rb
index 0ccc7f0..91045f2 100644
--- a/spec/unit/providers_spec.rb
+++ b/spec/unit/providers_spec.rb
@@ -12,12 +12,8 @@ describe 'providers' do
   end
 
   it '#load_all_providers' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(Vmpooler::Providers.load_all_providers).to match_array(p)
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(Vmpooler::Providers.load_all_providers.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
   it '#installed_providers' do
@@ -30,21 +26,18 @@ describe 'providers' do
   end
 
   it '#load_by_name' do
-    expect(Vmpooler::Providers.load_by_name('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(Vmpooler::Providers.load_by_name('dummy').join(', ')).to_not match(%r{,})
   end
 
-  it '#load only vpshere' do
-    expect(providers.load_from_gems('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+  it '#load only dummy' do
+    expect(providers.load_from_gems('dummy').join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
+    expect(providers.load_from_gems('dummy').join(', ')).to_not match(%r{,})
   end
 
   it '#load all providers from gems' do
-    p = [
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
-        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
-    ]
-    expect(providers.load_from_gems).to match_array(p)
-
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/base.rb})
+    expect(providers.load_from_gems.join(', ')).to match(%r{#{project_root_dir}/lib/vmpooler/providers/dummy.rb})
   end
 
 
diff --git a/spec/unit/queue_reliability_spec.rb b/spec/unit/queue_reliability_spec.rb
new file mode 100644
index 0000000..fe95548
--- /dev/null
+++ b/spec/unit/queue_reliability_spec.rb
@@ -0,0 +1,497 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'vmpooler/pool_manager'
+
+describe 'Vmpooler::PoolManager - Queue Reliability Features' do
+  let(:logger) { MockLogger.new }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { redis } }
+  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+  let(:config) { YAML.load(<<~EOT
+    ---
+    :config:
+      task_limit: 10
+      vm_checktime: 1
+      vm_lifetime: 12
+      prefix: 'pooler-'
+      dlq_enabled: true
+      dlq_ttl: 168
+      dlq_max_entries: 100
+      purge_enabled: true
+      purge_dry_run: false
+      max_pending_age: 7200
+      max_ready_age: 86400
+      max_completed_age: 3600
+      health_check_enabled: true
+      health_check_interval: 300
+      health_thresholds:
+        pending_queue_max: 100
+        ready_queue_max: 500
+        dlq_max_warning: 100
+        dlq_max_critical: 1000
+        stuck_vm_age_threshold: 7200
+    :providers:
+      :dummy: {}
+    :pools:
+      - name: 'test-pool'
+        size: 5
+        provider: 'dummy'
+    EOT
+    )
+  }
+
+  subject { Vmpooler::PoolManager.new(config, logger, redis_connection_pool, metrics) }
+
+  describe 'Dead-Letter Queue (DLQ)' do
+    let(:vm) { 'vm-abc123' }
+    let(:pool) { 'test-pool' }
+    let(:error_class) { 'StandardError' }
+    let(:error_message) { 'template does not exist' }
+    let(:request_id) { 'req-123' }
+    let(:pool_alias) { 'test-alias' }
+
+    before(:each) do
+      redis_connection_pool.with do |redis_connection|
+        allow(redis_connection).to receive(:zadd)
+        allow(redis_connection).to receive(:zcard).and_return(0)
+        allow(redis_connection).to receive(:expire)
+      end
+    end
+
+    describe '#dlq_enabled?' do
+      it 'returns true when dlq_enabled is true in config' do
+        expect(subject.dlq_enabled?).to be true
+      end
+
+      it 'returns false when dlq_enabled is false in config' do
+        config[:config]['dlq_enabled'] = false
+        expect(subject.dlq_enabled?).to be false
+      end
+    end
+
+    describe '#dlq_ttl' do
+      it 'returns configured TTL' do
+        expect(subject.dlq_ttl).to eq(168)
+      end
+
+      it 'returns default TTL when not configured' do
+        config[:config].delete('dlq_ttl')
+        expect(subject.dlq_ttl).to eq(168)
+      end
+    end
+
+    describe '#dlq_max_entries' do
+      it 'returns configured max entries' do
+        expect(subject.dlq_max_entries).to eq(100)
+      end
+
+      it 'returns default max entries when not configured' do
+        config[:config].delete('dlq_max_entries')
+        expect(subject.dlq_max_entries).to eq(10000)
+      end
+    end
+
+    describe '#move_to_dlq' do
+      context 'when DLQ is enabled' do
+        it 'adds entry to DLQ sorted set' do
+          redis_connection_pool.with do |redis_connection|
+            dlq_key = 'vmpooler__dlq__pending'
+            
+            expect(redis_connection).to receive(:zadd).with(dlq_key, anything, anything)
+            expect(redis_connection).to receive(:expire).with(dlq_key, anything)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, 
+                               redis_connection, request_id: request_id, pool_alias: pool_alias)
+          end
+        end
+
+        it 'includes error details in DLQ entry' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).to receive(:zadd) do |_key, _score, entry|
+              expect(entry).to include(vm)
+              expect(entry).to include(error_message)
+              expect(entry).to include(error_class)
+            end
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'increments DLQ metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with('vmpooler_dlq.pending.count')
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+
+        it 'enforces max entries limit' do
+          redis_connection_pool.with do |redis_connection|
+            allow(redis_connection).to receive(:zcard).and_return(150)
+            expect(redis_connection).to receive(:zremrangebyrank).with(anything, 0, 49)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+
+      context 'when DLQ is disabled' do
+        before { config[:config]['dlq_enabled'] = false }
+
+        it 'does not add entry to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(redis_connection).not_to receive(:zadd)
+            
+            subject.move_to_dlq(vm, pool, 'pending', error_class, error_message, redis_connection)
+          end
+        end
+      end
+    end
+  end
+
+  describe 'Auto-Purge' do
+    describe '#purge_enabled?' do
+      it 'returns true when purge_enabled is true in config' do
+        expect(subject.purge_enabled?).to be true
+      end
+
+      it 'returns false when purge_enabled is false in config' do
+        config[:config]['purge_enabled'] = false
+        expect(subject.purge_enabled?).to be false
+      end
+    end
+
+    describe '#purge_dry_run?' do
+      it 'returns false when purge_dry_run is false in config' do
+        expect(subject.purge_dry_run?).to be false
+      end
+
+      it 'returns true when purge_dry_run is true in config' do
+        config[:config]['purge_dry_run'] = true
+        expect(subject.purge_dry_run?).to be true
+      end
+    end
+
+    describe '#max_pending_age' do
+      it 'returns configured max age' do
+        expect(subject.max_pending_age).to eq(7200)
+      end
+
+      it 'returns default max age when not configured' do
+        config[:config].delete('max_pending_age')
+        expect(subject.max_pending_age).to eq(7200)
+      end
+    end
+
+    describe '#purge_pending_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old' }
+      let(:new_vm) { 'vm-new' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (3 hours old, exceeds 2 hour threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'clone', (Time.now - 10800).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__pending__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      context 'when not in dry-run mode' do
+        it 'purges stale pending VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be false
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", new_vm)).to be true
+          end
+        end
+
+        it 'moves purged VMs to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).to receive(:move_to_dlq).with(
+              old_vm, pool, 'pending', 'Purge', anything, redis_connection, anything
+            )
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+
+        it 'increments purge metrics' do
+          redis_connection_pool.with do |redis_connection|
+            expect(metrics).to receive(:increment).with("vmpooler_purge.pending.#{pool}.count")
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+
+      context 'when in dry-run mode' do
+        before { config[:config]['purge_dry_run'] = true }
+
+        it 'detects but does not purge stale VMs' do
+          redis_connection_pool.with do |redis_connection|
+            purged_count = subject.purge_pending_queue(pool, redis_connection)
+            
+            expect(purged_count).to eq(1)
+            expect(redis_connection.sismember("vmpooler__pending__#{pool}", old_vm)).to be true
+          end
+        end
+
+        it 'does not move to DLQ' do
+          redis_connection_pool.with do |redis_connection|
+            expect(subject).not_to receive(:move_to_dlq)
+            
+            subject.purge_pending_queue(pool, redis_connection)
+          end
+        end
+      end
+    end
+
+    describe '#purge_ready_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-ready' }
+      let(:new_vm) { 'vm-new-ready' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (25 hours old, exceeds 24 hour threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'ready', (Time.now - 90000).to_s)
+          
+          # New VM (2 hours old, within threshold)
+          redis_connection.sadd("vmpooler__ready__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'ready', (Time.now - 7200).to_s)
+        end
+      end
+
+      it 'moves stale ready VMs to completed queue' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_ready_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be true
+          expect(redis_connection.sismember("vmpooler__ready__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+
+    describe '#purge_completed_queue' do
+      let(:pool) { 'test-pool' }
+      let(:old_vm) { 'vm-old-completed' }
+      let(:new_vm) { 'vm-new-completed' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          # Old VM (2 hours old, exceeds 1 hour threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", old_vm)
+          redis_connection.hset("vmpooler__vm__#{old_vm}", 'destroy', (Time.now - 7200).to_s)
+          
+          # New VM (30 minutes old, within threshold)
+          redis_connection.sadd("vmpooler__completed__#{pool}", new_vm)
+          redis_connection.hset("vmpooler__vm__#{new_vm}", 'destroy', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'removes stale completed VMs' do
+        redis_connection_pool.with do |redis_connection|
+          purged_count = subject.purge_completed_queue(pool, redis_connection)
+          
+          expect(purged_count).to eq(1)
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", old_vm)).to be false
+          expect(redis_connection.sismember("vmpooler__completed__#{pool}", new_vm)).to be true
+        end
+      end
+    end
+  end
+
+  describe 'Health Checks' do
+    describe '#health_check_enabled?' do
+      it 'returns true when health_check_enabled is true in config' do
+        expect(subject.health_check_enabled?).to be true
+      end
+
+      it 'returns false when health_check_enabled is false in config' do
+        config[:config]['health_check_enabled'] = false
+        expect(subject.health_check_enabled?).to be false
+      end
+    end
+
+    describe '#health_thresholds' do
+      it 'returns configured thresholds' do
+        thresholds = subject.health_thresholds
+        expect(thresholds['pending_queue_max']).to eq(100)
+        expect(thresholds['stuck_vm_age_threshold']).to eq(7200)
+      end
+
+      it 'merges with defaults when partially configured' do
+        config[:config]['health_thresholds'] = { 'pending_queue_max' => 200 }
+        thresholds = subject.health_thresholds
+        
+        expect(thresholds['pending_queue_max']).to eq(200)
+        expect(thresholds['ready_queue_max']).to eq(500) # default
+      end
+    end
+
+    describe '#calculate_queue_ages' do
+      let(:pool) { 'test-pool' }
+      let(:vm1) { 'vm-1' }
+      let(:vm2) { 'vm-2' }
+      let(:vm3) { 'vm-3' }
+
+      before(:each) do
+        redis_connection_pool.with do |redis_connection|
+          redis_connection.hset("vmpooler__vm__#{vm1}", 'clone', (Time.now - 3600).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm2}", 'clone', (Time.now - 7200).to_s)
+          redis_connection.hset("vmpooler__vm__#{vm3}", 'clone', (Time.now - 1800).to_s)
+        end
+      end
+
+      it 'calculates ages for all VMs' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, vm2, vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(3)
+          expect(ages[0]).to be_within(5).of(3600)
+          expect(ages[1]).to be_within(5).of(7200)
+          expect(ages[2]).to be_within(5).of(1800)
+        end
+      end
+
+      it 'skips VMs with missing timestamps' do
+        redis_connection_pool.with do |redis_connection|
+          vms = [vm1, 'vm-nonexistent', vm3]
+          ages = subject.calculate_queue_ages(vms, 'clone', redis_connection)
+          
+          expect(ages.length).to eq(2)
+        end
+      end
+    end
+
+    describe '#determine_health_status' do
+      let(:base_metrics) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50 }
+            }
+          },
+          'errors' => {
+            'dlq_total_size' => 50,
+            'stuck_vm_count' => 2
+          }
+        }
+      end
+
+      it 'returns healthy when all metrics are within thresholds' do
+        status = subject.determine_health_status(base_metrics)
+        expect(status).to eq('healthy')
+      end
+
+      it 'returns degraded when DLQ size exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 150
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when DLQ size exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['dlq_total_size'] = 1500
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns degraded when pending queue exceeds warning threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 120
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('degraded')
+      end
+
+      it 'returns unhealthy when pending queue exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['queues']['test-pool']['pending']['size'] = 250
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+
+      it 'returns unhealthy when stuck VM count exceeds critical threshold' do
+        metrics = base_metrics.dup
+        metrics['errors']['stuck_vm_count'] = 60
+        
+        status = subject.determine_health_status(metrics)
+        expect(status).to eq('unhealthy')
+      end
+    end
+
+    describe '#push_health_metrics' do
+      let(:metrics_data) do
+        {
+          'queues' => {
+            'test-pool' => {
+              'pending' => { 'size' => 10, 'oldest_age' => 3600, 'stuck_count' => 2 },
+              'ready' => { 'size' => 50, 'oldest_age' => 7200 },
+              'completed' => { 'size' => 5 }
+            }
+          },
+          'tasks' => {
+            'clone' => { 'active' => 3 },
+            'ondemand' => { 'active' => 2, 'pending' => 5 }
+          },
+          'errors' => {
+            'dlq_total_size' => 25,
+            'stuck_vm_count' => 2,
+            'orphaned_metadata_count' => 3
+          }
+        }
+      end
+
+      it 'pushes status metric' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.status', 0)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes error metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.dlq.total_size', 25)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.stuck_vms.count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.orphaned_metadata.count', 3)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes per-pool queue metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.size', 10)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.oldest_age', 3600)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.pending.stuck_count', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.queue.test-pool.ready.size', 50)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+
+      it 'pushes task metrics' do
+        allow(metrics).to receive(:gauge)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.clone.active', 3)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.active', 2)
+        expect(metrics).to receive(:gauge).with('vmpooler_health.tasks.ondemand.pending', 5)
+        
+        subject.push_health_metrics(metrics_data, 'healthy')
+      end
+    end
+  end
+end
diff --git a/spec/unit/util/parsing_spec.rb b/spec/unit/util/parsing_spec.rb
new file mode 100644
index 0000000..c0277f9
--- /dev/null
+++ b/spec/unit/util/parsing_spec.rb
@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+# The only method previously tested here was '#get_domain_for_pool'
+# which was moved to Vmpooler::Dns as the more appropriate class
+#
+# TODO: Add tests for last remaining method, or move to more appropriate class
+describe 'Vmpooler::Parsing' do
+  let(:pool) { 'pool1' }
+  subject { Vmpooler::Parsing }
+
+end
diff --git a/spec/unit/vmpooler_spec.rb b/spec/unit/vmpooler_spec.rb
index ac9f620..8e6d39a 100644
--- a/spec/unit/vmpooler_spec.rb
+++ b/spec/unit/vmpooler_spec.rb
@@ -20,6 +20,21 @@ describe 'Vmpooler' do
           expect(Vmpooler.config[:pools]).to eq(default_config[:pools])
         end
       end
+
+      it 'keeps a copy of the original pools at startup' do
+        Dir.chdir(fixtures_dir) do
+          configuration = Vmpooler.config
+          expect(configuration[:pools]).to eq(configuration[:pools_at_startup])
+        end
+      end
+
+      it 'the copy is a separate object and not a reference' do
+        Dir.chdir(fixtures_dir) do
+          configuration = Vmpooler.config
+          configuration[:pools][0]['template'] = 'sam'
+          expect(configuration[:pools]).not_to eq(configuration[:pools_at_startup])
+        end
+      end
     end
 
     context 'when config variable is set' do
@@ -30,10 +45,55 @@ describe 'Vmpooler' do
     end
 
     context 'when config file is set' do
-      it 'should use the file' do
+      before(:each) do
         ENV['VMPOOLER_CONFIG_FILE'] = config_file
+      end
+      it 'should use the file' do
         expect(Vmpooler.config[:pools]).to eq(config[:pools])
       end
+      it 'merges one extra file, results in two providers' do
+        ENV['EXTRA_CONFIG'] = File.join(fixtures_dir, 'extra_config1.yaml')
+        expect(Vmpooler.config[:providers].keys).to include(:dummy)
+        expect(Vmpooler.config[:providers].keys).to include(:alice)
+      end
+      it 'merges two extra file, results in three providers and an extra pool' do
+        extra1 = File.join(fixtures_dir, 'extra_config1.yaml')
+        extra2 = File.join(fixtures_dir, 'extra_config2.yaml')
+        ENV['EXTRA_CONFIG'] = "#{extra1},#{extra2}"
+        expect(Vmpooler.config[:providers].keys).to include(:dummy)
+        expect(Vmpooler.config[:providers].keys).to include(:alice)
+        expect(Vmpooler.config[:providers].keys).to include(:bob)
+        merged_pools = [{"name"=>"pool03", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5},
+                        {"name"=>"pool04", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5},
+                        {"name"=>"pool05", "provider"=>"dummy", "dns_plugin"=>"example", "ready_ttl"=>5, "size"=>5}]
+        expect(Vmpooler.config[:pools]).to eq(merged_pools)
+        expect(Vmpooler.config[:config]).not_to be_nil #merge does not deleted existing keys
+      end
+    end
+
+    context 'when domain' do
+      context 'is set as a variable' do
+        before(:each) do
+          ENV['VMPOOLER_CONFIG_FILE'] = config_file
+          ENV['DOMAIN'] = 'example.com'
+        end
+
+        it 'should exit' do
+          expect { Vmpooler.config }.to raise_error(SystemExit)
+        end
+      end
+
+      context 'is set in a config file' do
+        let(:config_file) { File.join(fixtures_dir, 'vmpooler_domain.yaml') }
+
+        before(:each) do
+          ENV['VMPOOLER_CONFIG_FILE'] = config_file
+        end
+
+        it 'should exit' do
+          expect { Vmpooler.config }.to raise_error(SystemExit)
+        end
+      end
     end
   end
 end
diff --git a/update-gemfile-lock b/update-gemfile-lock
new file mode 100755
index 0000000..ec95ac1
--- /dev/null
+++ b/update-gemfile-lock
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment
+docker run -it --rm \
+  -v $(pwd):/app \
+  jruby:9.4.12.1-jdk11 \
+  /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase build-essential && cd /app && gem install bundler && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"'
diff --git a/utils/vmp_utils b/utils/vmp_utils
new file mode 100755
index 0000000..613abd7
--- /dev/null
+++ b/utils/vmp_utils
@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'redis'
+require 'thor'
+
+class VMPUtils < Thor
+  desc 'export_tokens', 'Export VMPooler tokens'
+  long_desc <<-LONGDESC
+    vmp_utils export_tokens will export the tokens stored in redis as JSON
+    to the specified file. By default it will export all tokens used within
+    the last year. This can be overridden by setting the "since-date" option.
+  LONGDESC
+  option :since_date, default: (DateTime.now - 365).iso8601
+  option :redis_host, required: true
+  option :redis_password
+  option :redis_port, type: :numeric, default: 6379
+  option :output_file, required: true
+  def export_tokens
+    redis = Redis.new(host: options[:redis_host], port: options[:redis_port], password: options[:redis_password])
+    begin
+      redis.ping
+      puts "connection successful to #{options[:redis_host]}"
+      tokens_to_export = {}
+      date_filter = DateTime.parse(options[:since_date])
+      redis.scan_each(match: 'vmpooler__token__*') do |key|
+        print('.')
+        key_data = redis.hgetall(key)
+        export = false
+        if key_data.key?('last') && date_filter < DateTime.parse(key_data['last'])
+          export = true
+        elsif date_filter < DateTime.parse(key_data['created'])
+          export = true
+        end
+        tokens_to_export[key] = key_data if export
+      end
+      puts ''
+      File.open(options[:output_file], 'w') do |output|
+        output.puts JSON.pretty_generate(tokens_to_export)
+      end
+    rescue Redis::CannotConnectError => e
+      raise Thor::Error, e
+    rescue Redis::CommandError => e
+      raise Thor::Error, e
+    end
+  end
+
+  desc 'import_tokens', 'Import previously exported VMPooler tokens'
+  long_desc <<-LONGDESC
+    vmp_utils import_tokens will load tokens from a previous export into the
+    specified redis instance. It expects a JSON file like the one generated by
+    export_tokens to be provided as input.
+  LONGDESC
+  option :redis_host, required: true
+  option :redis_password
+  option :redis_port, type: :numeric, default: 6379
+  option :input_file, required: true
+  def import_tokens
+    raise Thor::Error, "input-file '#{options[:input_file]}' doesn't exist" unless File.exist?(options[:input_file])
+
+    begin
+      data_file = File.read(options[:input_file])
+      tokens_to_import = JSON.parse(data_file)
+    rescue JSON::ParserError
+      raise Thor::Error, "Unable to parse please verify that #{options[:input_file]}. Please make sure it is a valid JSON file"
+    end
+
+    redis = Redis.new(host: options[:redis_host], port: options[:redis_port], password: options[:redis_password])
+    begin
+      redis.ping
+      puts "connection successful to #{options[:redis_host]}"
+      tokens_to_import.each do |key, value|
+        puts "importing token for #{value['user']}"
+        redis.hset(key, value)
+      end
+    rescue Redis::CannotConnectError => e
+      raise Thor::Error, e
+    rescue Redis::CommandError => e
+      raise Thor::Error, e
+    end
+  end
+end
+
+VMPUtils.start(ARGV)
diff --git a/vmpooler.gemspec b/vmpooler.gemspec
index f6fedeb..fe53085 100644
--- a/vmpooler.gemspec
+++ b/vmpooler.gemspec
@@ -11,41 +11,41 @@ Gem::Specification.new do |s|
   s.summary       = 'vmpooler provides configurable pools of instantly-available (running) virtual machines'
   s.homepage      = 'https://github.com/puppetlabs/vmpooler'
   s.license       = 'Apache-2.0'
-  s.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
   s.files         = Dir[ "bin/*", "lib/**/*" ]
   s.bindir        = 'bin'
   s.executables   = 'vmpooler'
   s.require_paths = ["lib"]
   s.add_dependency 'concurrent-ruby', '~> 1.1'
-  s.add_dependency 'connection_pool', '~> 2.2'
+  s.add_dependency 'connection_pool', '~> 2.4'
+  s.add_dependency 'deep_merge', '~> 1.2'
   s.add_dependency 'net-ldap', '~> 0.16'
-  s.add_dependency 'nokogiri', '~> 1.10'
-  s.add_dependency 'opentelemetry-api', '~> 0.7.0'
-  s.add_dependency 'opentelemetry-exporter-jaeger', '~> 0.7.0'
-  s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '~> 0.7.0'
-  s.add_dependency 'opentelemetry-instrumentation-redis', '~> 0.7.0'
-  s.add_dependency 'opentelemetry-instrumentation-sinatra', '~> 0.7.1'
-  s.add_dependency 'opentelemetry-resource_detectors', '~> 0.7.0'
-  s.add_dependency 'opentelemetry-sdk', '~> 0.7.0'
+  s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.23.0'
+  s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.21.1'
+  s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.22.2'
+  s.add_dependency 'opentelemetry-instrumentation-rack', '= 0.23.4'
+  s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.25.3'
+  s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.23.2'
+  s.add_dependency 'opentelemetry-resource_detectors', '= 0.24.2'
+  s.add_dependency 'opentelemetry-sdk', '~> 1.8'
   s.add_dependency 'pickup', '~> 0.0.11'
-  s.add_dependency 'prometheus-client', '~> 2.0'
-  s.add_dependency 'puma', '~> 4.3'
-  s.add_dependency 'rack', '~> 2.2'
+  s.add_dependency 'prometheus-client', '>= 2', '< 5'
+  s.add_dependency 'puma', '>= 5.0.4', '< 7'
+  s.add_dependency 'rack', '>= 2.2', '< 4.0'
   s.add_dependency 'rake', '~> 13.0'
-  s.add_dependency 'rbvmomi', '>= 2.1', '< 4.0'
-  s.add_dependency 'redis', '~> 4.1'
-  s.add_dependency 'sinatra', '~> 2.0'
+  s.add_dependency 'redis', '~> 5.0'
+  s.add_dependency 'sinatra', '>= 2', '< 4'
   s.add_dependency 'spicy-proton', '~> 2.1'
   s.add_dependency 'statsd-ruby', '~> 1.4'
 
   # Testing dependencies
   s.add_development_dependency 'climate_control', '>= 0.2.0'
-  s.add_development_dependency 'mock_redis', '>= 0.17.0'
+  s.add_development_dependency 'mock_redis', '= 0.37.0'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rack-test', '>= 0.6'
   s.add_development_dependency 'rspec', '>= 3.2'
-  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'rubocop', '~> 1.56.0'
   s.add_development_dependency 'simplecov', '>= 0.11.2'
+  s.add_development_dependency 'thor', '~> 1.0', '>= 1.0.1'
   s.add_development_dependency 'yarjuf', '>= 2.0'
-end
+end
\ No newline at end of file
diff --git a/vmpooler.yaml.dummy-example b/vmpooler.yaml.dummy-example
index 3a15630..e27efd6 100644
--- a/vmpooler.yaml.dummy-example
+++ b/vmpooler.yaml.dummy-example
@@ -3,6 +3,12 @@
   :dummy:
     filename: '/tmp/dummy-backing.yaml'
 
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    #domain: 'localhost' # Flip these out for local requests
+    domain: 'example.com'
+
 :redis:
   server: 'localhost'
 
@@ -17,14 +23,13 @@
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
-  #domain: 'localhost' # Flip these out for local requests
   prefix: 'poolvm-'
 
 :pools:
@@ -35,8 +40,10 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
+    dns_plugin: 'example'
   - name: 'debian-7-x86_64'
     alias: [ 'debian-7-64', 'debian-7-amd64' ]
     template: 'Templates/debian-7-x86_64'
@@ -44,6 +51,7 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: dummy
-
+    dns_plugin: 'example'
diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example
index 6194167..f05ded2 100644
--- a/vmpooler.yaml.example
+++ b/vmpooler.yaml.example
@@ -11,22 +11,29 @@
 #     For multiple providers, specify one of the supported backing services (vsphere or dummy)
 #     (optional: will default to it's parent :key: name eg. 'vsphere')
 #
-#   - purge_unconfigured_folders
-#     Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-#     Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-#     Nested child folders will not be destroyed. An optional whitelist can be provided to exclude folders
-#     A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
-#     Setting this on the provider will enable folder purging for the provider
+#   - purge_unconfigured_folders DEPRECATED, use purge_unconfigured_resources
+#   - purge_unconfigured_resources
+#     Enable purging of resources (typically VMs) or other items based on the provider. Provides a high-level cleanup
+#     mechanism for things that are live but not found in the vmpooler config ie in a pool config. See the provider's
+#     implementation for more details.
+#     Setting this on the provider will enable purging for the provider
 #     Expects a boolean value
 #     (optional; default: false)
 #
-#   - folder_whitelist
-#     Specify folders that are within the base folder path, not in the configuration, and should not be destroyed
-#     To exclude 'vmpooler/myfolder' from being destroyed when the base path is 'vmpooler' you would specify 'myfolder' in the whitelist
-#     This option is only evaluated when 'purge_unconfigured_folders' is enabled
-#     Expects an array of strings specifying the whitelisted folders by name
+#   - folder_whitelist DEPRECATED, use resources_allowlist
+#   - resources_allowlist
+#     Specify items names that should be ignored when purging. See the provider's
+#     implementation for more details.
+#     This option is only evaluated when 'purge_unconfigured_resources' is enabled
+#     Expects an array of strings specifying the allowed items by name
 #     (optional; default: nil)
 #
+#   - skip_dns_check_before_creating_vm
+#     Setting this configuration parameter in a provider will make vmpooler skip the check it normally does for a DNS
+#     record. The normal behavior is to check DNS and if a record already exists (conflict) to re-generate a new hostname.
+#     By using this configuration parameter, it will skip the check and continue with the same conflicting name. This is
+#     useful for providers that can handle that case, for instance by replacing the existing DNS record with a new one.
+#
 # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
 # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
 # the provider to use.
@@ -360,6 +367,15 @@
 #   - user_object
 #     The LDAP object-type used to designate a user object.
 #
+#   - service_account_hash
+#     A hash containing the following parameters for a service account to perform the
+#     initial bind. After the initial bind, then a search query is performed using the
+#     'base' and 'user_object', then re-binds as the returned user.
+#     - :user_dn
+#     The full distinguished name (DN) of the service account used to bind.
+#     - :password
+#     The password for the service account used to bind.
+#
 # Example:
 # :auth:
 #   provider: 'ldap'
@@ -368,12 +384,33 @@
 #     port: 389
 #     base: 'ou=users,dc=company,dc=com'
 #     user_object: 'uid'
+#
+# :auth:
+#   provider: 'ldap'
+#   :ldap:
+#     host: 'ldap.example.com'
+#     port: 636
+#     service_account_hash:
+#       :user_dn: 'cn=Service Account,ou=Accounts,dc=ldap,dc=example,dc=com'
+#       :password: 'service-account-password'
+#     encryption:
+#       :method: :simple_tls
+#       :tls_options:
+#         :ssl_version: 'TLSv1_2'
+#     base:
+#       - 'ou=Accounts,dc=company,dc=com'
+#     user_object:
+#       - 'samAccountName'
 
 :auth:
   provider: 'ldap'
   :ldap:
     host: 'ldap.example.com'
-    port: 389
+    port: 636
+    encryption:
+      :method: :simple_tls
+      :tls_options:
+        :ssl_version: 'TLSv1_2'
     base: 'ou=users,dc=company,dc=com'
     user_object: 'uid'
 
@@ -419,6 +456,12 @@
 #     How long (in minutes) before marking a clone in 'pending' queues as 'failed' and retrying.
 #     (default: 15)
 #
+#   - max_vm_retries
+#     Maximum number of times to retry VM creation for a failed request before marking it as permanently failed.
+#     This helps prevent infinite retry loops when there are configuration issues like invalid template paths.
+#     Permanent errors (like invalid template paths) are detected and will not be retried.
+#     (default: 3)
+#
 #   - vm_checktime
 #     How often (in minutes) to check the sanity of VMs in 'ready' queues.
 #     (default: 1)
@@ -434,9 +477,6 @@
 #   - allowed_tags
 #     If set, restricts tags to those specified in this array.
 #
-#   - domain
-#     If set, returns a top-level 'domain' JSON key in POST requests
-#
 #   - prefix
 #     If set, prefixes all created VMs with this string.  This should include
 #     a separator.
@@ -514,10 +554,11 @@
 #     Expects a boolean value
 #     (optional; default: false)
 #
-#   - purge_unconfigured_folders (vSphere Provider only)
-#     Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
-#     Only a single layer of folders and their child VMs are evaluated from detected base folder paths
-#     A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+#   - purge_unconfigured_folders DEPRECATED, use purge_unconfigured_resources
+#   - purge_unconfigured_resources
+#     Enable purging of resources (typically VMs) or other items based on the provider. Provides a high-level cleanup
+#     mechanism for things that are live but not found in the vmpooler config ie in a pool config. See the provider's
+#     implementation for more details.
 #     When enabled in the global configuration then purging is enabled for all providers
 #     Expects a boolean value
 #     (optional; default: false)
@@ -532,13 +573,13 @@
 #
 #   - usage_stats
 #     Enable shipping of VM usage stats
-#     When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize
+#     When enabled a metric is emitted when a user requested to allocate and destroy a VM. Tags are inspected and used to organize
 #     shipped metrics if there is a jenkins_build_url tag set for the VM.
-#     Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$pool_name".
+#     Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$operation.$pool_name".
 #     When the jenkins_build_url tag is set the metric will be sent with additional data. Here is an example
 #     based off of the following URL, and requested by the user ABS;
 #     https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/
-#     "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$pool_name", which translates to
+#     "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$operation.$pool_name", which translates to
 #     "usage.$user.jenkins_example_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool_name"
 #     Expects a boolean value
 #     (optional; default: false)
@@ -575,23 +616,48 @@
 #
 # Example:
 
-:config: site_name: 'vmpooler'
+:config:
+  site_name: 'vmpooler'
   logfile: '/var/log/vmpooler.log'
   task_limit: 10
   timeout: 15
+  timeout_notification: 5
   vm_checktime: 1
   vm_lifetime: 12
   vm_lifetime_auth: 24
+  max_vm_retries: 3
   allowed_tags:
     - 'created_by'
     - 'project'
-  domain: 'example.com'
   prefix: 'poolvm-'
   experimental_features: true
   backend_weight:
     'backend1': 60
     'backend2': 40
 
+# :dns_configs:
+#
+# This section a list of dns configurations to be referenced by one or more pools.
+#
+# The currently supported backing services are:
+# - dynamic-dns (This assumes that dynamic dns is handling record management and VMPooler does not require interaction)
+# - gcp (Google Cloud DNS https://github.com/puppetlabs/vmpooler-dns-gcp)
+#
+#   - dns_class
+#     Specify one of the supported backing services.
+#
+#   - domain
+#     The domain expected to make up the FQDN when attempting to resolve VM instances.
+#
+# See DNS plugin docs for additional options specific to that class.
+#
+# Example
+
+:dns_configs:
+  :example:
+    dns_class: dynamic-dns
+    domain: 'example.com'
+
 # :pools:
 #
 # This section contains a list of virtual machine 'pools' for vmpooler to
@@ -622,6 +688,12 @@
 #     If you have more than one provider, this is where you would choose which
 #     one to use for this pool
 #
+#   - dns_plugin
+#     The name of the DNS plugin to use with this pool in order to determine the
+#     domain and settings specific to a DNS service. This should match
+#     a name in the :dns_configs: section above. e.g. example
+#     (required)
+#
 #   - clone_target
 #     Per-pool option to override the global 'clone_target' cluster.
 #     (optional)
@@ -654,6 +726,15 @@
 #       The datacenter within vCenter to manage VMs.
 #       (optional: default is the first datacenter in vSphere)
 #
+#     - snapshot_mainMem_ioBlockPages
+#       Provisions VMs with the option "mainMem.ioBlockPages". This setting can be useful
+#       (paired with mainMem.iowait below) for tuning the performance of snapshot management actions.
+#       See: https://kb.vmware.com/s/article/76687
+#
+#     - snapshot_mainMem_iowait
+#       Provisions VMs with the option "mainMem.iowait". This setting can be useful
+#       for tuning the performance of snapshot management actions
+#
 # Example:
 
 :pools:
@@ -664,6 +745,7 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: vsphere
     create_linked_clone: true
@@ -674,6 +756,9 @@
     datastore: 'vmstorage'
     size: 5
     timeout: 15
+    timeout_notification: 5
     ready_ttl: 1440
     provider: vsphere
     create_linked_clone: false
+    snapshot_mainMem_ioBlockPages: '2048'
+    snapshot_mainMem_iowait: '0'
diff --git a/vmpooler.yml.example b/vmpooler.yml.example
new file mode 100644
index 0000000..31060c2
--- /dev/null
+++ b/vmpooler.yml.example
@@ -0,0 +1,92 @@
+---
+# VMPooler Configuration Example with Dead-Letter Queue, Auto-Purge, and Health Checks
+
+# Redis Configuration
+:redis:
+  server: 'localhost'
+  port: 6379
+  data_ttl: 168  # hours - how long to keep VM metadata in Redis
+  
+  # Dead-Letter Queue (DLQ) Configuration
+  dlq_enabled: true
+  dlq_ttl: 168  # hours (7 days) - how long to keep DLQ entries
+  dlq_max_entries: 10000  # maximum entries per DLQ queue before trimming
+
+# Application Configuration
+:config:
+  # ... other existing config ...
+  
+  # Dead-Letter Queue (DLQ) - Optional, defaults shown
+  dlq_enabled: false  # Set to true to enable DLQ
+  dlq_ttl: 168  # hours (7 days)
+  dlq_max_entries: 10000  # per DLQ queue
+  
+  # Auto-Purge Stale Queue Entries
+  purge_enabled: false  # Set to true to enable auto-purge
+  purge_interval: 3600  # seconds (1 hour) - how often to run purge cycle
+  purge_dry_run: false  # Set to true to log what would be purged without actually purging
+  
+  # Auto-Purge Age Thresholds (in seconds)
+  max_pending_age: 7200  # 2 hours - VMs stuck in pending
+  max_ready_age: 86400  # 24 hours - VMs idle in ready queue
+  max_completed_age: 3600  # 1 hour - VMs in completed queue
+  max_orphaned_age: 86400  # 24 hours - orphaned VM metadata
+  max_request_age: 86400  # 24 hours - stale on-demand requests
+  
+  # Health Checks
+  health_check_enabled: false  # Set to true to enable health checks
+  health_check_interval: 300  # seconds (5 minutes) - how often to run health checks
+  
+  # Health Check Thresholds
+  health_thresholds:
+    pending_queue_max: 100  # Warning threshold for pending queue size
+    ready_queue_max: 500  # Warning threshold for ready queue size
+    dlq_max_warning: 100  # Warning threshold for DLQ size
+    dlq_max_critical: 1000  # Critical threshold for DLQ size
+    stuck_vm_age_threshold: 7200  # 2 hours - age at which VM is considered "stuck"
+    stuck_vm_max_warning: 10  # Warning threshold for stuck VM count
+    stuck_vm_max_critical: 50  # Critical threshold for stuck VM count
+
+# Pool Configuration
+:pools:
+  - name: 'centos-7-x86_64'
+    size: 5
+    provider: 'vsphere'
+    # ... other pool settings ...
+
+# Provider Configuration
+:providers:
+  :vsphere:
+    server: 'vcenter.example.com'
+    username: 'vmpooler'
+    password: 'secret'
+    # ... other provider settings ...
+
+# Example: Production Configuration
+# For production use, you might want:
+# :config:
+#   dlq_enabled: true
+#   dlq_ttl: 168  # Keep failed VMs for a week
+#   
+#   purge_enabled: true
+#   purge_interval: 1800  # Run every 30 minutes
+#   purge_dry_run: false
+#   max_pending_age: 3600  # Purge pending VMs after 1 hour
+#   max_ready_age: 172800  # Purge ready VMs after 2 days
+#   
+#   health_check_enabled: true
+#   health_check_interval: 300  # Check every 5 minutes
+
+# Example: Development Configuration
+# For development/testing, you might want:
+# :config:
+#   dlq_enabled: true
+#   dlq_ttl: 24  # Keep failed VMs for a day
+#   
+#   purge_enabled: true
+#   purge_interval: 600  # Run every 10 minutes
+#   purge_dry_run: true  # Test mode - log but don't actually purge
+#   max_pending_age: 1800  # More aggressive - 30 minutes
+#   
+#   health_check_enabled: true
+#   health_check_interval: 60  # Check every minute