Merge branch 'main' into dependabot/bundler/opentelemetry-instrumentation-redis-0.21.3

.rubocop.yml

@@ -66,6 +66,7 @@ Style/Next:
   Enabled: false
 Metrics/ParameterLists:
   Max: 10
+  MaxOptionalParameters: 10
 Style/GuardClause:
   Enabled: false
 

.rubocop_todo.yml

@@ -13,6 +13,7 @@
 Layout/ParameterAlignment:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 9
 # Cop supports --auto-correct.
@@ -22,12 +23,14 @@ Layout/CaseIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
 Layout/ClosingParenthesisIndentation:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -59,6 +62,7 @@ Layout/FirstHashElementIndentation:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -66,6 +70,7 @@ Layout/FirstHashElementIndentation:
 Layout/IndentationWidth:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -74,6 +79,7 @@ Layout/IndentationWidth:
 Layout/MultilineMethodCallBraceLayout:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -88,6 +94,7 @@ Layout/SpaceAroundEqualsInParameterDefault:
 Layout/SpaceAroundKeyword:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -95,6 +102,7 @@ Layout/SpaceAroundKeyword:
 Layout/SpaceAroundOperators:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 8
 # Cop supports --auto-correct.
@@ -110,6 +118,7 @@ Layout/SpaceInsideHashLiteralBraces:
 Layout/SpaceInsideParens:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 2
 # Configuration parameters: AllowSafeAssignment.
@@ -117,6 +126,7 @@ Lint/AssignmentInCondition:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 2
 Lint/SuppressedException:
@@ -149,6 +159,7 @@ Style/AndOr:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 Style/CaseEquality:
@@ -170,6 +181,7 @@ Style/HashSyntax:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
@@ -178,6 +190,7 @@ Style/IfUnlessModifier:
   Exclude:
     - 'lib/vmpooler/api/helpers.rb'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
@@ -186,12 +199,14 @@ Style/IfUnlessModifier:
 Style/NegatedIf:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 3
 # Cop supports --auto-correct.
 Style/Not:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -201,12 +216,14 @@ Style/NumericPredicate:
   Exclude:
     - 'spec/**/*'
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/ParallelAssignment:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -214,12 +231,14 @@ Style/ParallelAssignment:
 Style/ParenthesesAroundCondition:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/PerlBackrefs:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist.
@@ -236,6 +255,7 @@ Naming/PredicateName:
 Style/RedundantParentheses:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -257,6 +277,7 @@ Style/RedundantSelf:
 Style/StringLiterals:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -272,6 +293,7 @@ Style/TernaryParentheses:
 Naming/VariableName:
   Exclude:
     - 'lib/vmpooler/api/v1.rb'
+    - 'lib/vmpooler/api/v2.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.

Gemfile.lock

@@ -9,8 +9,8 @@ PATH
       nokogiri (~> 1.10)
       opentelemetry-exporter-jaeger (= 0.20.1)
       opentelemetry-instrumentation-concurrent_ruby (= 0.19.2)
-      opentelemetry-instrumentation-http_client (= 0.19.3)
       opentelemetry-instrumentation-redis (= 0.21.3)
+      opentelemetry-instrumentation-http_client (= 0.19.4)
       opentelemetry-instrumentation-sinatra (= 0.19.3)
       opentelemetry-resource_detectors (= 0.19.1)
       opentelemetry-sdk (~> 1.0, >= 1.0.2)
@@ -64,7 +64,7 @@ GEM
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
     method_source (1.0.0)
-    mock_redis (0.30.0)
+    mock_redis (0.31.0)
       ruby2_keywords
     multipart-post (2.1.1)
     mustermann (1.1.1)
@@ -89,7 +89,7 @@ GEM
     opentelemetry-instrumentation-concurrent_ruby (0.19.2)
       opentelemetry-api (~> 1.0)
       opentelemetry-instrumentation-base (~> 0.19.0)
-    opentelemetry-instrumentation-http_client (0.19.3)
+    opentelemetry-instrumentation-http_client (0.19.4)
       opentelemetry-api (~> 1.0)
       opentelemetry-common (~> 0.19.3)
       opentelemetry-instrumentation-base (~> 0.19.0)
@@ -112,7 +112,7 @@ GEM
     opentelemetry-semantic_conventions (1.8.0)
       opentelemetry-api (~> 1.0)
     parallel (1.22.1)
-    parser (3.1.1.0)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
     pickup (0.0.11)
     prometheus-client (2.1.0)
@@ -123,9 +123,9 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
       spoon (~> 0.0)
-    puma (5.6.2)
+    puma (5.6.4)
       nio4r (~> 2.0)
-    puma (5.6.2-java)
+    puma (5.6.4-java)
       nio4r (~> 2.0)
     racc (1.6.0)
     racc (1.6.0-java)
@@ -137,7 +137,7 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     redis (4.6.0)
-    regexp_parser (2.2.1)
+    regexp_parser (2.3.1)
     rexml (3.2.5)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
@@ -152,16 +152,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
-    rubocop (1.1.0)
+    rubocop (1.28.2)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.0.1)
+      rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.16.0)
+    rubocop-ast (1.17.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
@@ -184,7 +184,7 @@ GEM
     thor (1.2.1)
     thrift (0.16.0)
     tilt (2.0.10)
-    unicode-display_width (1.8.0)
+    unicode-display_width (2.1.0)
     yarjuf (2.0.0)
       builder
       rspec (~> 3)
@@ -199,7 +199,7 @@ DEPENDENCIES
   pry
   rack-test (>= 0.6)
   rspec (>= 3.2)
-  rubocop (~> 1.1.0)
+  rubocop (~> 1.28.2)
   simplecov (>= 0.11.2)
   thor (~> 1.0, >= 1.0.1)
   vmpooler!

docs/API-v2.md

@@ -0,0 +1,945 @@
+# Table of contents
+1. [API](#API)
+2. [Token operations](#token)
+3. [VM operations](#vmops)
+4. [Add disks](#adddisks)
+5. [VM snapshots](#vmsnapshots)
+6. [Status and metrics](#statusmetrics)
+7. [Pool configuration](#poolconfig)
+8. [Ondemand VM provisioning](#ondemandvm)
+
+### API <a name="API"></a>
+
+vmpooler provides a REST API for VM management.  The following examples use `curl` for communication.
+
+## Major change in V2 versus V1
+
+The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider,
+the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each provider instead of only one.
+
+Clients using some of the direct API paths (without specifying api/v1 or api/v2) will still be redirected to v1, but this behavior is notw deprecated and will be changed to v2 in the next major version.
+
+### updavig clients from v1 to v2
+
+Clients need to update their paths to using api/v2 instead of api/v1. Please note the API responses that used to return a "domain" key, will no longer have a separate "domain" key and now return
+the FQDN (includes the domain in the hostname).
+
+One way to support both v1 and v2 in the client logic is to look for a "domain" and append it to the hostname if it exists (existing v1 behavior). If the "domain" key does not exist, you can use the hostname 
+as is since it is a FQDN (v2 behavor).
+
+#### Token operations <a name="token"></a>
+
+Token-based authentication can be used when requesting or modifying VMs.  The `/token` route can be used to create, query, or delete tokens.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations.
+
+##### GET /token
+
+Get a list of issued tokens.
+
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found or other error
+
+```
+$ curl -u jdoe --url vmpooler.example.com/api/v2/token
+Enter host password for user 'jdoe':
+```
+```json
+{
+    "ok": true,
+    "utpg2i2xswor6h8ttjhu3d47z53yy47y": {
+      "created": "2015-04-28 19:17:47 -0700"
+    }
+}
+```
+
+##### POST /token
+
+Generate a new authentication token.
+
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found
+
+```
+$ curl -X POST -u jdoe --url vmpooler.example.com/api/v2/token
+Enter host password for user 'jdoe':
+```
+```json
+{
+    "ok": true,
+    "token": "utpg2i2xswor6h8ttjhu3d47z53yy47y"
+}
+```
+
+##### GET /token/&lt;token&gt;
+
+Get information about an existing token (including associated VMs).
+
+Return codes:
+* 200  OK
+* 404  when config:auth or token not found
+
+```
+$ curl --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+```
+```json
+{
+  "ok": true,
+  "utpg2i2xswor6h8ttjhu3d47z53yy47y": {
+    "user": "jdoe",
+    "created": "2015-04-28 19:17:47 -0700",
+    "last": "2015-11-04 12:28:37 -0700",
+    "vms": {
+      "running": [
+        "dqs4914g2wjyy5w",
+        "hul7ib0ssr0f4o0"
+      ]
+    }
+  }
+}
+```
+
+##### DELETE /token/&lt;token&gt;
+
+Delete an authentication token.
+
+Return codes:
+* 200  OK
+* 401  when not authorized
+* 404  when config:auth not found
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y
+Enter host password for user 'jdoe':
+```
+```json
+{
+  "ok": true
+}
+```
+
+#### VM operations <a name="vmops"></a>
+
+##### GET /vm
+
+Retrieve a list of available VM pools.
+
+Return codes:
+* 200  OK
+
+```
+$ curl --url vmpooler.example.com/api/v2/vm
+```
+```json
+[
+  "debian-7-i386",
+  "debian-7-x86_64"
+]
+```
+
+##### POST /vm
+
+Useful for batch operations; post JSON (see format below), get back allocated VMs.
+
+If an authentication store is configured, an authentication token supplied via the `X-AUTH-TOKEN` HTTP header will modify a VM's default lifetime.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information.
+
+Return codes:
+* 200  OK
+* 404  when sending invalid JSON in the request body or requesting an invalid VM pool name
+* 503  when the vm failed to allocate a vm, or the pool is empty
+
+```
+$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v2/vm
+```
+```json
+{
+  "ok": true,
+  "debian-7-i386": {
+    "hostname": [
+      "o41xtodlvnvu5cw.example.com",
+      "khirruvwfjlmx3y.example.com"
+    ]
+  },
+  "debian-7-x86_64": {
+    "hostname": "y91qbrpbfj6d13q.example.com"
+  }
+}
+```
+
+**NOTE: Returns either all requested VMs or no VMs.**
+
+##### POST /vm/&lt;pool&gt;
+
+Check-out a VM or VMs.
+
+Return codes:
+* 200  OK
+* 404  when sending invalid JSON in the request body or requesting an invalid VM pool name
+* 503  when the vm failed to allocate a vm, or the pool is empty
+
+```
+$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386
+```
+```json
+{
+  "ok": true,
+  "debian-7-i386": {
+    "hostname": "fq6qlpjlsskycq6.example.com"
+  }
+}
+```
+
+Multiple VMs can be requested by using multiple query parameters in the URL:
+
+```
+$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
+```
+
+```json
+{
+  "ok": true,
+  "debian-7-i386": {
+    "hostname": [
+      "sc0o4xqtodlul5w.example.com",
+      "4m4dkhqiufnjmxy.example.com"
+    ]
+  },
+  "debian-7-x86_64": {
+    "hostname": "zb91y9qbrbf6d3q.example.com"
+  }
+}
+```
+
+**NOTE: Returns either all requested VMs or no VMs.**
+
+##### GET /vm/&lt;hostname&gt;
+
+Query metadata information for a checked-out VM.
+
+Return codes:
+* 200  OK
+* 404  when requesting an invalid VM hostname
+
+```
+$ curl --url vmpooler.example.com/api/v2/vm/pxpmtoonx7fiqg6
+```
+```json
+{
+  "ok": true,
+  "pxpmtoonx7fiqg6": {
+    "template": "centos-6-x86_64",
+    "lifetime": 12,
+    "running": 3,
+    "remaining": 9, 
+    "state": "running",
+    "tags": {
+      "department": "engineering",
+      "user": "jdoe"
+    },
+    "ip": "192.168.0.1",
+    "host": "host1.example.com",
+    "migrated": "true"
+  }
+}
+```
+
+##### PUT /vm/&lt;hostname&gt;
+
+Modify a checked-out VM.
+
+The following are valid PUT parameters and their required data structures:
+
+parameter | description | required structure
+--------- | ----------- | ------------------
+*lifetime* | VM TTL (in hours) | integer
+*tags* | free-form VM tagging | hash
+
+Any modifications can be verified using the [GET /vm/&lt;hostname&gt;](#get-vmhostname) endpoint.
+
+If an authentication store is configured, an authentication token is required (via the `X-AUTH-TOKEN` HTTP header) to access this route.  See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information.
+
+Return codes:
+* 200  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+* 400  when supplied PUT parameters fail validation
+
+```
+$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6
+```
+```json
+{
+  "ok": true
+}
+```
+
+```
+$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### DELETE /vm/&lt;hostname&gt;
+
+Schedule a checked-out VM for deletion.
+
+Return codes:
+* 200  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+
+```
+$ curl -X DELETE --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6
+```
+```json
+{
+  "ok": true
+}
+```
+
+#### Adding additional disk(s) <a name="adddisks"></a>
+
+##### POST /vm/&lt;hostname&gt;/disk/&lt;size&gt;
+
+Add an additional disk to a running VM.
+
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname or size is not an integer
+
+````
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/disk/8
+````
+````json
+{
+  "ok": true,
+  "fq6qlpjlsskycq6": {
+    "disk": "+8gb"
+  }
+}
+````
+
+Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/<hostname>` query:
+
+````
+$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6
+````
+````json
+{
+  "ok": true,
+  "fq6qlpjlsskycq6": {
+    "template": "debian-7-x86_64",
+    "lifetime": 2,
+    "running": 0.08,
+    "state": "running",
+    "disk": [
+      "+8gb"
+    ]
+  }
+}
+
+````
+
+#### VM snapshots <a name="vmsnapshots"></a>
+
+##### POST /vm/&lt;hostname&gt;/snapshot
+
+Create a snapshot of a running VM.
+
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname
+
+````
+$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot
+````
+````json
+{
+  "ok": true,
+  "fq6qlpjlsskycq6": {
+    "snapshot": "n4eb4kdtp7rwv4x158366vd9jhac8btq"
+  }
+}
+````
+
+Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/<hostname>` query:
+
+````
+$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6
+````
+````json
+{
+  "ok": true,
+  "fq6qlpjlsskycq6": {
+    "template": "debian-7-x86_64",
+    "lifetime": 2,
+    "running": 0.08,
+    "state": "running",
+    "snapshots": [
+      "n4eb4kdtp7rwv4x158366vd9jhac8btq"
+    ]
+  }
+}
+````
+
+##### POST /vm/&lt;hostname&gt;/snapshot/&lt;snapshot&gt;
+
+Revert a VM back to a snapshot.
+
+Return codes:
+* 202  OK
+* 401  when you need an auth token
+* 404  when requesting an invalid VM hostname or snapshot is not valid
+
+````
+$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq
+````
+````json
+{
+  "ok": true
+}
+````
+
+#### Status and metrics <a name="statusmetrics"></a>
+
+##### GET /status
+
+A "live" status endpoint, representing the current state of the service.
+
+```
+$ curl --url vmpooler.example.com/api/v2/status
+```
+```json
+{
+  "capacity": {
+    "current": 716,
+    "total": 717,
+    "percent": 99.9
+  },
+  "clone": {
+    "duration": {
+      "average": 8.8,
+      "min": 2.79,
+      "max": 69.76
+    },
+    "count": {
+      "total": 1779
+    }
+  },
+  "queue": {
+    "pending": 1,
+    "cloning": 0,
+    "booting": 1,
+    "ready": 716,
+    "running": 142,
+    "completed": 0,
+    "total": 859
+  },
+  "status": {
+    "ok": true,
+    "message": "Battle station fully armed and operational."
+  }
+}
+```
+
+If there are empty pools, the "status" section will convey this:
+
+```json
+  "status": {
+    "ok": false,
+    "message": "Found 2 empty pools.",
+    "empty": [
+      "centos-6-x86_64",
+      "debian-7-x86_64"
+    ]
+  }
+```
+
+The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "status".
+If the query parameter 'view' is provided, it will be used to select which top level
+element to compute and return. Select them by specifying which one you want in a comma 
+separated list.
+For example `vmpooler.example.com/api/v2/status?view=capacity,boot`
+
+##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]]
+
+Returns a summary, or report, for the timespan between `from` and `to` (inclusive)
+parameters. The response includes both an overall and daily view of tracked
+metrics, such as boot and cloning durations.
+
+Any omitted query parameter will default to now/today. A request without any
+parameters will result in the current day's summary.
+
+Return codes:
+* 200  OK
+* 400  Invalid date format or range
+
+
+```
+$ curl --url vmpooler.example.com/api/v2/summary
+```
+```json
+{
+  "boot": {
+    "duration": {
+      "average": 106.6,
+      "min": 83.09,
+      "max": 121.06,
+      "total": 639.36,
+    },
+    "count": {
+      "average": 6,
+      "min": 6,
+      "max": 6,
+      "total": 6,
+    }
+  },
+  "clone": {
+    "duration": {
+      "average": 4.6,
+      "min": 2.78,
+      "max": 8.1,
+      "total": 63.94,
+    },
+    "count": {
+      "average": 14,
+      "min": 14,
+      "max": 14,
+      "total": 14,
+    }
+  },
+  "tag": {
+    "department": {
+      "engineering": 14,
+      "help desk": 10,
+      "IT": 44,
+      "total": 68
+    },
+    "user": {
+      "arodgers": 54,
+      "cmatthews": 10,
+      "jnelson": 4,
+      "total": 68
+    }
+  },
+  "daily": [
+    {
+      "date": "2015-03-11",
+      "boot": {
+        "duration": {
+          "average": 106.6,
+          "min": 83.09,
+          "max": 121.06,
+          "total": 639.36
+        },
+        "count": {
+          "total": 6
+        }
+      },
+      "clone": {
+        "duration": {
+          "average": 4.6,
+          "min": 2.78,
+          "max": 8.1,
+          "total": 63.94
+        },
+        "count": {
+          "total": 14
+        }
+      },
+      "tag": {
+        "department": {
+          "engineering": 14,
+          "help desk": 10,
+          "IT": 44,
+          "total": 68
+        },
+        "user": {
+          "arodgers": 54,
+          "cmatthews": 10,
+          "jnelson": 4,
+          "total": 68
+        }
+      }
+    }
+  ]
+}
+```
+
+
+```
+$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v2/summary
+```
+```json
+{
+  "boot": {...},
+  "clone": {...},
+  "daily": [
+    {
+      "date": "2015-03-10",
+      "boot": {
+        "duration": {
+          "average": 0,
+          "min": 0,
+          "max": 0,
+          "total": 0
+        },
+        "count": {
+          "total": 0
+        }
+      },
+      "clone": {
+        "duration": {
+          "average": 0,
+          "min": 0,
+          "max": 0,
+          "total": 0
+        },
+        "count": {
+          "total": 0
+        }
+      },
+      "tag": { }
+    },
+    {
+      "date": "2015-03-11",
+      "boot": {
+        "duration": {
+          "average": 106.6,
+          "min": 83.09,
+          "max": 121.06,
+          "total": 639.36
+        },
+        "count": {
+          "total": 6
+        }
+      },
+      "clone": {
+        "duration": {
+          "average": 4.6,
+          "min": 2.78,
+          "max": 8.1,
+          "total": 63.94
+        },
+        "count": {
+          "total": 14
+        }
+      },
+      "tag": { }
+    }
+  ]
+}
+```
+
+You can also query only the specific top level section you want by including it after `summary/`.
+The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v2/summary/boot/`.
+You can further drill-down the data by specifying the second level parameter to query eg
+`vmpooler.example.com/api/v2/summary/tag/created_by`
+
+##### GET /poolstat?pool=FOO
+
+For parameter `pool`, containing a comma separated list of pool names to query, this endpoint returns
+each of the pool's ready, max and alias information. It can be used to get a fast response for
+the required pools instead of using the /status API endpoint
+
+Return codes
+* 200  OK 
+
+```
+$ curl https://vmpooler.example.com/api/v2/poolstat?pool=centos-6-x86_64
+```
+```json
+{
+  "pools": {
+    "centos-6-x86_64": {
+      "ready": 25,
+      "max": 25,
+      "alias": [
+        "centos-6-64",
+        "centos-6-amd64"
+      ]
+    }
+  }
+}
+```
+
+##### GET /totalrunning
+
+Fast endpoint to return the total number of VMs in a 'running' state
+
+Return codes
+* 200  OK 
+
+```
+$ curl https://vmpooler.example.com/api/v2/totalrunning
+```
+
+```json
+{
+
+  "running": 362
+
+}
+```
+
+#### Managing pool configuration via API <a name="poolconfig"></a>
+
+##### GET /config
+
+Returns the running pool configuration
+
+Return codes
+* 200  OK 
+* 400  No configuration found  
+
+```
+$ curl https://vmpooler.example.com/api/v2/config
+```
+```json
+{
+  "pool_configuration": [
+    {
+      "name": "redhat-7-x86_64",
+      "template": "templates/redhat-7.2-x86_64-0.0.3",
+      "folder": "vmpooler/redhat-7-x86_64",
+      "datastore": "stor1",
+      "size": 1,
+      "datacenter": "dc1",
+      "provider": "vsphere",
+      "capacity": 1,
+      "major": "redhat",
+      "template_ready": true
+    }
+  ],
+  "status": {
+    "ok": true
+  }
+}
+```
+
+Note: to enable poolsize and pooltemplate config endpoints it is necessary to set 'experimental_features: true' in your vmpooler configuration. A 405 is returned when you attempt to interact with these endpoints when this configuration option is not set.
+
+##### POST /config/poolsize
+
+Change pool size without having to restart the service.
+
+All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned
+
+When a pool size is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned.
+
+Pool size configuration changes persist through application restarts, and take precedence over a pool size value configured in the pool configuration provided when the application starts. This persistence is dependent on redis. So, if the redis data is lost then the configuration updates revert to those provided at startup at the next application start.
+
+An authentication token is required in order to change pool configuration when authentication is configured.
+Responses:
+* 200 - No changes required
+* 201 - Changes made on at least one pool with changes requested
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v2/config/poolsize
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### DELETE /config/poolsize/&lt;pool&gt;
+
+Delete an overridden pool size. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - size reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/poolsize/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "pool_size_before_overrides": 2,
+    "pool_size_before_reset": 4
+}
+```
+
+##### POST /config/pooltemplate
+
+Change the template configured for a pool, and replenish the pool with instances built from the new template.
+
+All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned
+
+When a pool template is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned.
+
+A pool template being updated will cause the following actions, which are logged in vmpooler.log:
+* Destroy all instances for the pool template being updated that are in the ready and pending state
+* Halt repopulating the pool while creating template deltas for the newly configured template
+* Unblock pool population and let the pool replenish with instances based on the newly configured template
+
+Pool template changes persist through application restarts, and take precedence over a pool template configured in the pool configuration provided when the application starts. This persistence is dependent on redis. As a result, if the redis data is lost then the configuration values revert to those provided at startup at the next application start.
+
+An authentication token is required in order to change pool configuration when authentication is configured.
+
+Responses:
+* 200 - No changes required
+* 201 - Changes made on at least one pool with changes requested
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v2/config/pooltemplate
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### DELETE /config/pooltemplate/&lt;pool&gt;
+
+Delete an overridden pool template. This results in the values from VMPooler's config being used.
+
+Return codes:
+* 200 - when nothing was changed but no error occurred
+* 201 - template reset successful
+* 401 - when not authorized
+* 404 - pool does not exist
+* 405 - The endpoint is disabled because experimental features are disabled
+
+```
+$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/pooltemplate/almalinux-8-x86_64
+```
+```json
+{
+    "ok": true,
+    "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2",
+    "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta"
+}
+```
+
+##### POST /poolreset
+
+Clear all pending and ready instances in a pool, and deploy replacements
+
+All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned.
+
+When a pool reset is requested a 201 status will be returned.
+
+A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances.
+
+For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured.
+
+Responses:
+* 201 - Pool reset requested received
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v2/poolreset
+```
+```json
+{
+  "ok": true
+}
+```
+
+#### Ondemand VM operations <a name="ondemandvm"></a>
+
+Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated.
+
+##### POST /ondemandvm
+
+All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned
+
+When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status.
+
+An authentication token is required in order to request instances on demand when authentication is configured.
+
+Responses:
+* 201 - Provisioning request accepted
+* 400 - Payload contains invalid JSON and cannot be parsed
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 403 - Request exceeds the configured per pool maximum
+* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred.
+* 409 - A request of the matching ID has already been created
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v2/ondemandvm
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a"
+}
+```
+
+##### GET /ondemandvm
+
+Get the status of an ondemandvm request that has already been posted.
+
+When the request is ready the ready status will change to 'true'.
+
+The number of instances pending vs ready will be reflected in the API response.
+
+Responses:
+* 200 - The API request was successful and the status is ok
+* 202 - The request is not ready yet
+* 404 - The request can not be found, or an unknown error occurred
+```
+$ curl https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": false,
+  "debian-7-i386": {
+    "ready": "3",
+    "pending": "1"
+  }
+}
+```
+```json
+{
+  "ok": true,
+  "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a",
+  "ready": true,
+  "debian-7-i386": {
+    "hostname": [
+      "vm1",
+      "vm2",
+      "vm3",
+      "vm4"
+    ]
+  }
+}
+```
+
+##### DELETE /ondemandvm
+
+Delete a ondemand request
+
+Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted.
+
+Responses:
+* 200 - The API request was sucessful. A message will indicate if the request has already been deleted.
+* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled
+* 404 - The request can not be found, or an unknown error occurred.
+```
+$ curl -X DELETE https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a
+```
+```json
+{
+  "ok": true
+}
+```

docs/configuration.md

@@ -21,7 +21,7 @@ Path to a the file to use when loading the vmpooler configuration. This is only
 
 ### DOMAIN
 
-If set, returns a top-level 'domain' JSON key in POST requests
+This environment variable is to set the configuration config.domain
 (optional)
 
 ### REDIS\_SERVER

lib/vmpooler.rb

@@ -43,6 +43,7 @@ module Vmpooler
       if parsed_config[:config]['extra_config']
         extra_configs = parsed_config[:config]['extra_config'].split(',')
         extra_configs.each do |config|
+          puts "loading extra_config file #{config}"
           extra_config = YAML.load_file(config)
           parsed_config.deep_merge(extra_config)
         end
@@ -133,6 +134,7 @@ module Vmpooler
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
     unless parsed_config[:pools]
+      puts 'loading pools configuration from redis, since the config[:pools] is empty'
       redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
       parsed_config[:pools] = load_pools_from_redis(redis)
     end

lib/vmpooler/api.rb

@@ -3,7 +3,7 @@
 module Vmpooler
   class API < Sinatra::Base
     # Load API components
-    %w[helpers dashboard reroute v1 request_logger healthcheck].each do |lib|
+    %w[helpers dashboard reroute v1 v2 request_logger healthcheck].each do |lib|
       require "vmpooler/api/#{lib}"
     end
     # Load dashboard components
@@ -54,6 +54,7 @@ module Vmpooler
         use Vmpooler::API::Dashboard
         use Vmpooler::API::Reroute
         use Vmpooler::API::V1
+        use Vmpooler::API::V2
       end
 
       # Get thee started O WebServer

lib/vmpooler/api/helpers.rb

@@ -181,12 +181,9 @@ module Vmpooler
         /^\d{4}-\d{2}-\d{2}$/ === date_str
       end
 
+      # NOTE: domain is not needed here, so we should update the callers of this method
       def hostname_shorten(hostname, domain=nil)
-        if domain && hostname =~ /^[\w-]+\.#{domain}$/
+        hostname[/[^.]+/]
-          hostname = hostname[/[^.]+/]
-        end
-
-        hostname
       end
 
       def get_task_times(backend, task, date_str)

lib/vmpooler/api/reroute.rb

@@ -6,66 +6,82 @@ module Vmpooler
       api_version = '1'
 
       get '/status/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /status/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/status")
       end
 
       get '/summary/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/summary")
       end
 
       get '/summary/:route/?:key?/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/:route/?:key?/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}")
       end
 
       get '/token/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
       end
 
       post '/token/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/token")
       end
 
       get '/token/:token/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
       end
 
       delete '/token/:token/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}")
       end
 
       get '/vm/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
       end
 
       post '/vm/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm")
       end
 
       post '/vm/:template/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:template/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}")
       end
 
       get '/vm/:hostname/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
       end
 
       delete '/vm/:hostname/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
       end
 
       put '/vm/:hostname/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}")
       end
 
       post '/vm/:hostname/snapshot/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot")
       end
 
       post '/vm/:hostname/snapshot/:snapshot/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/:snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}")
       end
 
       put '/vm/:hostname/disk/:size/?' do
+        puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/disk/:size/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance"
         call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}")
       end
     end

lib/vmpooler/api/v1.rb

@@ -24,6 +24,10 @@ module Vmpooler
       Vmpooler::API.settings.config[:config]
     end
 
+    def full_config
+      Vmpooler::API.settings.config
+    end
+
     def pools
       Vmpooler::API.settings.config[:pools]
     end
@@ -1515,7 +1519,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
       content_type :json
-      metrics.increment('http_requests_vm_total.post.vm.disksize')
+      metrics.increment('http_requests_vm_total.post.vm.snapshot')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -1734,6 +1738,20 @@ module Vmpooler
       end
       JSON.pretty_generate(result)
     end
+
+    get "#{api_prefix}/full_config/?" do
+      content_type :json
+
+      result = {
+        full_config: full_config,
+        status: {
+          ok: true
+        }
+      }
+
+      status 200
+      JSON.pretty_generate(result)
+    end
   end
   end
 end

lib/vmpooler/api/v2.rb

@@ -0,0 +1,505 @@
+# frozen_string_literal: true
+
+require 'vmpooler/api/v1'
+
+module Vmpooler
+  class API
+    class V2 < Vmpooler::API::V1
+      api_version = '2'
+      api_prefix  = "/api/v#{api_version}"
+
+      def full_config
+        Vmpooler::API.settings.config
+      end
+
+      def get_template_aliases(template)
+        tracer.in_span("Vmpooler::API::V2.#{__method__}") do
+          result = []
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            result += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+          end
+          result
+        end
+      end
+
+      # Fetch a single vm from a pool
+      #
+      # @param [String] template
+      #   The template that the vm should be created from
+      #
+      # @return [Tuple] vmname, vmpool, vmtemplate
+      #   Returns a tuple containing the vm's name, the pool it came from, and
+      #   what template was used, if successful. Otherwise the tuple contains.
+      #   nil values.
+      def fetch_single_vm(template)
+        tracer.in_span("Vmpooler::API::V2.#{__method__}") do
+          template_backends = [template]
+          aliases = Vmpooler::API.settings.config[:alias]
+          if aliases
+            template_backends += aliases[template] if aliases[template].is_a?(Array)
+            template_backends << aliases[template] if aliases[template].is_a?(String)
+            pool_index = pool_index(pools)
+            weighted_pools = {}
+            template_backends.each do |t|
+              next unless pool_index.key? t
+
+              index = pool_index[t]
+              clone_target = pools[index]['clone_target'] || config['clone_target']
+              next unless config.key?('backend_weight')
+
+              weight = config['backend_weight'][clone_target]
+              if weight
+                weighted_pools[t] = weight
+              end
+            end
+
+            if weighted_pools.count == template_backends.count
+              pickup = Pickup.new(weighted_pools)
+              selection = pickup.pick
+              template_backends.delete(selection)
+              template_backends.unshift(selection)
+            else
+              first = template_backends.sample
+              template_backends.delete(first)
+              template_backends.unshift(first)
+            end
+          end
+
+          checkoutlock.synchronize do
+            template_backends.each do |template_backend|
+              vms = backend.smembers("vmpooler__ready__#{template_backend}")
+              next if vms.empty?
+
+              vm = vms.pop
+              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+              if smoved
+                return [vm, template_backend, template]
+              end
+            end
+            [nil, nil, nil]
+          end
+        end
+      end
+
+      # The domain in the result body will be set to the one associated with the
+      # last vm added. The part of the response is only being retained for
+      # backwards compatibility as the hostnames are now fqdn's instead of bare
+      # hostnames. This change is a result of now being able to specify a domain
+      # per pool. If no vm's in the result had a domain sepcified then the
+      # domain key will be omitted similar to how it was previously omitted if
+      # the global option domain wasn't specified.
+      def atomically_allocate_vms(payload)
+        tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
+          result = { 'ok' => false }
+          failed = false
+          vms = [] # vmpool, vmname, vmtemplate
+
+          validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
+
+          payload.each do |requested, count|
+            count.to_i.times do |_i|
+              vmname, vmpool, vmtemplate = fetch_single_vm(requested)
+              if vmname
+                account_for_starting_vm(vmpool, vmname)
+                vms << [vmpool, vmname, vmtemplate]
+                metrics.increment("checkout.success.#{vmpool}")
+                update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
+              else
+                failed = true
+                metrics.increment("checkout.empty.#{requested}")
+                break
+              end
+            end
+          end
+
+          if failed
+            vms.each do |(vmpool, vmname, _vmtemplate)|
+              return_vm_to_ready_state(vmpool, vmname)
+            end
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V2.atomically_allocate_vms',
+              'error.message' => '503 due to failing to allocate one or more vms'
+            })
+            status 503
+          else
+            vm_names = []
+            vms.each do |(vmpool, vmname, vmtemplate)|
+              vmdomain = Parsing.get_domain_for_pool(full_config, vmpool)
+              if vmdomain
+                vmfqdn = "#{vmname}.#{vmdomain}"
+                update_result_hosts(result, vmtemplate, vmfqdn)
+                vm_names.append(vmfqdn)
+              else
+                update_result_hosts(result, vmtemplate, vmname)
+                vm_names.append(vmname)
+              end
+            end
+
+            span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
+
+            result['ok'] = true
+          end
+
+          result
+        end
+      end
+
+      def generate_ondemand_request(payload)
+        tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
+          result = { 'ok': false }
+
+          requested_instances = payload.reject { |k, _v| k == 'request_id' }
+          if too_many_requested?(requested_instances)
+            e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+            result['message'] = e_message
+            status 403
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
+              'error.message' => "403 due to #{e_message}"
+            })
+            return result
+          end
+
+          score = Time.now.to_i
+          request_id = payload['request_id']
+          request_id ||= generate_request_id
+          result['request_id'] = request_id
+          span.set_attribute('vmpooler.request_id', request_id)
+
+          if backend.exists?("vmpooler__odrequest__#{request_id}")
+            e_message = "request_id '#{request_id}' has already been created"
+            result['message'] = e_message
+            status 409
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
+              'error.message' => "409 due to #{e_message}"
+            })
+            metrics.increment('ondemandrequest_generate.duplicaterequests')
+            return result
+          end
+
+          status 201
+
+          platforms_with_aliases = []
+          requested_instances.each do |poolname, count|
+            selection = evaluate_template_aliases(poolname, count)
+            selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+          end
+          platforms_string = platforms_with_aliases.join(',')
+
+          return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+          backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+          if Vmpooler::API.settings.config[:auth] and has_token?
+            token_token = request.env['HTTP_X_AUTH_TOKEN']
+            token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
+            backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
+            span.set_attribute('enduser.id', token_user)
+          end
+
+          result[:ok] = true
+          metrics.increment('ondemandrequest_generate.success')
+          result
+        end
+      end
+
+      # Endpoints that use overridden methods
+
+      post "#{api_prefix}/vm/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.post.vm.checkout')
+
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/vm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.get.vm.template')
+
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload)
+          if invalid.empty?
+            result = atomically_allocate_vms(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("checkout.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('checkout.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/vm/:hostname/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.vm.hostname')
+
+        result = {}
+
+        status 404
+        result['ok'] = false
+
+        params[:hostname] = hostname_shorten(params[:hostname], nil)
+
+        rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
+        unless rdata.empty?
+          status 200
+          result['ok'] = true
+
+          result[params[:hostname]] = {}
+
+          result[params[:hostname]]['template'] = rdata['template']
+          result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
+
+          if rdata['destroy']
+            result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout']
+            result[params[:hostname]]['state'] = 'destroyed'
+          elsif rdata['checkout']
+            result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
+            result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
+            result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
+            result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
+            result[params[:hostname]]['state'] = 'running'
+          elsif rdata['check']
+            result[params[:hostname]]['state'] = 'ready'
+          else
+            result[params[:hostname]]['state'] = 'pending'
+          end
+
+          rdata.keys.each do |key|
+            if key.match('^tag\:(.+?)$')
+              result[params[:hostname]]['tags'] ||= {}
+              result[params[:hostname]]['tags'][$1] = rdata[key]
+            end
+
+            if key.match('^snapshot\:(.+?)$')
+              result[params[:hostname]]['snapshots'] ||= []
+              result[params[:hostname]]['snapshots'].push($1)
+            end
+          end
+
+          if rdata['disk']
+            result[params[:hostname]]['disk'] = rdata['disk'].split(':')
+          end
+
+          # Look up IP address of the hostname
+          begin
+            ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
+          rescue StandardError
+            ipAddress = ""
+          end
+
+          result[params[:hostname]]['ip'] = ipAddress
+
+          if rdata['pool']
+            vmdomain = Parsing.get_domain_for_pool(full_config, rdata['pool'])
+            if vmdomain
+              result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}"
+            end
+          end
+
+          result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
+          result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
+
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.post.ondemand.requestid')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        result = { 'ok' => false }
+
+        begin
+          payload = JSON.parse(request.body.read)
+
+          if payload
+            invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+            if invalid.empty?
+              result = generate_ondemand_request(payload)
+            else
+              result[:bad_templates] = invalid
+              invalid.each do |bad_template|
+                metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+              end
+              status 404
+            end
+          else
+            metrics.increment('ondemandrequest_fail.invalid.unknown')
+            status 404
+          end
+        rescue JSON::ParserError
+          span = OpenTelemetry::Trace.current_span
+          span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
+          status 400
+          result = {
+            'ok' => false,
+            'message' => 'JSON payload could not be parsed'
+          }
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      post "#{api_prefix}/ondemandvm/:template/?" do
+        content_type :json
+        result = { 'ok' => false }
+        metrics.increment('http_requests_vm_total.delete.ondemand.template')
+
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = extract_templates_from_query_params(params[:template])
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest_fail.invalid.unknown')
+          status 404
+        end
+
+        JSON.pretty_generate(result)
+      end
+
+      get "#{api_prefix}/ondemandvm/:requestid/?" do
+        content_type :json
+        metrics.increment('http_requests_vm_total.get.ondemand.request')
+
+        status 404
+        result = check_ondemand_request(params[:requestid])
+
+        JSON.pretty_generate(result)
+      end
+
+      def check_ondemand_request(request_id)
+        tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
+          span.set_attribute('vmpooler.request_id', request_id)
+          result = { 'ok' => false }
+          request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+          if request_hash.empty?
+            e_message = "no request found for request_id '#{request_id}'"
+            result['message'] = e_message
+            span.add_event('error', attributes: {
+              'error.type' => 'Vmpooler::API::V2.check_ondemand_request',
+              'error.message' => e_message
+            })
+            return result
+          end
+
+          result['request_id'] = request_id
+          result['ready'] = false
+          result['ok'] = true
+          status 202
+
+          case request_hash['status']
+          when 'ready'
+            result['ready'] = true
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+              instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              domain = Parsing.get_domain_for_pool(full_config, pool)
+              instances.map! { |instance| instance.concat(".#{domain}") } if domain
+
+              if result.key?(platform_alias)
+                result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
+              else
+                result[platform_alias] = { 'hostname': instances }
+              end
+            end
+            status 200
+          when 'failed'
+            result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+            status 200
+          when 'deleted'
+            result['message'] = 'The request has been deleted'
+            status 200
+          else
+            Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+              instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+              instances_pending = count.to_i - instance_count.to_i
+
+              if result.key?(platform_alias) && result[platform_alias].key?(:ready)
+                result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
+                result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
+              else
+                result[platform_alias] = {
+                  'ready': instance_count.to_s,
+                  'pending': instances_pending.to_s
+                }
+              end
+            end
+          end
+
+          result
+        end
+      end
+
+      # Endpoints that only use bits from the V1 api are called here
+      # Note that traces will be named based on the route used in the V1 api
+      # but the http.url trace attribute will still have the actual requested url in it
+
+      delete "#{api_prefix}/*" do
+        versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
+        request.path_info = "/api/v1/#{versionless_path_info}"
+        call env
+      end
+
+      get "#{api_prefix}/*" do
+        versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
+        request.path_info = "/api/v1/#{versionless_path_info}"
+        call env
+      end
+
+      post "#{api_prefix}/*" do
+        versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
+        request.path_info = "/api/v1/#{versionless_path_info}"
+        call env
+      end
+
+      put "#{api_prefix}/*" do
+        versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
+        request.path_info = "/api/v1/#{versionless_path_info}"
+        call env
+      end
+    end
+  end
+end

lib/vmpooler/pool_manager.rb

@@ -60,6 +60,7 @@ module Vmpooler
             to_set[k] = pool[k]
           end
           to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
+          to_set['domain'] = Parsing.get_domain_for_pool(config, pool['name'])
           redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
         end
         previously_configured_pools.each do |pool|
@@ -361,35 +362,47 @@ module Vmpooler
       max_hostname_retries = 3
       while hostname_retries < max_hostname_retries
         hostname, hostname_available = generate_and_check_hostname
-        domain = $config[:config]['domain']
+        domain = Parsing.get_domain_for_pool(config, pool_name)
-        dns_ip, dns_available = check_dns_available(hostname, domain)
+        if domain
+          fqdn = "#{hostname}.#{domain}"
+        else
+          fqdn = hostname
+        end
+
+        # skip dns check if the provider is set to skip_dns_check_before_creating_vm
+        provider = get_provider_for_pool(pool_name)
+        if provider && provider.provider_config['skip_dns_check_before_creating_vm']
+          dns_available = true
+        else
+          dns_ip, dns_available = check_dns_available(fqdn)
+        end
+
         break if hostname_available && dns_available
 
         hostname_retries += 1
 
         if !hostname_available
           $metrics.increment("errors.duplicatehostname.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
         elsif !dns_available
           $metrics.increment("errors.staledns.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS")
         end
       end
 
-      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
+      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available
 
       hostname
     end
 
-    def check_dns_available(vm_name, domain = nil)
+    # Query the DNS for the name we want to create and if it already exists, mark it unavailable
-      # Query the DNS for the name we want to create and if it already exists, mark it unavailable
+    # This protects against stale DNS records
-      # This protects against stale DNS records
+    def check_dns_available(vm_name)
-      vm_name = "#{vm_name}.#{domain}" if domain
       begin
         dns_ip = Resolv.getaddress(vm_name)
       rescue Resolv::ResolvError
         # this is the expected case, swallow the error
-        # eg "no address for blah-daisy"
+        # eg "no address for blah-daisy.example.com"
         return ['', true]
       end
       [dns_ip, false]
@@ -397,6 +410,7 @@ module Vmpooler
 
     def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
       new_vmname = find_unique_hostname(pool_name)
+      pool_domain = Parsing.get_domain_for_pool(config, pool_name)
       mutex = vm_mutex(new_vmname)
       mutex.synchronize do
         @redis.with_metrics do |redis|
@@ -406,6 +420,7 @@ module Vmpooler
           redis.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now)
           redis.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool.
           redis.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name)
+          redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) if pool_domain
           redis.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id
           redis.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias
           redis.exec
@@ -879,7 +894,7 @@ module Vmpooler
           loop_count = 1
           loop_delay = loop_delay_min
           provider = get_provider_for_pool(pool['name'])
-          raise("Could not find provider '#{pool['provider']}") if provider.nil?
+          raise("Could not find provider '#{pool['provider']}'") if provider.nil?
 
           sync_pool_template(pool)
           loop do
@@ -1366,7 +1381,7 @@ module Vmpooler
 
         return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
       end
-      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
+      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil?
     end
 
     def check_ondemand_requests(maxloop = 0,
@@ -1581,19 +1596,19 @@ module Vmpooler
       $config[:pools].each do |pool|
         provider_name = pool['provider']
         # The provider_class parameter can be defined in the provider's data eg
-        #:providers:
+        # :providers:
-        # :vsphere:
+        #   :vsphere:
-        #  provider_class: 'vsphere'
+        #     provider_class: 'vsphere'
-        # :another-vsphere:
+        #   :another-vsphere:
-        #  provider_class: 'vsphere'
+        #     provider_class: 'vsphere'
         # the above would create two providers/vsphere.rb class objects named 'vsphere' and 'another-vsphere'
         # each pools would then define which provider definition to use: vsphere or another-vsphere
         #
         # if provider_class is not defined it will try to use the provider_name as the class, this is to be
         # backwards compatible for example when there is only one provider listed
         # :providers:
-        #  :dummy:
+        #   :dummy:
-        #   filename: 'db.txs'
+        #     filename: 'db.txs'
         # the above example would create an object based on the class providers/dummy.rb
         if $config[:providers].nil? || $config[:providers][provider_name.to_sym].nil? || $config[:providers][provider_name.to_sym]['provider_class'].nil?
           provider_class = provider_name

lib/vmpooler/util/parsing.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# utility class shared between apps
+# utility class shared between apps api and pool_manager
 module Vmpooler
   class Parsing
     def self.get_platform_pool_count(requested, &_block)
@@ -12,5 +12,25 @@ module Vmpooler
         yield platform_alias, pool, count
       end
     end
+
+    # @param config [String] - the full config structure
+    # @param pool_name [String] - the name of the pool
+    # @return [String] - domain name for pool, if set in the provider for the pool or in the config block
+    def self.get_domain_for_pool(config, pool_name)
+      pool = config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      provider_name = pool.fetch('provider', 'vsphere') # see vmpooler.yaml.example where it states defaulting to vsphere
+
+      if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain']
+        domain = config[:providers][provider_name.to_sym]['domain']
+      elsif config[:config] && config[:config]['domain']
+        domain = config[:config]['domain']
+      else
+        domain = nil
+      end
+
+      domain
+    end
   end
 end

lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end

spec/integration/api/v1/vm_spec.rb

@@ -124,8 +124,7 @@ describe Vmpooler::API::V1 do
       end
 
       it 'returns 503 for empty pool referenced by alias' do
-        create_ready_vm 'pool1', vmname, redis
+        create_ready_vm 'pool2', vmname, redis
-        post "#{prefix}/vm/poolone"
         post "#{prefix}/vm/poolone"
 
         expected = { ok: false }

spec/integration/api/v1/vm_template_spec.rb

@@ -24,6 +24,7 @@ describe Vmpooler::API::V1 do
           'site_name' => 'test pooler',
           'vm_lifetime_auth' => 2,
         },
+        providers: { vsphere: {} },
         pools: [
           {'name' => 'pool1', 'size' => 5},
           {'name' => 'pool2', 'size' => 10},

spec/integration/api/v2/config_spec.rb

@@ -0,0 +1,402 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  let(:config) {
+    {
+      config: {
+        'site_name' => 'test pooler',
+        'vm_lifetime_auth' => 2,
+        'experimental_features' => true
+      },
+      pools: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      pools_at_startup: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      statsd: { 'prefix' => 'stats_prefix'},
+      alias: { 'poolone' => 'pool1' },
+      pool_names: [ 'pool1', 'pool2', 'poolone' ]
+    }
+  }
+
+  describe '/config/pooltemplate' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'DELETE /config/pooltemplate/:pool' do
+      it 'resets a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          template_before_reset: 'templates/new_template',
+          template_before_overrides: 'templates/pool1'
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when the pool has not been overridden' do
+        delete "#{prefix}/config/pooltemplate/pool1"
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails on nonexistent pools' do
+        delete "#{prefix}/config/pooltemplate/poolpoolpool"
+        expect_json(ok = false, http = 404)
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/pooltemplate/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+    end
+
+    describe 'POST /config/pooltemplate' do
+      it 'updates a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/config/pooltemplate", '{"poolpoolpool":"templates/newtemplate"}'
+        expect_json(ok = false, http = 400)
+      end
+
+      it 'updates multiple pools' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool2":"templates/new_template2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool3":"templates/new_template2"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_templates: ['pool3']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns no changes when the template does not change' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/pool1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a invalid template parameter is provided' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"template1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_templates: ['pool1']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a template starts with /' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"/template1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_templates: ['pool1']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a template ends with /' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"template1/"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_templates: ['pool1']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/config/pooltemplate", '{"pool1":"template/template1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+    end
+
+    describe 'DELETE /config/poolsize' do
+      it 'resets a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 201)
+
+        expected = {
+          ok: true,
+          pool_size_before_reset: 2,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        delete "#{prefix}/config/poolsize/pool10"
+        expect_json(ok = false, http = 404)
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds when a pool has not been overridden' do
+        delete "#{prefix}/config/poolsize/pool1"
+        expect_json(ok = true, http = 200)
+        expected = {
+          ok: true,
+          pool_size_before_reset: 5,
+          pool_size_before_overrides: 5
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          delete "#{prefix}/config/poolsize/pool1"
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
+    describe 'POST /config/poolsize' do
+      it 'changes a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'changes a pool size for multiple pools' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2","pool2":"2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        post "#{prefix}/config/poolsize", '{"pool10":"2"}'
+        expect_json(ok = false, http = 400)
+        expected = {
+          ok: false,
+          not_configured: ['pool10']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 200 when no change is required' do
+        post "#{prefix}/config/poolsize", '{"pool1":"5"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 201 when at least one pool changes' do
+        post "#{prefix}/config/poolsize", '{"pool1":"5","pool2":"5"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a non-integer value is provided for size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"four"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          not_configured: ['pool1']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a negative value is provided for size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"-1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          not_configured: ['pool1']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/config/poolsize", '{"pool1":"1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
+    describe 'POST /config/clonetarget' do
+      it 'changes the clone target' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'changes a pool size for multiple pools' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        post "#{prefix}/config/clonetarget", '{"pool10":"cluster1"}'
+        expect_json(ok = false, http = 400)
+        expected = {
+          ok: false,
+          bad_templates: ['pool10']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 200 when no change is required' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 201 when at least one pool changes' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
+    describe 'GET /config' do
+      let(:prefix) { '/api/v2' }
+
+      it 'returns pool configuration when set' do
+        get "#{prefix}/config"
+
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result['pool_configuration']).to eq(config[:pools])
+      end
+    end
+  end
+end

spec/integration/api/v2/ondemandvm_spec.rb

@@ -0,0 +1,412 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/ondemandvm' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+          'max_ondemand_instances_per_request' => 50,
+          'backend_weight' => {
+            'compute1' => 5,
+            'compute2' => 0
+          }
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1'},
+          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2'},
+          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1'}
+        ],
+        alias: {
+          'poolone' => ['pool1'],
+          'pool2' => ['pool1']
+        },
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ],
+        providers: {
+          :dummy => {
+            'domain' => 'dummy.com'
+          }
+        }
+      }
+    }
+    let(:current_time) { Time.now }
+    let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
+    let(:uuid) { SecureRandom.uuid }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+      config[:pools].each do |pool|
+        redis.sadd('vmpooler__pools', pool['name'])
+      end
+    end
+
+    describe 'POST /ondemandvm' do
+
+      context 'with a configured pool' do
+
+        context 'with no request_id provided in payload' do
+          before(:each) do
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          end
+
+          it 'generates a request_id when none is provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'uses a configured platform to fulfill a ondemand request' do
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+            expect_json(true, 201)
+            expected = {
+              "ok": true,
+              "request_id": uuid
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'creates a provisioning request in redis' do
+            expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1)
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          it 'sets a platform string in redis for the request to indicate selected platforms' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1')
+            post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          end
+
+          context 'with a backend of 0 weight' do
+            before(:each) do
+              config[:config]['backend_weight']['compute1'] = 0
+            end
+
+            it 'sets the platform string in redis for the request to indicate the selected platforms' do
+              expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool1:pool1:1')
+              post "#{prefix}/ondemandvm", '{"pool1":"1"}'
+            end
+          end
+
+          it 'sets the platform string in redis for the request to indicate the selected platforms using weight' do
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool2:pool1:1')
+            post "#{prefix}/ondemandvm", '{"pool2":"1"}'
+          end
+
+          context 'with domain set in the config' do
+            let(:domain) { 'example.com' }
+            before(:each) do
+              config[:config]['domain'] = domain
+            end
+
+            it 'should include domain in the return reply' do
+              post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+              expect_json(true, 201)
+              expected = {
+                "ok": true,
+                "request_id": uuid,
+              }
+              expect(last_response.body).to eq(JSON.pretty_generate(expected))
+            end
+          end
+        end
+
+        context 'with a resource request that exceeds the specified limit' do
+          let(:max_instances) { 50 }
+          before(:each) do
+            config[:config]['max_ondemand_instances_per_request'] = max_instances
+          end
+
+          it 'should reject the request with a message' do
+            post "#{prefix}/ondemandvm", '{"pool1":"51"}'
+            expect_json(false, 403)
+            expected = {
+              "ok": false,
+              "message": "requested amount of instances exceeds the maximum #{max_instances}"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with request_id provided in the payload' do
+          it 'uses the given request_id when provided' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(true, 201)
+
+            expected = {
+              "ok": true,
+              "request_id": "1234"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          it 'returns 409 conflict error when the request_id has been used' do
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
+            expect_json(false, 409)
+
+            expected = {
+              "ok": false,
+              "request_id": "1234",
+              "message": "request_id '1234' has already been created"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with auth configured' do
+
+          it 'sets the token and user' do
+            app.settings.set :config, auth: true
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+            allow(redis).to receive(:hset)
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345')
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe')
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}', {
+              'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+            }
+          end
+        end
+      end
+
+      context 'with a pool that is not configured' do
+        let(:badpool) { 'pool4' }
+        it 'returns the bad template' do
+          post "#{prefix}/ondemandvm", '{"pool4":"1"}'
+          expect_json(false, 404)
+
+          expected = {
+            "ok": false,
+            "bad_templates": [ badpool ]
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'returns 400 and a message when JSON is invalid' do
+        post "#{prefix}/ondemandvm", '{"pool1":"1}'
+        expect_json(false, 400)
+        expected = {
+          "ok": false,
+          "message": "JSON payload could not be parsed"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+
+    describe 'GET /ondemandvm' do
+      it 'returns 404 with message when request is not found' do
+        get "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:score) { current_time }
+        let(:platforms_string) { 'pool1:pool1:1' }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 202 while the request is waiting' do
+          get "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 202)
+          expected = {
+            "ok": true,
+            "request_id": uuid,
+            "ready": false,
+            "pool1": {
+              "ready": "0",
+              "pending": "1"
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        context 'with ready instances' do
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis)
+            set_ondemand_request_status(uuid, 'ready', redis)
+          end
+
+          it 'returns 200 with hostnames when the request is ready' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": true,
+              "pool1": {
+                "hostname": [
+                  vmname
+                ]
+              }
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+
+          context 'with domain set' do
+            let(:domain) { 'example.com' }
+            before(:each) do
+              config[:config]['domain'] = domain
+            end
+
+            it 'should include the domain in the hostname as fqdn, not a separate key unlike in v1' do
+              get "#{prefix}/ondemandvm/#{uuid}"
+              expected = {
+                "ok": true,
+                "request_id": uuid,
+                "ready": true,
+                "pool1": {
+                  "hostname": [
+                    "#{vmname}.#{domain}"
+                  ]
+                }
+              }
+              expect(last_response.body).to eq(JSON.pretty_generate(expected))
+            end
+          end
+
+          context 'with domain set in the provider' do
+            let(:domain) { 'dummy.com' }
+            before(:each) do
+              config[:pools][0]['provider'] = 'dummy'
+            end
+
+            it 'should include the domain in the hostname as fqdn, not a separate key unlike in v1' do
+              get "#{prefix}/ondemandvm/#{uuid}"
+              expected = {
+                "ok": true,
+                "request_id": uuid,
+                "ready": true,
+                "pool1": {
+                  "hostname": [
+                    "#{vmname}.#{domain}"
+                  ]
+                }
+              }
+              expect(last_response.body).to eq(JSON.pretty_generate(expected))
+            end
+          end
+        end
+
+        context 'with a deleted request' do
+          before(:each) do
+            set_ondemand_request_status(uuid, 'deleted', redis)
+          end
+
+          it 'returns a message that the request has been deleted' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request has been deleted"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+
+        context 'with a failed request' do
+          let(:ondemand_request_ttl) { 5 }
+          before(:each) do
+            config[:config]['ondemand_request_ttl'] = ondemand_request_ttl
+            set_ondemand_request_status(uuid, 'failed', redis)
+          end
+
+          it 'returns a message that the request has failed' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": false,
+              "message": "The request failed to provision instances within the configured ondemand_request_ttl '#{ondemand_request_ttl}'"
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+      end
+    end
+
+    describe 'DELETE /ondemandvm' do
+      let(:expiration) { 129_600_0 }
+      it 'returns 404 with message when request is not found' do
+        delete "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:platforms_string) { 'pool1:pool1:1' }
+        let(:score) { current_time.to_i }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 200 for a deleted request' do
+          delete "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 200)
+          expected = { 'ok': true }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        it 'marks the request hash for expiration in two weeks' do
+          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{uuid}", expiration)
+          delete "#{prefix}/ondemandvm/#{uuid}"
+        end
+
+        context 'with running instances' do
+          let(:pool) { 'pool1' }
+          let(:pool_alias) { pool }
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, pool, pool_alias, redis)
+          end
+
+          it 'moves allocated instances to the completed queue' do
+            expect(redis).to receive(:smove).with("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vmname)
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+
+          it 'deletes the set tracking instances allocated for the request' do
+            expect(redis).to receive(:del).with("vmpooler__#{uuid}__#{pool_alias}__#{pool}")
+            delete "#{prefix}/ondemandvm/#{uuid}"
+          end
+        end
+      end
+    end
+  end
+end

spec/integration/api/v2/poolreset.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  after(:each) do
+    Vmpooler::API.reset!
+  end
+
+  let(:config) {
+    {
+      config: {
+        'site_name' => 'test pooler',
+        'vm_lifetime_auth' => 2,
+        'experimental_features' => true
+      },
+      pools: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      statsd: { 'prefix' => 'stats_prefix'},
+      alias: { 'poolone' => 'pool1' },
+      pool_names: [ 'pool1', 'pool2', 'poolone' ]
+    }
+  }
+
+  describe '/poolreset' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /poolreset' do
+      it 'refreshes ready and pending instances from a pool' do
+        post "#{prefix}/poolreset", '{"pool1":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/poolreset", '{"poolpoolpool":"1"}'
+        expect_json(ok = false, http = 400)
+      end
+
+      it 'resets multiple pools' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_pools: ['pool3']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/poolreset", '{"pool1":"1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'should return 400 for invalid json' do
+        post "#{prefix}/poolreset", '{"pool1":"1}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 400 with a bad pool name' do
+        post "#{prefix}/poolreset", '{"pool11":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 404 when there is no payload' do
+        post "#{prefix}/poolreset"
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+  end
+end

spec/integration/api/v2/status_spec.rb

@@ -0,0 +1,247 @@
+require 'spec_helper'
+require 'rack/test'
+
+def has_set_tag?(vm, tag, value)
+  value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
+end
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe 'status and metrics endpoints' do
+    let(:prefix) { '/api/v2' }
+
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 5, 'alias' => ['poolone', 'poolun']},
+          {'name' => 'pool2', 'size' => 10},
+          {'name' => 'pool3', 'size' => 10, 'alias' => 'NotArray'}
+        ]
+      }
+    }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'GET /status' do
+      it 'returns the configured maximum size for each pool' do
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"]["pool1"]["max"]).to be(5)
+        expect(result["pools"]["pool2"]["max"]).to be(10)
+      end
+
+      it 'returns the number of ready vms for each pool' do
+        3.times {|i| create_ready_vm("pool1", "vm-#{i}", redis) }
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"]["pool1"]["ready"]).to be(3)
+        expect(result["pools"]["pool2"]["ready"]).to be(0)
+      end
+
+      it 'returns the number of running vms for each pool' do
+        3.times {|i| create_running_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_running_vm("pool2", "vm-#{i}", redis) }
+
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"]["pool1"]["running"]).to be(3)
+        expect(result["pools"]["pool2"]["running"]).to be(4)
+      end
+
+      it 'returns the number of pending vms for each pool' do
+        3.times {|i| create_pending_vm("pool1", "vm-#{i}", redis) }
+        4.times {|i| create_pending_vm("pool2", "vm-#{i}", redis) }
+
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"]["pool1"]["pending"]).to be(3)
+        expect(result["pools"]["pool2"]["pending"]).to be(4)
+      end
+
+      it 'returns aliases if configured in the pool' do
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun'])
+        expect(result["pools"]["pool2"]["alias"]).to be(nil)
+        expect(result["pools"]["pool3"]["alias"]).to eq('NotArray')
+      end
+
+      it '(for v1 backwards compatibility) lists any empty pools in the status section' do
+        get "#{prefix}/status/"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["status"]["empty"].sort).to eq(["pool1", "pool2", "pool3"])
+      end
+    end
+    describe 'GET /status with view query parameter' do
+      it 'returns capacity when specified' do
+        get "#{prefix}/status?view=capacity"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to_not be(nil)
+        expect(result["queue"]).to be(nil)
+        expect(result["clone"]).to be(nil)
+        expect(result["boot"]).to be(nil)
+        expect(result["pools"]).to be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+      it 'returns pools and queue when specified' do
+        get "#{prefix}/status?view=pools,queue"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to be(nil)
+        expect(result["queue"]).to_not be(nil)
+        expect(result["clone"]).to be(nil)
+        expect(result["boot"]).to be(nil)
+        expect(result["pools"]).to_not be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+      it 'does nothing with invalid view names' do
+        get "#{prefix}/status?view=clone,boot,invalidThingToView"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to be(nil)
+        expect(result["queue"]).to be(nil)
+        expect(result["clone"]).to_not be(nil)
+        expect(result["boot"]).to_not be(nil)
+        expect(result["pools"]).to be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+      it 'returns everything when view is not specified' do
+        get "#{prefix}/status"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to_not be(nil)
+        expect(result["queue"]).to_not be(nil)
+        expect(result["clone"]).to_not be(nil)
+        expect(result["boot"]).to_not be(nil)
+        expect(result["pools"]).to_not be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+      it 'returns everything when view is alone' do
+        get "#{prefix}/status?view"
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to_not be(nil)
+        expect(result["queue"]).to_not be(nil)
+        expect(result["clone"]).to_not be(nil)
+        expect(result["boot"]).to_not be(nil)
+        expect(result["pools"]).to_not be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+      it 'returns status only when view is empty' do
+        get "#{prefix}/status?view="
+
+        # of course /status doesn't conform to the weird standard everything else uses...
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["capacity"]).to be(nil)
+        expect(result["queue"]).to be(nil)
+        expect(result["clone"]).to be(nil)
+        expect(result["boot"]).to be(nil)
+        expect(result["pools"]).to be(nil)
+        expect(result["status"]).to_not be(nil)
+      end
+    end
+
+    describe 'GET /poolstat' do
+      it 'returns empty list when pool is not set' do
+        get "#{prefix}/poolstat"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result  == {})
+      end
+      it 'returns empty list when pool is not found' do
+        get "#{prefix}/poolstat?pool=unknownpool"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result  == {})
+      end
+      it 'returns one pool when requesting one with alias' do
+        get "#{prefix}/poolstat?pool=pool1"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 1)
+        expect(result["pools"]["pool1"]["ready"]).to eq(0)
+        expect(result["pools"]["pool1"]["max"]).to eq(5)
+        expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun'])
+      end
+      it 'returns one pool when requesting one without alias' do
+        get "#{prefix}/poolstat?pool=pool2"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 1)
+        expect(result["pools"]["pool2"]["ready"]).to eq(0)
+        expect(result["pools"]["pool2"]["max"]).to eq(10)
+        expect(result["pools"]["pool2"]["alias"]).to be(nil)
+      end
+      it 'returns multiple pools when requesting csv' do
+        get "#{prefix}/poolstat?pool=pool1,pool2"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        result = JSON.parse(last_response.body)
+        expect(result["pools"].size == 2)
+      end
+    end
+
+    describe 'GET /totalrunning' do
+      it 'returns the number of running VMs' do
+        get "#{prefix}/totalrunning"
+        expect(last_response.header['Content-Type']).to eq('application/json')
+        5.times {|i| create_running_vm("pool1", "vm-#{i}", redis, redis) }
+        5.times {|i| create_running_vm("pool3", "vm-#{i}", redis, redis) }
+        result = JSON.parse(last_response.body)
+        expect(result["running"] == 10)
+      end
+    end
+  end
+end

spec/integration/api/v2/token_spec.rb

@@ -0,0 +1,210 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/token' do
+    let(:prefix) { '/api/v2' }
+    let(:current_time) { Time.now }
+    let(:config) { { 
+      config: {}
+    } }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
+    end
+
+    describe 'GET /token' do
+      context '(auth not configured)' do
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
+
+        it 'returns a 404' do
+          get "#{prefix}/token"
+          expect_json(ok = false, http = 404)
+        end
+      end
+
+      context '(auth configured)' do
+        let(:config) {
+          {
+            config: {},
+            auth: {
+              'provider' => 'dummy'
+            }
+          }
+        }
+        let(:username_str) { 'admin' }
+        let(:password_str) { 's3cr3t' }
+
+        it 'returns a 401 if not authed' do
+          get "#{prefix}/token"
+          expect_json(ok = false, http = 401)
+        end
+
+        it 'returns a list of tokens if authed' do
+          create_token "abc", "admin", current_time
+
+          authorize 'admin', 's3cr3t'
+          get "#{prefix}/token"
+          expect_json(ok = true, http = 200)
+
+          expect(JSON.parse(last_response.body)['abc']['created']).to eq(current_time.to_s)
+        end
+      end
+    end
+
+    describe 'POST /token' do
+      context '(auth not configured)' do
+        let(:config) { { 
+          config: {}, 
+          auth: false 
+        } }
+
+        it 'returns a 404' do
+          post "#{prefix}/token"
+          expect_json(ok = false, http = 404)
+        end
+      end
+
+      context '(auth configured)' do
+        let(:config) {
+          {
+            config: {},
+            auth: {
+              'provider' => 'dummy'
+            }
+          }
+        }
+
+        it 'returns a 401 if not authed' do
+          post "#{prefix}/token"
+          expect_json(ok = false, http = 401)
+        end
+
+        it 'returns a newly created token if authed' do
+          authorize 'admin', 's3cr3t'
+          post "#{prefix}/token"
+          expect_json(ok = true, http = 200)
+
+          returned_token = JSON.parse(last_response.body)['token']
+          expect(returned_token.length).to be(32)
+          expect(get_token_data(returned_token)['user']).to eq("admin")
+        end
+      end
+    end
+  end
+
+  describe '/token/:token' do
+    let(:prefix) { '/api/v2' }
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, nil, nil)
+      app.settings.set :config, config
+      app.settings.set :redis, redis
+    end
+
+    def create_vm_for_token(token, pool, vm)
+      redis.sadd("vmpooler__running__#{pool}", vm)
+      redis.hset("vmpooler__vm__#{vm}", "token:token", token)
+    end
+
+    describe 'GET /token/:token' do
+      context '(auth not configured)' do
+        let(:config) { { 
+          config: {},
+          auth: false 
+        } }
+
+        it 'returns a 404' do
+          get "#{prefix}/token/this"
+          expect_json(ok = false, http = 404)
+        end
+      end
+
+      context '(auth configured)' do
+        let(:config) { {
+          config: {},
+          auth: true,
+          pools: [
+            {'name' => 'pool1', 'size' => 5}
+          ]
+        } }
+
+        it 'returns a token' do
+          create_token "mytoken", "admin", current_time
+          create_vm_for_token "mytoken", "pool1", "vmhostname"
+
+          get "#{prefix}/token/mytoken"
+          expect_json(ok = true, http = 200)
+
+          expect(JSON.parse(last_response.body)['ok']).to eq(true)
+          expect(JSON.parse(last_response.body)['mytoken']['user']).to eq('admin')
+          expect(JSON.parse(last_response.body)['mytoken']['vms']['running']).to include('vmhostname')
+        end
+      end
+    end
+
+    describe 'DELETE /token/:token' do
+      context '(auth not configured)' do
+        let(:config) { { 
+          config: {},
+          auth: false
+        } }
+
+        it 'returns a 404' do
+          delete "#{prefix}/token/this"
+          expect_json(ok = false, http = 404)
+        end
+      end
+
+      context '(auth configured)' do
+        let(:config) {
+          {
+            config: {},
+            auth: {
+              'provider' => 'dummy'
+            }
+          }
+        }
+
+        it 'returns a 401 if not authed' do
+          delete "#{prefix}/token/this"
+          expect_json(ok = false, http = 401)
+        end
+
+        it 'deletes a token if authed' do
+          create_token("mytoken", "admin", current_time)
+          authorize 'admin', 's3cr3t'
+
+          delete "#{prefix}/token/mytoken"
+          expect_json(ok = true, http = 200)
+        end
+
+        it 'fails if token does not exist' do
+          authorize 'admin', 's3cr3t'
+
+          delete "#{prefix}/token/missingtoken"
+          expect_json(ok = false, http = 401)  # TODO: should this be 404?
+        end
+      end
+    end
+  end
+end

spec/integration/api/v2/vm_hostname_spec.rb

@@ -0,0 +1,352 @@
+require 'spec_helper'
+require 'rack/test'
+
+def has_set_tag?(vm, tag, value)
+  value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}")
+end
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/vm/:hostname' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 5},
+          {'name' => 'pool2', 'size' => 10}
+        ],
+        alias: { 'poolone' => 'pool1' },
+        auth: false
+      }
+    }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'PUT /vm/:hostname' do
+      it 'allows tags to be set' do
+        create_vm('testhost', redis)
+        put "#{prefix}/vm/testhost", '{"tags":{"tested_by":"rspec"}}'
+        expect_json(ok = true, http = 200)
+
+        expect has_set_tag?('testhost', 'tested_by', 'rspec')
+      end
+
+      it 'skips empty tags' do
+        create_vm('testhost', redis)
+        put "#{prefix}/vm/testhost", '{"tags":{"tested_by":""}}'
+        expect_json(ok = true, http = 200)
+
+        expect !has_set_tag?('testhost', 'tested_by', '')
+      end
+
+      it 'does not set tags if request body format is invalid' do
+        create_vm('testhost', redis)
+        put "#{prefix}/vm/testhost", '{"tags":{"tested"}}'
+        expect_json(ok = false, http = 400)
+
+        expect !has_set_tag?('testhost', 'tested', '')
+      end
+
+      context '(allowed_tags configured)' do
+        it 'fails if specified tag is not in allowed_tags array' do
+          app.settings.set :config,
+             { :config => { 'allowed_tags' => ['created_by', 'project', 'url'] } }
+
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"tags":{"created_by":"rspec","tested_by":"rspec"}}'
+          expect_json(ok = false, http = 400)
+
+          expect !has_set_tag?('testhost', 'tested_by', 'rspec')
+        end
+      end
+
+      context '(tagfilter configured)' do
+        before(:each) do
+          app.settings.set :config, tagfilter: { 'url' => '(.*)\/' }
+        end
+
+        it 'correctly filters tags' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com/something.html"}}'
+          expect_json(ok = true, http = 200)
+
+          expect has_set_tag?('testhost', 'url', 'foo.com')
+        end
+
+        it "doesn't eat tags not matching filter" do
+          create_vm('testhost', redis)
+          put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com"}}'
+          expect_json(ok = true, http = 200)
+
+          expect has_set_tag?('testhost', 'url', 'foo.com')
+        end
+      end
+
+      context '(auth not configured)' do
+        before(:each) do
+          app.settings.set :config, auth: false
+        end
+
+        it 'allows VM lifetime to be modified without a token' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
+          expect_json(ok = true, http = 200)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime'].to_i).to eq(1)
+        end
+
+        it 'does not allow a lifetime to be 0' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"0"}'
+          expect_json(ok = false, http = 400)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to be_nil
+        end
+
+        it 'does not enforce a lifetime' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"20000"}'
+          expect_json(ok = true, http = 200)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to eq("20000")
+        end
+
+        it 'does not allow a lifetime to be initially past config max_lifetime_upper_limit' do
+          app.settings.set :config,
+                           { :config => { 'max_lifetime_upper_limit' => 168 } }
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"200"}'
+          expect_json(ok = false, http = 400)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime']).to be_nil
+        end
+
+#       it 'does not allow a lifetime to be extended past config 168' do
+#         app.settings.set :config,
+#                          { :config => { 'max_lifetime_upper_limit' => 168 } }
+#         create_vm('testhost', redis)
+#
+#         set_vm_data('testhost', "checkout", (Time.now - (69*60*60)), redis)
+#         puts redis.hget("vmpooler__vm__testhost", 'checkout')
+#         put "#{prefix}/vm/testhost", '{"lifetime":"100"}'
+#         expect_json(ok = false, http = 400)
+#
+#         vm = fetch_vm('testhost')
+#         expect(vm['lifetime']).to be_nil
+#       end
+      end
+
+      context '(auth configured)' do
+        before(:each) do
+          app.settings.set :config, auth: true
+        end
+
+        it 'allows VM lifetime to be modified with a token' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"1"}', {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 200)
+
+          vm = fetch_vm('testhost')
+          expect(vm['lifetime'].to_i).to eq(1)
+        end
+
+        it 'does not allows VM lifetime to be modified without a token' do
+          create_vm('testhost', redis)
+
+          put "#{prefix}/vm/testhost", '{"lifetime":"1"}'
+          expect_json(ok = false, http = 401)
+        end
+      end
+    end
+
+    describe 'DELETE /vm/:hostname' do
+      context '(auth not configured)' do
+        it 'does not delete a non-existant VM' do
+          delete "#{prefix}/vm/testhost"
+          expect_json(ok = false, http = 404)
+        end
+
+        it 'deletes an existing VM' do
+          create_running_vm('pool1', 'testhost', redis)
+          expect fetch_vm('testhost')
+
+          delete "#{prefix}/vm/testhost"
+          expect_json(ok = true, http = 200)
+          expect !fetch_vm('testhost')
+        end
+      end
+
+      context '(auth configured)' do
+        before(:each) do
+          app.settings.set :config, auth: true
+        end
+
+        context '(checked-out without token)' do
+          it 'deletes a VM without supplying a token' do
+            create_running_vm('pool1', 'testhost', redis)
+            expect fetch_vm('testhost')
+
+            delete "#{prefix}/vm/testhost"
+            expect_json(ok = true, http = 200)
+            expect !fetch_vm('testhost')
+          end
+        end
+
+        context '(checked-out with token)' do
+          it 'fails to delete a VM without supplying a token' do
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
+            expect fetch_vm('testhost')
+
+            delete "#{prefix}/vm/testhost"
+            expect_json(ok = false, http = 401)
+            expect fetch_vm('testhost')
+          end
+
+          it 'deletes a VM when token is supplied' do
+            create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345')
+            expect fetch_vm('testhost')
+
+            delete "#{prefix}/vm/testhost", "", {
+              'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+            }
+            expect_json(ok = true, http = 200)
+
+            expect !fetch_vm('testhost')
+          end
+        end
+      end
+    end
+
+    describe 'POST /vm/:hostname/snapshot' do
+      context '(auth not configured)' do
+        it 'creates a snapshot' do
+          create_vm('testhost', redis)
+          post "#{prefix}/vm/testhost/snapshot"
+          expect_json(ok = true, http = 202)
+          expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
+        end
+      end
+
+      context '(auth configured)' do
+        before(:each) do
+          app.settings.set :config, auth: true
+        end
+
+        it 'returns a 401 if not authed' do
+          post "#{prefix}/vm/testhost/snapshot"
+          expect_json(ok = false, http = 401)
+          expect !has_vm_snapshot?('testhost', redis)
+        end
+
+        it 'creates a snapshot if authed' do
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
+
+          post "#{prefix}/vm/testhost/snapshot", "", {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 202)
+          expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32)
+          expect has_vm_snapshot?('testhost', redis)
+        end
+      end
+    end
+
+    describe 'POST /vm/:hostname/snapshot/:snapshot' do
+      context '(auth not configured)' do
+        it 'reverts to a snapshot' do
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
+
+          post "#{prefix}/vm/testhost/snapshot/testsnapshot"
+          expect_json(ok = true, http = 202)
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
+        end
+
+        it 'fails if the specified snapshot does not exist' do
+          create_vm('testhost', redis)
+
+          post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = false, http = 404)
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
+        end
+      end
+
+      context '(auth configured)' do
+        before(:each) do
+          app.settings.set :config, auth: true
+        end
+
+        it 'returns a 401 if not authed' do
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
+
+          post "#{prefix}/vm/testhost/snapshot/testsnapshot"
+          expect_json(ok = false, http = 401)
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
+        end
+
+        it 'fails if authed and the specified snapshot does not exist' do
+          create_vm('testhost', redis)
+
+          post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = false, http = 404)
+          expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
+        end
+
+        it 'reverts to a snapshot if authed' do
+          create_vm('testhost', redis)
+          snapshot_vm('testhost', 'testsnapshot', redis)
+
+          post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 202)
+          expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot')
+        end
+      end
+    end
+  end
+end

spec/integration/api/v2/vm_spec.rb

@@ -0,0 +1,418 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/vm' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2
+        },
+        providers: {
+          vsphere: {'domain' => 'one.example.com'},
+          gce: {'domain' => 'two.example.com'},
+          foo: {'domain' => 'three.example.com'}
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere'},
+          {'name' => 'pool2', 'size' => 10, 'provider' => 'gce'},
+          {'name' => 'pool3', 'size' => 10, 'provider' => 'foo'}
+        ],
+        statsd: { 'prefix' => 'stats_prefix'},
+        alias: { 'poolone' => ['pool1'] },
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone', 'genericpool' ]
+      }
+    }
+    let(:current_time) { Time.now }
+    let(:vmname) { 'abcdefghijkl' }
+    let(:checkoutlock) { Mutex.new }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'GET /vm/:hostname' do
+      it 'returns correct information on a running vm' do
+        create_running_vm 'pool1', vmname, redis
+        expect(TCPSocket).to receive(:gethostbyname).and_raise(RuntimeError)
+        get "#{prefix}/vm/#{vmname}"
+        expect_json(ok = true, http = 200)
+        response_body = (JSON.parse(last_response.body)[vmname])
+
+        expect(response_body["template"]).to eq("pool1")
+        expect(response_body["lifetime"]).to eq(0)
+        expect(response_body["running"]).to be >= 0
+        expect(response_body["remaining"]).to be <= 0
+        expect(response_body["start_time"]).to eq(current_time.to_datetime.rfc3339)
+        expect(response_body["end_time"]).to eq(current_time.to_datetime.rfc3339)
+        expect(response_body["state"]).to eq("running")
+        expect(response_body["ip"]).to eq("")
+        expect(response_body["host"]).to eq("host1")
+      end
+    end
+
+    describe 'POST /vm' do
+
+      let(:socket) { double('socket') }
+      it 'returns a single VM' do
+        create_ready_vm 'pool1', vmname, redis
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: "#{vmname}.one.example.com"
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns a single VM for an alias' do
+        create_ready_vm 'pool1', vmname, redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"poolone":"1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          poolone: {
+            hostname: "#{vmname}.one.example.com"
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistant pools' do
+        post "#{prefix}/vm", '{"poolpoolpool":"1"}'
+        expect_json(ok = false, http = 404)
+      end
+
+      it 'returns 503 for empty pool when aliases are not defined' do
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
+
+        create_ready_vm 'pool1', vmname, redis
+        post "#{prefix}/vm/pool1"
+        post "#{prefix}/vm/pool1"
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns 503 for empty pool referenced by alias' do
+        create_ready_vm 'pool2', vmname, redis
+        post "#{prefix}/vm/poolone"
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns multiple VMs' do
+        create_ready_vm 'pool1', vmname, redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: "#{vmname}.one.example.com"
+          },
+          pool2: {
+            hostname: 'qrstuvwxyz012345.two.example.com'
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns multiple VMs even when multiple instances from the same pool are requested' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+          },
+          pool2: {
+            hostname: 'qrstuvwxyz012345'
+          }
+        }
+
+        result = JSON.parse(last_response.body)
+        expect(result['ok']).to eq(true)
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345.two.example.com')
+
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+          },
+          pool2: {
+            hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ]
+          }
+        }
+
+        result = JSON.parse(last_response.body)
+        expect(result['ok']).to eq(true)
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.two.example.com', '2qrstuvwxyz012345.two.example.com', '3qrstuvwxyz012345.two.example.com')
+
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'returns VMs from multiple backend pools requested by an alias' do
+        Vmpooler::API.settings.config[:alias]['genericpool'] = ['pool1', 'pool2', 'pool3']
+
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool3', '1qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"genericpool":"3"}'
+
+        expected = {
+          ok: true,
+          genericpool: {
+            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop', '1qrstuvwxyz012345' ]
+          }
+        }
+
+        result = JSON.parse(last_response.body)
+        expect(result['ok']).to eq(true)
+        expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.two.example.com', '1qrstuvwxyz012345.three.example.com')
+
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'returns the first VM that was moved to the ready state when checking out a VM' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '3abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1"}'
+
+        expected = {
+          ok: true,
+          "pool1": {
+            "hostname": "1abcdefghijklmnop.one.example.com"
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails when not all requested vms can be allocated' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
+      end
+
+      it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
+      end
+
+      it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}'
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop', redis)).to eq(true)
+      end
+
+      context '(auth not configured)' do
+        it 'does not extend VM lifetime if auth token is provided' do
+          app.settings.set :config, auth: false
+
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm", '{"pool1":"1"}', {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 200)
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop.one.example.com'
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime']).to be_nil
+        end
+      end
+
+      context '(auth configured)' do
+        it 'extends VM lifetime if auth token is provided' do
+          app.settings.set :config, auth: true
+
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm", '{"pool1":"1"}', {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 200)
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop.one.example.com'
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime'].to_i).to eq(2)
+        end
+
+        it 'does not extend VM lifetime if auth token is not provided' do
+          app.settings.set :config, auth: true
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm", '{"pool1":"1"}'
+          expect_json(ok = true, http = 200)
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop.one.example.com'
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime']).to be_nil
+        end
+      end
+    end
+  end
+end

spec/integration/api/v2/vm_template_spec.rb

@@ -0,0 +1,327 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V2 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  # Added to ensure no leakage in rack state from previous tests.
+  # Removes all routes, filters, middleware and extension hooks from the current class
+  # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method 
+  before(:each) do
+    app.reset!
+  end
+
+  describe '/vm/:template' do
+    let(:prefix) { '/api/v2' }
+    let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+        },
+        providers: { vsphere: {} },
+        pools: [
+          {'name' => 'pool1', 'size' => 5},
+          {'name' => 'pool2', 'size' => 10},
+          {'name' => 'poolone', 'size' => 1}
+        ],
+        statsd: { 'prefix' => 'stats_prefix'},
+        alias: { 'poolone' => 'pool1' },
+        pool_names: [ 'pool1', 'pool2', 'poolone' ]
+      }
+    }
+
+    let(:current_time) { Time.now }
+    let(:socket) { double('socket') }
+    let(:checkoutlock) { Mutex.new }
+
+    before(:each) do
+      expect(app).to receive(:run!).once
+      app.execute([:api], config, redis, metrics, nil)
+      app.settings.set :config, auth: false
+      app.settings.set :checkoutlock, checkoutlock
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /vm/:template' do
+      it 'returns a single VM' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1", ''
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: 'abcdefghijklmnop'
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns a single VM for an alias' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/poolone", ''
+
+        expected = {
+          ok: true,
+          poolone: {
+            hostname: 'abcdefghijklmnop'
+          }
+        }
+        expect_json(ok = true, http = 200)
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistant pools' do
+        post "#{prefix}/vm/poolpoolpool", ''
+        expect_json(ok = false, http = 404)
+      end
+
+      it 'returns 503 for empty pool when aliases are not defined' do
+        app.settings.config.delete(:alias)
+        app.settings.config[:pool_names] = ['pool1', 'pool2']
+
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        post "#{prefix}/vm/pool1"
+        post "#{prefix}/vm/pool1"
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns 503 for empty pool referenced by alias' do
+        create_ready_vm 'pool2', 'abcdefghijklmnop', redis
+        post "#{prefix}/vm/poolone"
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns multiple VMs' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', 'qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1+pool2", ''
+        expect_json(ok = true, http = 200)
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: 'abcdefghijklmnop'
+          },
+          pool2: {
+            hostname: 'qrstuvwxyz012345'
+          }
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns multiple VMs even when multiple instances from multiple pools are requested' do
+        create_ready_vm 'pool1', '1abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '2abcdefghijklmnop', redis
+
+        create_ready_vm 'pool2', '1qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '2qrstuvwxyz012345', redis
+        create_ready_vm 'pool2', '3qrstuvwxyz012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
+
+        expected = {
+          ok: true,
+          pool1: {
+            hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ]
+          },
+          pool2: {
+            hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ]
+          }
+        }
+
+        result = JSON.parse(last_response.body)
+        expect(result['ok']).to eq(true)
+        expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop')
+        expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345')
+        expect_json(ok = true, http = 200)
+      end
+
+      it 'fails when not all requested vms can be allocated' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        post "#{prefix}/vm/pool1+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+      end
+
+      it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
+
+        post "#{prefix}/vm/pool1+pool1+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool1', '0123456789012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1+pool1+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool1', '0123456789012345', redis)).to eq(true)
+      end
+
+      it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
+
+        post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+      end
+
+      it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do
+        create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+        create_ready_vm 'pool2', '0123456789012345', redis
+
+        allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+        post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", ''
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        expect_json(ok = false, http = 503)
+
+        expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true)
+        expect(pool_has_ready_vm?('pool2', '0123456789012345', redis)).to eq(true)
+      end
+
+      context '(auth not configured)' do
+        it 'does not extend VM lifetime if auth token is provided' do
+          app.settings.set :config, auth: false
+
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm/pool1", '', {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 200)
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop'
+            }
+          }
+
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime']).to be_nil
+        end
+      end
+
+      context '(auth configured)' do
+        it 'extends VM lifetime if auth token is provided' do
+          app.settings.set :config, auth: true
+
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm/pool1", '', {
+            'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+          }
+          expect_json(ok = true, http = 200)
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop'
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime'].to_i).to eq(2)
+        end
+
+        it 'does not extend VM lifetime if auth token is not provided' do
+          app.settings.set :config, auth: true
+          create_ready_vm 'pool1', 'abcdefghijklmnop', redis
+
+          allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket)
+
+          post "#{prefix}/vm/pool1", ''
+
+          expected = {
+            ok: true,
+            pool1: {
+              hostname: 'abcdefghijklmnop'
+            }
+          }
+          expect_json(ok = true, http = 200)
+
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+
+          vm = fetch_vm('abcdefghijklmnop')
+          expect(vm['lifetime']).to be_nil
+        end
+      end
+    end
+  end
+end

spec/unit/api/helpers_spec.rb

@@ -16,12 +16,12 @@ describe Vmpooler::API::Helpers do
 
   describe '#hostname_shorten' do
     [
-        ['example.com', 'not-example.com', 'example.com'],
+        ['example.com', 'not-example.com', 'example'],
-        ['example.com', 'example.com', 'example.com'],
+        ['example.com', 'example.com', 'example'],
         ['sub.example.com', 'example.com', 'sub'],
         ['adjective-noun.example.com', 'example.com', 'adjective-noun'],
         ['abc123.example.com', 'example.com', 'abc123'],
-        ['example.com', nil, 'example.com']
+        ['example.com', nil, 'example']
     ].each do |hostname, domain, expected|
       it { expect(subject.hostname_shorten(hostname, domain)).to eq expected }
     end

spec/unit/pool_manager_spec.rb

@@ -869,6 +869,58 @@ EOT
     end
   end
 
+  describe '#find_unique_hostname' do
+    # it should return the hostname
+    before do
+      allow(subject).to receive(:generate_and_check_hostname).and_return(["spicy-proton", true])
+      allow(subject).to receive(:get_provider_for_pool).and_return(provider)
+    end
+
+    context 'with a setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:providers:
+  :mock_provider:
+    skip_dns_check_before_creating_vm: true
+:pools:
+  - name: '#{pool}'
+    size: 1
+        EOT
+        )
+      }
+      it 'should skip the dns check' do
+        #method is skipped
+        expect(subject).not_to receive(:check_dns_available)
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+    end
+    context 'without the setting to skip dns check' do
+      let(:config) {
+        YAML.load(<<-EOT
+---
+:providers:
+  :mock_provider:
+:pools:
+  - name: '#{pool}'
+    size: 1
+        EOT
+        )
+      }
+      it 'should run the dns check and pass' do
+        #method is skipped
+        expect(subject).to receive(:check_dns_available).and_return(["1.1.1.1",true])
+        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
+      end
+
+      it 'should run the dns check and fail' do
+        #method is skipped
+        allow(subject).to receive(:check_dns_available).and_return(["1.1.1.1",false]).exactly(3).times
+        expect{subject.find_unique_hostname(pool)}.to raise_error(RuntimeError)
+      end
+    end
+  end
+
   describe '#_clone_vm' do
     let (:pool_object) { { 'name' => pool } }
     let (:redis_ttl) { 1 }
@@ -2708,6 +2760,7 @@ EOT
       allow(subject).to receive(:check_snapshot_queue)
       allow(subject).to receive(:check_pool)
       allow(subject).to receive(:check_ondemand_requests)
+      allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
 
       allow(logger).to receive(:log)
     end

spec/unit/util/parsing_spec.rb

@@ -0,0 +1,104 @@
+require 'spec_helper'
+
+describe 'Parser' do
+  let(:pool) { 'pool1' }
+  subject { Vmpooler::Parsing }
+  describe '.get_domain_for_pool' do
+    let(:provider_name) { 'mock_provider' }
+    context 'No provider is set' do
+      let(:config) { YAML.load(<<~EOT
+          ---
+          :config:
+          :providers:
+            :mock_provider:
+          :pools:
+            - name: '#{pool}'
+              size: 1
+      EOT
+      )}
+
+      it 'should return nil' do
+        result = subject.get_domain_for_pool(config, pool)
+        expect(result).to be_nil
+      end
+    end
+    context 'Provider is vsphere by default' do
+      let(:config) { YAML.load(<<~EOT
+          ---
+          :config:
+          :providers:
+            :vsphere:
+              domain: myown.com
+          :pools:
+            - name: '#{pool}'
+              size: 1
+      EOT
+      )}
+
+      it 'should return the domain set for vsphere' do
+        result = subject.get_domain_for_pool(config, pool)
+        expect(result).to eq('myown.com')
+      end
+    end
+    context 'No domain is set' do
+      let(:config) { YAML.load(<<~EOT
+          ---
+          :config:
+          :providers:
+            :mock_provider:
+          :pools:
+            - name: '#{pool}'
+              size: 1
+              provider: #{provider_name}
+      EOT
+      )}
+
+      it 'should return nil' do
+        result = subject.get_domain_for_pool(config, pool)
+        expect(result).to be_nil
+      end
+    end
+
+    context 'Only a global domain is set' do
+      let(:config) { YAML.load(<<~EOT
+          ---
+          :config:
+            domain: example.com
+          :providers:
+            :mock_provider:
+          :pools:
+            - name: '#{pool}'
+              size: 1
+              provider: #{provider_name}
+      EOT
+      )}
+
+      it 'should return the domain set in the config section' do
+        result = subject.get_domain_for_pool(config, pool)
+        expect(result).to_not be_nil
+        expect(result).to eq('example.com')
+      end
+    end
+
+    context 'A provider specified a domain to use' do
+      let(:config) { YAML.load(<<~EOT
+          ---
+          :config:
+          :providers:
+            :mock_provider:
+              domain: m.example.com
+          :pools:
+            - name: '#{pool}'
+              size: 1
+              provider: #{provider_name}
+      EOT
+      )}
+
+      it 'should return the domain set in the config section' do
+        result = subject.get_domain_for_pool(config, pool)
+        expect(result).to_not be_nil
+        expect(result).to eq('m.example.com')
+      end
+    end
+  end
+end

vmpooler.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'nokogiri', '~> 1.10'
   s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.20.1'
   s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.19.2'
-  s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.19.3'
+  s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.19.4'
   s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.21.3'
   s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.19.3'
   s.add_dependency 'opentelemetry-resource_detectors', '= 0.19.1'
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rack-test', '>= 0.6'
   s.add_development_dependency 'rspec', '>= 3.2'
-  s.add_development_dependency 'rubocop', '~> 1.1.0'
+  s.add_development_dependency 'rubocop', '~> 1.28.2'
   s.add_development_dependency 'simplecov', '>= 0.11.2'
   s.add_development_dependency 'thor', '~> 1.0', '>= 1.0.1'
   s.add_development_dependency 'yarjuf', '>= 2.0'

vmpooler.yaml.example

@@ -28,6 +28,12 @@
 #     Expects an array of strings specifying the allowed items by name
 #     (optional; default: nil)
 #
+#   - skip_dns_check_before_creating_vm
+#     Setting this configuration parameter in a provider will make vmpooler skip the check it normally does for a DNS
+#     record. The normal behavior is to check DNS and if a record already exists (conflict) to re-generate a new hostname.
+#     By using this configuration parameter, it will skip the check and continue with the same conflicting name. This is
+#     useful for providers that can handle that case, for instance by replacing the existing DNS record with a new one.
+#
 # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
 # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
 # the provider to use.
@@ -440,7 +446,10 @@
 #     If set, restricts tags to those specified in this array.
 #
 #   - domain
-#     If set, returns a top-level 'domain' JSON key in POST requests
+#     If set, returns a top-level 'domain' JSON key in POST requests (v1).
+#     in the v2 API, it concatenates the hostname to include the domain, either set here or at the provider level
+#     With the introduction of vmpooler-provider-gce, domains can now be set at the provider level. The
+#     domain key is only retained in v1 for backwards compatibility and will be removed in the future.
 #
 #   - prefix
 #     If set, prefixes all created VMs with this string.  This should include