From 6b9eb2369f09af4d7814ee86268a97abe43c38a6 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Thu, 30 Dec 2021 14:04:34 -0500 Subject: [PATCH 01/20] Connect domain settings to pools, create v2 API --- .rubocop_todo.yml | 22 + docs/{API.md => API-v1.md} | 0 docs/API-v2.md | 936 ++++++++++++++++++++ docs/configuration.md | 5 +- lib/vmpooler/api.rb | 3 +- lib/vmpooler/api/helpers.rb | 2 + lib/vmpooler/api/v2.rb | 373 ++++++++ lib/vmpooler/pool_manager.rb | 54 +- spec/integration/api/v2/config_spec.rb | 402 +++++++++ spec/integration/api/v2/ondemandvm_spec.rb | 388 ++++++++ spec/integration/api/v2/poolreset.rb | 120 +++ spec/integration/api/v2/status_spec.rb | 247 ++++++ spec/integration/api/v2/token_spec.rb | 210 +++++ spec/integration/api/v2/vm_hostname_spec.rb | 352 ++++++++ spec/integration/api/v2/vm_spec.rb | 444 ++++++++++ spec/integration/api/v2/vm_template_spec.rb | 326 +++++++ spec/unit/api/helpers_spec.rb | 6 +- spec/unit/pool_manager_spec.rb | 88 ++ vmpooler.yaml.example | 5 +- 19 files changed, 3964 insertions(+), 19 deletions(-) rename docs/{API.md => API-v1.md} (100%) create mode 100644 docs/API-v2.md create mode 100644 lib/vmpooler/api/v2.rb create mode 100644 spec/integration/api/v2/config_spec.rb create mode 100644 spec/integration/api/v2/ondemandvm_spec.rb create mode 100644 spec/integration/api/v2/poolreset.rb create mode 100644 spec/integration/api/v2/status_spec.rb create mode 100644 spec/integration/api/v2/token_spec.rb create mode 100644 spec/integration/api/v2/vm_hostname_spec.rb create mode 100644 spec/integration/api/v2/vm_spec.rb create mode 100644 spec/integration/api/v2/vm_template_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 6105e46..4f0d4d3 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -13,6 +13,7 @@ Layout/ParameterAlignment: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 9 # Cop supports --auto-correct. @@ -22,12 +23,14 @@ Layout/CaseIndentation: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. Layout/ClosingParenthesisIndentation: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -59,6 +62,7 @@ Layout/FirstHashElementIndentation: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -66,6 +70,7 @@ Layout/FirstHashElementIndentation: Layout/IndentationWidth: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -74,6 +79,7 @@ Layout/IndentationWidth: Layout/MultilineMethodCallBraceLayout: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -88,6 +94,7 @@ Layout/SpaceAroundEqualsInParameterDefault: Layout/SpaceAroundKeyword: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -95,6 +102,7 @@ Layout/SpaceAroundKeyword: Layout/SpaceAroundOperators: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 8 # Cop supports --auto-correct. @@ -110,6 +118,7 @@ Layout/SpaceInsideHashLiteralBraces: Layout/SpaceInsideParens: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 2 # Configuration parameters: AllowSafeAssignment. @@ -117,6 +126,7 @@ Lint/AssignmentInCondition: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 2 Lint/SuppressedException: @@ -149,6 +159,7 @@ Style/AndOr: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 Style/CaseEquality: @@ -170,6 +181,7 @@ Style/HashSyntax: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 4 # Cop supports --auto-correct. @@ -178,6 +190,7 @@ Style/IfUnlessModifier: Exclude: - 'lib/vmpooler/api/helpers.rb' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 3 # Cop supports --auto-correct. @@ -186,12 +199,14 @@ Style/IfUnlessModifier: Style/NegatedIf: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 3 # Cop supports --auto-correct. Style/Not: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -201,12 +216,14 @@ Style/NumericPredicate: Exclude: - 'spec/**/*' - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 2 # Cop supports --auto-correct. Style/ParallelAssignment: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -214,12 +231,14 @@ Style/ParallelAssignment: Style/ParenthesesAroundCondition: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 2 # Cop supports --auto-correct. Style/PerlBackrefs: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist. @@ -236,6 +255,7 @@ Naming/PredicateName: Style/RedundantParentheses: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 2 # Cop supports --auto-correct. @@ -257,6 +277,7 @@ Style/RedundantSelf: Style/StringLiterals: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -272,6 +293,7 @@ Style/TernaryParentheses: Naming/VariableName: Exclude: - 'lib/vmpooler/api/v1.rb' + - 'lib/vmpooler/api/v2.rb' # Offense count: 1 # Cop supports --auto-correct. diff --git a/docs/API.md b/docs/API-v1.md similarity index 100% rename from docs/API.md rename to docs/API-v1.md diff --git a/docs/API-v2.md b/docs/API-v2.md new file mode 100644 index 0000000..5b73230 --- /dev/null +++ b/docs/API-v2.md @@ -0,0 +1,936 @@ +# Table of contents +1. [API](#API) +2. [Token operations](#token) +3. [VM operations](#vmops) +4. [Add disks](#adddisks) +5. [VM snapshots](#vmsnapshots) +6. [Status and metrics](#statusmetrics) +7. [Pool configuration](#poolconfig) +8. [Ondemand VM provisioning](#ondemandvm) + +### API + +vmpooler provides a REST API for VM management. The following examples use `curl` for communication. + +#### Token operations + +Token-based authentication can be used when requesting or modifying VMs. The `/token` route can be used to create, query, or delete tokens. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations. + +##### GET /token + +Get a list of issued tokens. + +Return codes: +* 200 OK +* 401 when not authorized +* 404 when config:auth not found or other error + +``` +$ curl -u jdoe --url vmpooler.example.com/api/v2/token +Enter host password for user 'jdoe': +``` +```json +{ + "ok": true, + "utpg2i2xswor6h8ttjhu3d47z53yy47y": { + "created": "2015-04-28 19:17:47 -0700" + } +} +``` + +##### POST /token + +Generate a new authentication token. + +Return codes: +* 200 OK +* 401 when not authorized +* 404 when config:auth not found + +``` +$ curl -X POST -u jdoe --url vmpooler.example.com/api/v2/token +Enter host password for user 'jdoe': +``` +```json +{ + "ok": true, + "token": "utpg2i2xswor6h8ttjhu3d47z53yy47y" +} +``` + +##### GET /token/<token> + +Get information about an existing token (including associated VMs). + +Return codes: +* 200 OK +* 404 when config:auth or token not found + +``` +$ curl --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y +``` +```json +{ + "ok": true, + "utpg2i2xswor6h8ttjhu3d47z53yy47y": { + "user": "jdoe", + "created": "2015-04-28 19:17:47 -0700", + "last": "2015-11-04 12:28:37 -0700", + "vms": { + "running": [ + "dqs4914g2wjyy5w", + "hul7ib0ssr0f4o0" + ] + } + } +} +``` + +##### DELETE /token/<token> + +Delete an authentication token. + +Return codes: +* 200 OK +* 401 when not authorized +* 404 when config:auth not found + +``` +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y +Enter host password for user 'jdoe': +``` +```json +{ + "ok": true +} +``` + +#### VM operations + +##### GET /vm + +Retrieve a list of available VM pools. + +Return codes: +* 200 OK + +``` +$ curl --url vmpooler.example.com/api/v2/vm +``` +```json +[ + "debian-7-i386", + "debian-7-x86_64" +] +``` + +##### POST /vm + +Useful for batch operations; post JSON (see format below), get back allocated VMs. + +If an authentication store is configured, an authentication token supplied via the `X-AUTH-TOKEN` HTTP header will modify a VM's default lifetime. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information. + +Return codes: +* 200 OK +* 404 when sending invalid JSON in the request body or requesting an invalid VM pool name +* 503 when the vm failed to allocate a vm, or the pool is empty + +``` +$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v2/vm +``` +```json +{ + "ok": true, + "debian-7-i386": { + "hostname": [ + "o41xtodlvnvu5cw.example.com", + "khirruvwfjlmx3y.example.com" + ] + }, + "debian-7-x86_64": { + "hostname": "y91qbrpbfj6d13q.example.com" + }, + "domain": "example.com" +} +``` + +**NOTE: Returns either all requested VMs or no VMs.** + +##### POST /vm/<pool> + +Check-out a VM or VMs. + +Return codes: +* 200 OK +* 404 when sending invalid JSON in the request body or requesting an invalid VM pool name +* 503 when the vm failed to allocate a vm, or the pool is empty + +``` +$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386 +``` +```json +{ + "ok": true, + "debian-7-i386": { + "hostname": "fq6qlpjlsskycq6.example.com" + }, + "domain": "example.com" +} +``` + +Multiple VMs can be requested by using multiple query parameters in the URL: + +``` +$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386+debian-7-i386+debian-7-x86_64 +``` + +```json +{ + "ok": true, + "debian-7-i386": { + "hostname": [ + "sc0o4xqtodlul5w.example.com", + "4m4dkhqiufnjmxy.example.com" + ] + }, + "debian-7-x86_64": { + "hostname": "zb91y9qbrbf6d3q.example.com" + }, + "domain": "example.com" +} +``` + +**NOTE: Returns either all requested VMs or no VMs.** + +##### GET /vm/<hostname> + +Query metadata information for a checked-out VM. + +Return codes: +* 200 OK +* 404 when requesting an invalid VM hostname + +``` +$ curl --url vmpooler.example.com/api/v2/vm/pxpmtoonx7fiqg6 +``` +```json +{ + "ok": true, + "pxpmtoonx7fiqg6": { + "template": "centos-6-x86_64", + "lifetime": 12, + "running": 3, + "remaining": 9, + "state": "running", + "tags": { + "department": "engineering", + "user": "jdoe" + }, + "ip": "192.168.0.1", + "domain": "example.com", + "host": "host1.example.com", + "migrated": "true" + } +} +``` + +##### PUT /vm/<hostname> + +Modify a checked-out VM. + +The following are valid PUT parameters and their required data structures: + +parameter | description | required structure +--------- | ----------- | ------------------ +*lifetime* | VM TTL (in hours) | integer +*tags* | free-form VM tagging | hash + +Any modifications can be verified using the [GET /vm/<hostname>](#get-vmhostname) endpoint. + +If an authentication store is configured, an authentication token is required (via the `X-AUTH-TOKEN` HTTP header) to access this route. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information. + +Return codes: +* 200 OK +* 401 when you need an auth token +* 404 when requesting an invalid VM hostname +* 400 when supplied PUT parameters fail validation + +``` +$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +``` +```json +{ + "ok": true +} +``` + +``` +$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +``` +```json +{ + "ok": true +} +``` + +##### DELETE /vm/<hostname> + +Schedule a checked-out VM for deletion. + +Return codes: +* 200 OK +* 401 when you need an auth token +* 404 when requesting an invalid VM hostname + +``` +$ curl -X DELETE --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +``` +```json +{ + "ok": true +} +``` + +#### Adding additional disk(s) + +##### POST /vm/<hostname>/disk/<size> + +Add an additional disk to a running VM. + +Return codes: +* 202 OK +* 401 when you need an auth token +* 404 when requesting an invalid VM hostname or size is not an integer + +```` +$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/disk/8 +```` +````json +{ + "ok": true, + "fq6qlpjlsskycq6": { + "disk": "+8gb" + } +} +```` + +Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/` query: + +```` +$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +```` +````json +{ + "ok": true, + "fq6qlpjlsskycq6": { + "template": "debian-7-x86_64", + "lifetime": 2, + "running": 0.08, + "state": "running", + "disk": [ + "+8gb" + ], + "domain": "delivery.puppetlabs.net" + } +} + +```` + +#### VM snapshots + +##### POST /vm/<hostname>/snapshot + +Create a snapshot of a running VM. + +Return codes: +* 202 OK +* 401 when you need an auth token +* 404 when requesting an invalid VM hostname + +```` +$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot +```` +````json +{ + "ok": true, + "fq6qlpjlsskycq6": { + "snapshot": "n4eb4kdtp7rwv4x158366vd9jhac8btq" + } +} +```` + +Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/` query: + +```` +$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +```` +````json +{ + "ok": true, + "fq6qlpjlsskycq6": { + "template": "debian-7-x86_64", + "lifetime": 2, + "running": 0.08, + "state": "running", + "snapshots": [ + "n4eb4kdtp7rwv4x158366vd9jhac8btq" + ], + "domain": "delivery.puppetlabs.net" + } +} +```` + +##### POST /vm/<hostname>/snapshot/<snapshot> + +Revert a VM back to a snapshot. + +Return codes: +* 202 OK +* 401 when you need an auth token +* 404 when requesting an invalid VM hostname or snapshot is not valid + +```` +$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq +```` +````json +{ + "ok": true +} +```` + +#### Status and metrics + +##### GET /status + +A "live" status endpoint, representing the current state of the service. + +``` +$ curl --url vmpooler.example.com/api/v2/status +``` +```json +{ + "capacity": { + "current": 716, + "total": 717, + "percent": 99.9 + }, + "clone": { + "duration": { + "average": 8.8, + "min": 2.79, + "max": 69.76 + }, + "count": { + "total": 1779 + } + }, + "queue": { + "pending": 1, + "cloning": 0, + "booting": 1, + "ready": 716, + "running": 142, + "completed": 0, + "total": 859 + }, + "status": { + "ok": true, + "message": "Battle station fully armed and operational." + } +} +``` + +If there are empty pools, the "status" section will convey this: + +```json + "status": { + "ok": false, + "message": "Found 2 empty pools.", + "empty": [ + "centos-6-x86_64", + "debian-7-x86_64" + ] + } +``` + +The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "status". +If the query parameter 'view' is provided, it will be used to select which top level +element to compute and return. Select them by specifying which one you want in a comma +separated list. +For example `vmpooler.example.com/api/v2/status?view=capacity,boot` + +##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]] + +Returns a summary, or report, for the timespan between `from` and `to` (inclusive) +parameters. The response includes both an overall and daily view of tracked +metrics, such as boot and cloning durations. + +Any omitted query parameter will default to now/today. A request without any +parameters will result in the current day's summary. + +Return codes: +* 200 OK +* 400 Invalid date format or range + + +``` +$ curl --url vmpooler.example.com/api/v2/summary +``` +```json +{ + "boot": { + "duration": { + "average": 106.6, + "min": 83.09, + "max": 121.06, + "total": 639.36, + }, + "count": { + "average": 6, + "min": 6, + "max": 6, + "total": 6, + } + }, + "clone": { + "duration": { + "average": 4.6, + "min": 2.78, + "max": 8.1, + "total": 63.94, + }, + "count": { + "average": 14, + "min": 14, + "max": 14, + "total": 14, + } + }, + "tag": { + "department": { + "engineering": 14, + "help desk": 10, + "IT": 44, + "total": 68 + }, + "user": { + "arodgers": 54, + "cmatthews": 10, + "jnelson": 4, + "total": 68 + } + }, + "daily": [ + { + "date": "2015-03-11", + "boot": { + "duration": { + "average": 106.6, + "min": 83.09, + "max": 121.06, + "total": 639.36 + }, + "count": { + "total": 6 + } + }, + "clone": { + "duration": { + "average": 4.6, + "min": 2.78, + "max": 8.1, + "total": 63.94 + }, + "count": { + "total": 14 + } + }, + "tag": { + "department": { + "engineering": 14, + "help desk": 10, + "IT": 44, + "total": 68 + }, + "user": { + "arodgers": 54, + "cmatthews": 10, + "jnelson": 4, + "total": 68 + } + } + } + ] +} +``` + + +``` +$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v2/summary +``` +```json +{ + "boot": {...}, + "clone": {...}, + "daily": [ + { + "date": "2015-03-10", + "boot": { + "duration": { + "average": 0, + "min": 0, + "max": 0, + "total": 0 + }, + "count": { + "total": 0 + } + }, + "clone": { + "duration": { + "average": 0, + "min": 0, + "max": 0, + "total": 0 + }, + "count": { + "total": 0 + } + }, + "tag": { } + }, + { + "date": "2015-03-11", + "boot": { + "duration": { + "average": 106.6, + "min": 83.09, + "max": 121.06, + "total": 639.36 + }, + "count": { + "total": 6 + } + }, + "clone": { + "duration": { + "average": 4.6, + "min": 2.78, + "max": 8.1, + "total": 63.94 + }, + "count": { + "total": 14 + } + }, + "tag": { } + } + ] +} +``` + +You can also query only the specific top level section you want by including it after `summary/`. +The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v2/summary/boot/`. +You can further drill-down the data by specifying the second level parameter to query eg +`vmpooler.example.com/api/v2/summary/tag/created_by` + +##### GET /poolstat?pool=FOO + +For parameter `pool`, containing a comma separated list of pool names to query, this endpoint returns +each of the pool's ready, max and alias information. It can be used to get a fast response for +the required pools instead of using the /status API endpoint + +Return codes +* 200 OK + +``` +$ curl https://vmpooler.example.com/api/v2/poolstat?pool=centos-6-x86_64 +``` +```json +{ + "pools": { + "centos-6-x86_64": { + "ready": 25, + "max": 25, + "alias": [ + "centos-6-64", + "centos-6-amd64" + ] + } + } +} +``` + +##### GET /totalrunning + +Fast endpoint to return the total number of VMs in a 'running' state + +Return codes +* 200 OK + +``` +$ curl https://vmpooler.example.com/api/v2/totalrunning +``` + +```json +{ + + "running": 362 + +} +``` + +#### Managing pool configuration via API + +##### GET /config + +Returns the running pool configuration + +Return codes +* 200 OK +* 400 No configuration found + +``` +$ curl https://vmpooler.example.com/api/v2/config +``` +```json +{ + "pool_configuration": [ + { + "name": "redhat-7-x86_64", + "template": "templates/redhat-7.2-x86_64-0.0.3", + "folder": "vmpooler/redhat-7-x86_64", + "datastore": "stor1", + "size": 1, + "datacenter": "dc1", + "provider": "vsphere", + "capacity": 1, + "major": "redhat", + "template_ready": true + } + ], + "status": { + "ok": true + } +} +``` + +Note: to enable poolsize and pooltemplate config endpoints it is necessary to set 'experimental_features: true' in your vmpooler configuration. A 405 is returned when you attempt to interact with these endpoints when this configuration option is not set. + +##### POST /config/poolsize + +Change pool size without having to restart the service. + +All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned + +When a pool size is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned. + +Pool size configuration changes persist through application restarts, and take precedence over a pool size value configured in the pool configuration provided when the application starts. This persistence is dependent on redis. So, if the redis data is lost then the configuration updates revert to those provided at startup at the next application start. + +An authentication token is required in order to change pool configuration when authentication is configured. +Responses: +* 200 - No changes required +* 201 - Changes made on at least one pool with changes requested +* 400 - An invalid configuration was provided causing requested changes to fail +* 404 - An unknown error occurred +* 405 - The endpoint is disabled because experimental features are disabled +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v2/config/poolsize +``` +```json +{ + "ok": true +} +``` + +##### DELETE /config/poolsize/<pool> + +Delete an overridden pool size. This results in the values from VMPooler's config being used. + +Return codes: +* 200 - when nothing was changed but no error occurred +* 201 - size reset successful +* 401 - when not authorized +* 404 - pool does not exist +* 405 - The endpoint is disabled because experimental features are disabled + +``` +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/poolsize/almalinux-8-x86_64 +``` +```json +{ + "ok": true, + "pool_size_before_overrides": 2, + "pool_size_before_reset": 4 +} +``` + +##### POST /config/pooltemplate + +Change the template configured for a pool, and replenish the pool with instances built from the new template. + +All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned + +When a pool template is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned. + +A pool template being updated will cause the following actions, which are logged in vmpooler.log: +* Destroy all instances for the pool template being updated that are in the ready and pending state +* Halt repopulating the pool while creating template deltas for the newly configured template +* Unblock pool population and let the pool replenish with instances based on the newly configured template + +Pool template changes persist through application restarts, and take precedence over a pool template configured in the pool configuration provided when the application starts. This persistence is dependent on redis. As a result, if the redis data is lost then the configuration values revert to those provided at startup at the next application start. + +An authentication token is required in order to change pool configuration when authentication is configured. + +Responses: +* 200 - No changes required +* 201 - Changes made on at least one pool with changes requested +* 400 - An invalid configuration was provided causing requested changes to fail +* 404 - An unknown error occurred +* 405 - The endpoint is disabled because experimental features are disabled +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v2/config/pooltemplate +``` +```json +{ + "ok": true +} +``` + +##### DELETE /config/pooltemplate/<pool> + +Delete an overridden pool template. This results in the values from VMPooler's config being used. + +Return codes: +* 200 - when nothing was changed but no error occurred +* 201 - template reset successful +* 401 - when not authorized +* 404 - pool does not exist +* 405 - The endpoint is disabled because experimental features are disabled + +``` +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/pooltemplate/almalinux-8-x86_64 +``` +```json +{ + "ok": true, + "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2", + "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta" +} +``` + +##### POST /poolreset + +Clear all pending and ready instances in a pool, and deploy replacements + +All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned. + +When a pool reset is requested a 201 status will be returned. + +A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances. + +For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured. + +Responses: +* 201 - Pool reset requested received +* 400 - An invalid configuration was provided causing requested changes to fail +* 404 - An unknown error occurred +* 405 - The endpoint is disabled because experimental features are disabled +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v2/poolreset +``` +```json +{ + "ok": true +} +``` + +#### Ondemand VM operations + +Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated. + +##### POST /ondemandvm + +All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned + +When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status. + +An authentication token is required in order to request instances on demand when authentication is configured. + +Responses: +* 201 - Provisioning request accepted +* 400 - Payload contains invalid JSON and cannot be parsed +* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled +* 403 - Request exceeds the configured per pool maximum +* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred. +* 409 - A request of the matching ID has already been created +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v2/ondemandvm +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a" +} +``` + +##### GET /ondemandvm + +Get the status of an ondemandvm request that has already been posted. + +When the request is ready the ready status will change to 'true'. + +The number of instances pending vs ready will be reflected in the API response. + +Responses: +* 200 - The API request was successful and the status is ok +* 202 - The request is not ready yet +* 404 - The request can not be found, or an unknown error occurred +``` +$ curl https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", + "ready": false, + "debian-7-i386": { + "ready": "3", + "pending": "1" + } +} +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", + "ready": true, + "debian-7-i386": { + "hostname": [ + "vm1", + "vm2", + "vm3", + "vm4" + ] + } +} +``` + +##### DELETE /ondemandvm + +Delete a ondemand request + +Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted. + +Responses: +* 200 - The API request was sucessful. A message will indicate if the request has already been deleted. +* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled +* 404 - The request can not be found, or an unknown error occurred. +``` +$ curl -X DELETE https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +``` +```json +{ + "ok": true +} +``` diff --git a/docs/configuration.md b/docs/configuration.md index 30b333f..0bf331f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,7 +21,10 @@ Path to a the file to use when loading the vmpooler configuration. This is only ### DOMAIN -If set, returns a top-level 'domain' JSON key in POST requests +If set, returns a top-level 'domain' JSON key in POST requests. Note that domains can now be set per-pool +and at the provider level. The domain key in the JSON response will correspond the last vm added to the +response and should not be relied upon as the hostnames of the returned vms now include the domain. The +domain key is only retained now for backwards compatibility and will be removed in the future. (optional) ### REDIS\_SERVER diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index eb856fc..a7eea30 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -3,7 +3,7 @@ module Vmpooler class API < Sinatra::Base # Load API components - %w[helpers dashboard reroute v1 request_logger healthcheck].each do |lib| + %w[helpers dashboard reroute v1 v2 request_logger healthcheck].each do |lib| require "vmpooler/api/#{lib}" end # Load dashboard components @@ -54,6 +54,7 @@ module Vmpooler use Vmpooler::API::Dashboard use Vmpooler::API::Reroute use Vmpooler::API::V1 + use Vmpooler::API::V2 end # Get thee started O WebServer diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 60d1db3..6d1a347 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -184,6 +184,8 @@ module Vmpooler def hostname_shorten(hostname, domain=nil) if domain && hostname =~ /^[\w-]+\.#{domain}$/ hostname = hostname[/[^.]+/] + elsif hostname =~ /^[\w-]+\..+$/ + hostname = hostname[/[^.]+/] end hostname diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb new file mode 100644 index 0000000..9e52d27 --- /dev/null +++ b/lib/vmpooler/api/v2.rb @@ -0,0 +1,373 @@ +# frozen_string_literal: true + +require 'vmpooler/api/v1' +module Vmpooler + class API + class V2 < Vmpooler::API::V1 + api_version = '2' + api_prefix = "/api/v#{api_version}" + + def get_template_aliases(template) + tracer.in_span("Vmpooler::API::V2.#{__method__}") do + result = [] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + result += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + end + result + end + end + + def get_domain_for_pool(poolname) + pool_index = pool_index(pools) + pools[pool_index[poolname]]['domain'] + end + + # Fetch a single vm from a pool + # + # @param [String] template + # The template that the vm should be created from + # + # @return [Tuple] vmname, vmpool, vmtemplate + # Returns a tuple containing the vm's name, the pool it came from, and + # what template was used, if successful. Otherwise the tuple contains. + # nil values. + def fetch_single_vm(template) + tracer.in_span("Vmpooler::API::V2.#{__method__}") do + template_backends = [template] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + template_backends += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + + if weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + selection = pickup.pick + template_backends.delete(selection) + template_backends.unshift(selection) + else + first = template_backends.sample + template_backends.delete(first) + template_backends.unshift(first) + end + end + + checkoutlock.synchronize do + template_backends.each do |template_backend| + vms = backend.smembers("vmpooler__ready__#{template_backend}") + next if vms.empty? + + vms.reverse.each do |vm| + vm_domain = get_domain_for_pool(template_backend) + ready = vm_ready?(vm, vm_domain) + if ready + smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) + if smoved + return [vm, template_backend, template] + else + metrics.increment("checkout.smove.failed.#{template_backend}") + return [nil, nil, nil] + end + else + backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) + metrics.increment("checkout.nonresponsive.#{template_backend}") + end + end + end + [nil, nil, nil] + end + end + end + + # The domain in the result body will be set to the one associated with the + # last vm added. The part of the response is only being retained for + # backwards compatibility as the hostnames are now fqdn's instead of bare + # hostnames. This change is a result of now being able to specify a domain + # per pool. If no vm's in the result had a domain sepcified then the + # domain key will be omitted similar to how it was previously omitted if + # the global option domain wasn't specified. + def atomically_allocate_vms(payload) + tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| + result = { 'ok' => false } + failed = false + vms = [] # vmpool, vmname, vmtemplate + + validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? + + payload.each do |requested, count| + count.to_i.times do |_i| + vmname, vmpool, vmtemplate = fetch_single_vm(requested) + if vmname + account_for_starting_vm(vmpool, vmname) + vms << [vmpool, vmname, vmtemplate] + metrics.increment("checkout.success.#{vmpool}") + update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] + else + failed = true + metrics.increment("checkout.empty.#{requested}") + break + end + end + end + + if failed + vms.each do |(vmpool, vmname, _vmtemplate)| + return_vm_to_ready_state(vmpool, vmname) + end + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V2.atomically_allocate_vms', + 'error.message' => '503 due to failing to allocate one or more vms' + }) + status 503 + else + vm_names = [] + vms.each do |(vmpool, vmname, vmtemplate)| + vmdomain = get_domain_for_pool(vmpool) + if vmdomain + vmfqdn = "#{vmname}.#{vmdomain}" + update_result_hosts(result, vmtemplate, vmfqdn) + vm_names.append(vmfqdn) + result['domain'] = vmdomain + else + update_result_hosts(result, vmtemplate, vmname) + vm_names.append(vmname) + end + end + + span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty? + + result['ok'] = true + end + + result + end + end + + def generate_ondemand_request(payload) + tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| + result = { 'ok': false } + + requested_instances = payload.reject { |k, _v| k == 'request_id' } + if too_many_requested?(requested_instances) + e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" + result['message'] = e_message + status 403 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request', + 'error.message' => "403 due to #{e_message}" + }) + return result + end + + score = Time.now.to_i + request_id = payload['request_id'] + request_id ||= generate_request_id + result['request_id'] = request_id + span.set_attribute('vmpooler.request_id', request_id) + + if backend.exists?("vmpooler__odrequest__#{request_id}") + e_message = "request_id '#{request_id}' has already been created" + result['message'] = e_message + status 409 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request', + 'error.message' => "409 due to #{e_message}" + }) + metrics.increment('ondemandrequest_generate.duplicaterequests') + return result + end + + status 201 + + platforms_with_aliases = [] + requested_instances.each do |poolname, count| + selection = evaluate_template_aliases(poolname, count) + selection.map do |selected_pool, selected_pool_count| + platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" + pool_domain = get_domain_for_pool(selected_pool) + result['domain'] = pool_domain if pool_domain + end + end + platforms_string = platforms_with_aliases.join(',') + + return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) + + backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) + if Vmpooler::API.settings.config[:auth] and has_token? + token_token = request.env['HTTP_X_AUTH_TOKEN'] + token_user = backend.hget("vmpooler__token__#{token_token}", 'user') + backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token) + backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user) + span.set_attribute('enduser.id', token_user) + end + + result[:ok] = true + metrics.increment('ondemandrequest_generate.success') + result + end + end + + # Endpoints that use overridden methods + + post "#{api_prefix}/vm/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.post.vm.checkout') + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment("checkout.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('checkout.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/vm/:template/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.get.vm.template') + + payload = extract_templates_from_query_params(params[:template]) + + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment("checkout.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('checkout.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/ondemandvm/?" do + content_type :json + metrics.increment('http_requests_vm_total.post.ondemand.requestid') + + need_token! if Vmpooler::API.settings.config[:auth] + + result = { 'ok' => false } + + begin + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('ondemandrequest_fail.invalid.unknown') + status 404 + end + rescue JSON::ParserError + span = OpenTelemetry::Trace.current_span + span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/ondemandvm/:template/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.delete.ondemand.template') + + need_token! if Vmpooler::API.settings.config[:auth] + + payload = extract_templates_from_query_params(params[:template]) + + if payload + invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('ondemandrequest_fail.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + # Endpoints that only use bits from the V1 api are called here + # Note that traces will be named based on the route used in the V1 api + # but the http.url trace attribute will still have the actual requested url in it + + delete "#{api_prefix}/*" do + versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") + request.path_info = "/api/v1/#{versionless_path_info}" + call env + end + + get "#{api_prefix}/*" do + versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") + request.path_info = "/api/v1/#{versionless_path_info}" + call env + end + + post "#{api_prefix}/*" do + versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") + request.path_info = "/api/v1/#{versionless_path_info}" + call env + end + + put "#{api_prefix}/*" do + versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") + request.path_info = "/api/v1/#{versionless_path_info}" + call env + end + end + end +end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 2ca99af..2b9a07f 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -60,6 +60,7 @@ module Vmpooler to_set[k] = pool[k] end to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias') + to_set['domain'] = get_domain_for_pool(pool['name']) redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty? end previously_configured_pools.each do |pool| @@ -361,35 +362,39 @@ module Vmpooler max_hostname_retries = 3 while hostname_retries < max_hostname_retries hostname, hostname_available = generate_and_check_hostname - domain = $config[:config]['domain'] - dns_ip, dns_available = check_dns_available(hostname, domain) + domain = get_domain_for_pool(pool_name) + if domain + fqdn = "#{hostname}.#{domain}" + else + fqdn = hostname + end + dns_ip, dns_available = check_dns_available(fqdn) break if hostname_available && dns_available hostname_retries += 1 if !hostname_available $metrics.increment("errors.duplicatehostname.#{pool_name}") - $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})") + $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})") elsif !dns_available $metrics.increment("errors.staledns.#{pool_name}") - $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS") + $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS") end end - raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available + raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available - hostname + fqdn end - def check_dns_available(vm_name, domain = nil) - # Query the DNS for the name we want to create and if it already exists, mark it unavailable - # This protects against stale DNS records - vm_name = "#{vm_name}.#{domain}" if domain + # Query the DNS for the name we want to create and if it already exists, mark it unavailable + # This protects against stale DNS records + def check_dns_available(vm_name) begin dns_ip = Resolv.getaddress(vm_name) rescue Resolv::ResolvError # this is the expected case, swallow the error - # eg "no address for blah-daisy" + # eg "no address for blah-daisy.example.com" return ['', true] end [dns_ip, false] @@ -397,6 +402,7 @@ module Vmpooler def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil) new_vmname = find_unique_hostname(pool_name) + pool_domain = get_domain_for_pool(pool_name) mutex = vm_mutex(new_vmname) mutex.synchronize do @redis.with_metrics do |redis| @@ -406,6 +412,7 @@ module Vmpooler redis.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now) redis.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool. redis.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name) + redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) if pool_domain redis.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id redis.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias redis.exec @@ -658,6 +665,27 @@ module Vmpooler $providers[provider_name] end + # @param pool_name [String] - the name of the pool + # @return [String] - domain name for pool, if set + def get_domain_for_pool(pool_name) + pool = $config[:pools].find { |p| p['name'] == pool_name } + return nil unless pool + + provider_name = pool.fetch('provider', nil) + return nil unless provider_name + + $logger.log('d', "[*] [get_domain_for_pool] #{pool_name}'s provider is #{provider_name}") + if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain'] + domain = config[:providers][provider_name.to_sym]['domain'] + elsif config[:config] && config[:config]['domain'] + domain = config[:config]['domain'] + else + domain = nil + end + + domain + end + def check_disk_queue(maxloop = 0, loop_delay = 5) $logger.log('d', '[*] [disk_manager] starting worker thread') @@ -879,7 +907,7 @@ module Vmpooler loop_count = 1 loop_delay = loop_delay_min provider = get_provider_for_pool(pool['name']) - raise("Could not find provider '#{pool['provider']}") if provider.nil? + raise("Could not find provider '#{pool['provider']}'") if provider.nil? sync_pool_template(pool) loop do @@ -1366,7 +1394,7 @@ module Vmpooler return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options) end - raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil? + raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil? end def check_ondemand_requests(maxloop = 0, diff --git a/spec/integration/api/v2/config_spec.rb b/spec/integration/api/v2/config_spec.rb new file mode 100644 index 0000000..24543dd --- /dev/null +++ b/spec/integration/api/v2/config_spec.rb @@ -0,0 +1,402 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + 'experimental_features' => true + }, + pools: [ + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, + {'name' => 'pool2', 'size' => 10} + ], + pools_at_startup: [ + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, + {'name' => 'pool2', 'size' => 10} + ], + statsd: { 'prefix' => 'stats_prefix'}, + alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] + } + } + + describe '/config/pooltemplate' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + + let(:current_time) { Time.now } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + app.settings.set :config, auth: false + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'DELETE /config/pooltemplate/:pool' do + it 'resets a pool template' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}' + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = true, http = 201) + + expected = { + ok: true, + template_before_reset: 'templates/new_template', + template_before_overrides: 'templates/pool1' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds when the pool has not been overridden' do + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = true, http = 200) + end + + it 'fails on nonexistent pools' do + delete "#{prefix}/config/pooltemplate/poolpoolpool" + expect_json(ok = false, http = 404) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + delete "#{prefix}/config/pooltemplate/pool1" + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + end + + describe 'POST /config/pooltemplate' do + it 'updates a pool template' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails on nonexistent pools' do + post "#{prefix}/config/pooltemplate", '{"poolpoolpool":"templates/newtemplate"}' + expect_json(ok = false, http = 400) + end + + it 'updates multiple pools' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool2":"templates/new_template2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when not all pools exist' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool3":"templates/new_template2"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_templates: ['pool3'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns no changes when the template does not change' do + post "#{prefix}/config/pooltemplate", '{"pool1":"templates/pool1"}' + expect_json(ok = true, http = 200) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a invalid template parameter is provided' do + post "#{prefix}/config/pooltemplate", '{"pool1":"template1"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_templates: ['pool1'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a template starts with /' do + post "#{prefix}/config/pooltemplate", '{"pool1":"/template1"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_templates: ['pool1'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a template ends with /' do + post "#{prefix}/config/pooltemplate", '{"pool1":"template1/"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_templates: ['pool1'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/config/pooltemplate", '{"pool1":"template/template1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + end + + describe 'DELETE /config/poolsize' do + it 'resets a pool size' do + post "#{prefix}/config/poolsize", '{"pool1":"2"}' + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = true, http = 201) + + expected = { + ok: true, + pool_size_before_reset: 2, + pool_size_before_overrides: 5 + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a specified pool does not exist' do + delete "#{prefix}/config/poolsize/pool10" + expect_json(ok = false, http = 404) + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds when a pool has not been overridden' do + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = true, http = 200) + expected = { + ok: true, + pool_size_before_reset: 5, + pool_size_before_overrides: 5 + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + delete "#{prefix}/config/poolsize/pool1" + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + + describe 'POST /config/poolsize' do + it 'changes a pool size' do + post "#{prefix}/config/poolsize", '{"pool1":"2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'changes a pool size for multiple pools' do + post "#{prefix}/config/poolsize", '{"pool1":"2","pool2":"2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a specified pool does not exist' do + post "#{prefix}/config/poolsize", '{"pool10":"2"}' + expect_json(ok = false, http = 400) + expected = { + ok: false, + not_configured: ['pool10'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 200 when no change is required' do + post "#{prefix}/config/poolsize", '{"pool1":"5"}' + expect_json(ok = true, http = 200) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 201 when at least one pool changes' do + post "#{prefix}/config/poolsize", '{"pool1":"5","pool2":"5"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a non-integer value is provided for size' do + post "#{prefix}/config/poolsize", '{"pool1":"four"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + not_configured: ['pool1'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a negative value is provided for size' do + post "#{prefix}/config/poolsize", '{"pool1":"-1"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + not_configured: ['pool1'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/config/poolsize", '{"pool1":"1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + + describe 'POST /config/clonetarget' do + it 'changes the clone target' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'changes a pool size for multiple pools' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1","pool2":"cluster2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a specified pool does not exist' do + post "#{prefix}/config/clonetarget", '{"pool10":"cluster1"}' + expect_json(ok = false, http = 400) + expected = { + ok: false, + bad_templates: ['pool10'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 200 when no change is required' do + post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster"}' + expect_json(ok = true, http = 200) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 201 when at least one pool changes' do + post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster","pool2":"cluster2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + + describe 'GET /config' do + let(:prefix) { '/api/v2' } + + it 'returns pool configuration when set' do + get "#{prefix}/config" + + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result['pool_configuration']).to eq(config[:pools]) + end + end + end +end diff --git a/spec/integration/api/v2/ondemandvm_spec.rb b/spec/integration/api/v2/ondemandvm_spec.rb new file mode 100644 index 0000000..9aeac3a --- /dev/null +++ b/spec/integration/api/v2/ondemandvm_spec.rb @@ -0,0 +1,388 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe '/ondemandvm' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + 'max_ondemand_instances_per_request' => 50, + 'backend_weight' => { + 'compute1' => 5, + 'compute2' => 0 + } + }, + pools: [ + {'name' => 'pool1', 'size' => 0, 'clone_target' => 'compute1'}, + {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute2'}, + {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1'} + ], + alias: { + 'poolone' => ['pool1'], + 'pool2' => ['pool1'] + }, + pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ] + } + } + let(:current_time) { Time.now } + let(:vmname) { 'abcdefghijkl' } + let(:checkoutlock) { Mutex.new } + let(:uuid) { SecureRandom.uuid } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + app.settings.set :config, auth: false + app.settings.set :checkoutlock, checkoutlock + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + config[:pools].each do |pool| + redis.sadd('vmpooler__pools', pool['name']) + end + end + + describe 'POST /ondemandvm' do + + context 'with a configured pool' do + + context 'with no request_id provided in payload' do + before(:each) do + expect(SecureRandom).to receive(:uuid).and_return(uuid) + end + + it 'generates a request_id when none is provided' do + post "#{prefix}/ondemandvm", '{"pool1":"1"}' + expect_json(true, 201) + + expected = { + "ok": true, + "request_id": uuid + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'uses a configured platform to fulfill a ondemand request' do + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + expect_json(true, 201) + expected = { + "ok": true, + "request_id": uuid + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'creates a provisioning request in redis' do + expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1) + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + end + + it 'sets a platform string in redis for the request to indicate selected platforms' do + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1') + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + end + + context 'with a backend of 0 weight' do + before(:each) do + config[:config]['backend_weight']['compute1'] = 0 + end + + it 'sets the platform string in redis for the request to indicate the selected platforms' do + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool1:pool1:1') + post "#{prefix}/ondemandvm", '{"pool1":"1"}' + end + end + + it 'sets the platform string in redis for the request to indicate the selected platforms using weight' do + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'pool2:pool1:1') + post "#{prefix}/ondemandvm", '{"pool2":"1"}' + end + + context 'with domain set in the config' do + let(:domain) { 'example.com' } + before(:each) do + config[:config]['domain'] = domain + config[:pools][0]['domain'] = domain + end + + it 'should include domain in the return reply' do + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + expect_json(true, 201) + expected = { + "ok": true, + "request_id": uuid, + "domain": domain + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + + context 'with a resource request that exceeds the specified limit' do + let(:max_instances) { 50 } + before(:each) do + config[:config]['max_ondemand_instances_per_request'] = max_instances + end + + it 'should reject the request with a message' do + post "#{prefix}/ondemandvm", '{"pool1":"51"}' + expect_json(false, 403) + expected = { + "ok": false, + "message": "requested amount of instances exceeds the maximum #{max_instances}" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + context 'with request_id provided in the payload' do + it 'uses the given request_id when provided' do + post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' + expect_json(true, 201) + + expected = { + "ok": true, + "request_id": "1234" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns 409 conflict error when the request_id has been used' do + post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' + post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' + expect_json(false, 409) + + expected = { + "ok": false, + "request_id": "1234", + "message": "request_id '1234' has already been created" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + context 'with auth configured' do + + it 'sets the token and user' do + app.settings.set :config, auth: true + expect(SecureRandom).to receive(:uuid).and_return(uuid) + allow(redis).to receive(:hset) + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345') + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe') + post "#{prefix}/ondemandvm", '{"pool1":"1"}', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + end + end + end + + context 'with a pool that is not configured' do + let(:badpool) { 'pool4' } + it 'returns the bad template' do + post "#{prefix}/ondemandvm", '{"pool4":"1"}' + expect_json(false, 404) + + expected = { + "ok": false, + "bad_templates": [ badpool ] + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + it 'returns 400 and a message when JSON is invalid' do + post "#{prefix}/ondemandvm", '{"pool1":"1}' + expect_json(false, 400) + expected = { + "ok": false, + "message": "JSON payload could not be parsed" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + describe 'GET /ondemandvm' do + it 'returns 404 with message when request is not found' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(false, 404) + expected = { + "ok": false, + "message": "no request found for request_id '#{uuid}'" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'when the request is found' do + let(:score) { current_time } + let(:platforms_string) { 'pool1:pool1:1' } + before(:each) do + create_ondemand_request_for_test(uuid, score, platforms_string, redis) + end + + it 'returns 202 while the request is waiting' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 202) + expected = { + "ok": true, + "request_id": uuid, + "ready": false, + "pool1": { + "ready": "0", + "pending": "1" + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with ready instances' do + before(:each) do + create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis) + set_ondemand_request_status(uuid, 'ready', redis) + end + + it 'returns 200 with hostnames when the request is ready' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 200) + expected = { + "ok": true, + "request_id": uuid, + "ready": true, + "pool1": { + "hostname": [ + vmname + ] + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with domain set' do + let(:domain) { 'example.com' } + before(:each) do + config[:config]['domain'] = domain + end + + it 'should include the domain in the result' do + get "#{prefix}/ondemandvm/#{uuid}" + expected = { + "ok": true, + "request_id": uuid, + "ready": true, + "pool1": { + "hostname": [ + vmname + ] + }, + "domain": domain + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + + context 'with a deleted request' do + before(:each) do + set_ondemand_request_status(uuid, 'deleted', redis) + end + + it 'returns a message that the request has been deleted' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 200) + expected = { + "ok": true, + "request_id": uuid, + "ready": false, + "message": "The request has been deleted" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + context 'with a failed request' do + let(:ondemand_request_ttl) { 5 } + before(:each) do + config[:config]['ondemand_request_ttl'] = ondemand_request_ttl + set_ondemand_request_status(uuid, 'failed', redis) + end + + it 'returns a message that the request has failed' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 200) + expected = { + "ok": true, + "request_id": uuid, + "ready": false, + "message": "The request failed to provision instances within the configured ondemand_request_ttl '#{ondemand_request_ttl}'" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + end + + describe 'DELETE /ondemandvm' do + let(:expiration) { 129_600_0 } + it 'returns 404 with message when request is not found' do + delete "#{prefix}/ondemandvm/#{uuid}" + expect_json(false, 404) + expected = { + "ok": false, + "message": "no request found for request_id '#{uuid}'" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'when the request is found' do + let(:platforms_string) { 'pool1:pool1:1' } + let(:score) { current_time.to_i } + before(:each) do + create_ondemand_request_for_test(uuid, score, platforms_string, redis) + end + + it 'returns 200 for a deleted request' do + delete "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 200) + expected = { 'ok': true } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'marks the request hash for expiration in two weeks' do + expect(redis).to receive(:expire).with("vmpooler__odrequest__#{uuid}", expiration) + delete "#{prefix}/ondemandvm/#{uuid}" + end + + context 'with running instances' do + let(:pool) { 'pool1' } + let(:pool_alias) { pool } + before(:each) do + create_ondemand_vm(vmname, uuid, pool, pool_alias, redis) + end + + it 'moves allocated instances to the completed queue' do + expect(redis).to receive(:smove).with("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vmname) + delete "#{prefix}/ondemandvm/#{uuid}" + end + + it 'deletes the set tracking instances allocated for the request' do + expect(redis).to receive(:del).with("vmpooler__#{uuid}__#{pool_alias}__#{pool}") + delete "#{prefix}/ondemandvm/#{uuid}" + end + end + end + end + end +end diff --git a/spec/integration/api/v2/poolreset.rb b/spec/integration/api/v2/poolreset.rb new file mode 100644 index 0000000..37457a3 --- /dev/null +++ b/spec/integration/api/v2/poolreset.rb @@ -0,0 +1,120 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + after(:each) do + Vmpooler::API.reset! + end + + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + 'experimental_features' => true + }, + pools: [ + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, + {'name' => 'pool2', 'size' => 10} + ], + statsd: { 'prefix' => 'stats_prefix'}, + alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] + } + } + + describe '/poolreset' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + + let(:current_time) { Time.now } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + app.settings.set :config, auth: false + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'POST /poolreset' do + it 'refreshes ready and pending instances from a pool' do + post "#{prefix}/poolreset", '{"pool1":"1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails on nonexistent pools' do + post "#{prefix}/poolreset", '{"poolpoolpool":"1"}' + expect_json(ok = false, http = 400) + end + + it 'resets multiple pools' do + post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when not all pools exist' do + post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_pools: ['pool3'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/poolreset", '{"pool1":"1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + it 'should return 400 for invalid json' do + post "#{prefix}/poolreset", '{"pool1":"1}' + expect_json(ok = false, http = 400) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'should return 400 with a bad pool name' do + post "#{prefix}/poolreset", '{"pool11":"1"}' + expect_json(ok = false, http = 400) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'should return 404 when there is no payload' do + post "#{prefix}/poolreset" + expect_json(ok = false, http = 404) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end +end diff --git a/spec/integration/api/v2/status_spec.rb b/spec/integration/api/v2/status_spec.rb new file mode 100644 index 0000000..d412128 --- /dev/null +++ b/spec/integration/api/v2/status_spec.rb @@ -0,0 +1,247 @@ +require 'spec_helper' +require 'rack/test' + +def has_set_tag?(vm, tag, value) + value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}") +end + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe 'status and metrics endpoints' do + let(:prefix) { '/api/v2' } + + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + }, + pools: [ + {'name' => 'pool1', 'size' => 5, 'alias' => ['poolone', 'poolun']}, + {'name' => 'pool2', 'size' => 10}, + {'name' => 'pool3', 'size' => 10, 'alias' => 'NotArray'} + ] + } + } + + let(:current_time) { Time.now } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, nil, nil) + app.settings.set :config, auth: false + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'GET /status' do + it 'returns the configured maximum size for each pool' do + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"]["pool1"]["max"]).to be(5) + expect(result["pools"]["pool2"]["max"]).to be(10) + end + + it 'returns the number of ready vms for each pool' do + 3.times {|i| create_ready_vm("pool1", "vm-#{i}", redis) } + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"]["pool1"]["ready"]).to be(3) + expect(result["pools"]["pool2"]["ready"]).to be(0) + end + + it 'returns the number of running vms for each pool' do + 3.times {|i| create_running_vm("pool1", "vm-#{i}", redis) } + 4.times {|i| create_running_vm("pool2", "vm-#{i}", redis) } + + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"]["pool1"]["running"]).to be(3) + expect(result["pools"]["pool2"]["running"]).to be(4) + end + + it 'returns the number of pending vms for each pool' do + 3.times {|i| create_pending_vm("pool1", "vm-#{i}", redis) } + 4.times {|i| create_pending_vm("pool2", "vm-#{i}", redis) } + + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"]["pool1"]["pending"]).to be(3) + expect(result["pools"]["pool2"]["pending"]).to be(4) + end + + it 'returns aliases if configured in the pool' do + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun']) + expect(result["pools"]["pool2"]["alias"]).to be(nil) + expect(result["pools"]["pool3"]["alias"]).to eq('NotArray') + end + + it '(for v1 backwards compatibility) lists any empty pools in the status section' do + get "#{prefix}/status/" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["status"]["empty"].sort).to eq(["pool1", "pool2", "pool3"]) + end + end + describe 'GET /status with view query parameter' do + it 'returns capacity when specified' do + get "#{prefix}/status?view=capacity" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to_not be(nil) + expect(result["queue"]).to be(nil) + expect(result["clone"]).to be(nil) + expect(result["boot"]).to be(nil) + expect(result["pools"]).to be(nil) + expect(result["status"]).to_not be(nil) + end + it 'returns pools and queue when specified' do + get "#{prefix}/status?view=pools,queue" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to be(nil) + expect(result["queue"]).to_not be(nil) + expect(result["clone"]).to be(nil) + expect(result["boot"]).to be(nil) + expect(result["pools"]).to_not be(nil) + expect(result["status"]).to_not be(nil) + end + it 'does nothing with invalid view names' do + get "#{prefix}/status?view=clone,boot,invalidThingToView" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to be(nil) + expect(result["queue"]).to be(nil) + expect(result["clone"]).to_not be(nil) + expect(result["boot"]).to_not be(nil) + expect(result["pools"]).to be(nil) + expect(result["status"]).to_not be(nil) + end + it 'returns everything when view is not specified' do + get "#{prefix}/status" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to_not be(nil) + expect(result["queue"]).to_not be(nil) + expect(result["clone"]).to_not be(nil) + expect(result["boot"]).to_not be(nil) + expect(result["pools"]).to_not be(nil) + expect(result["status"]).to_not be(nil) + end + it 'returns everything when view is alone' do + get "#{prefix}/status?view" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to_not be(nil) + expect(result["queue"]).to_not be(nil) + expect(result["clone"]).to_not be(nil) + expect(result["boot"]).to_not be(nil) + expect(result["pools"]).to_not be(nil) + expect(result["status"]).to_not be(nil) + end + it 'returns status only when view is empty' do + get "#{prefix}/status?view=" + + # of course /status doesn't conform to the weird standard everything else uses... + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["capacity"]).to be(nil) + expect(result["queue"]).to be(nil) + expect(result["clone"]).to be(nil) + expect(result["boot"]).to be(nil) + expect(result["pools"]).to be(nil) + expect(result["status"]).to_not be(nil) + end + end + + describe 'GET /poolstat' do + it 'returns empty list when pool is not set' do + get "#{prefix}/poolstat" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result == {}) + end + it 'returns empty list when pool is not found' do + get "#{prefix}/poolstat?pool=unknownpool" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result == {}) + end + it 'returns one pool when requesting one with alias' do + get "#{prefix}/poolstat?pool=pool1" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 1) + expect(result["pools"]["pool1"]["ready"]).to eq(0) + expect(result["pools"]["pool1"]["max"]).to eq(5) + expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun']) + end + it 'returns one pool when requesting one without alias' do + get "#{prefix}/poolstat?pool=pool2" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 1) + expect(result["pools"]["pool2"]["ready"]).to eq(0) + expect(result["pools"]["pool2"]["max"]).to eq(10) + expect(result["pools"]["pool2"]["alias"]).to be(nil) + end + it 'returns multiple pools when requesting csv' do + get "#{prefix}/poolstat?pool=pool1,pool2" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 2) + end + end + + describe 'GET /totalrunning' do + it 'returns the number of running VMs' do + get "#{prefix}/totalrunning" + expect(last_response.header['Content-Type']).to eq('application/json') + 5.times {|i| create_running_vm("pool1", "vm-#{i}", redis, redis) } + 5.times {|i| create_running_vm("pool3", "vm-#{i}", redis, redis) } + result = JSON.parse(last_response.body) + expect(result["running"] == 10) + end + end + end +end diff --git a/spec/integration/api/v2/token_spec.rb b/spec/integration/api/v2/token_spec.rb new file mode 100644 index 0000000..0dfee9e --- /dev/null +++ b/spec/integration/api/v2/token_spec.rb @@ -0,0 +1,210 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe '/token' do + let(:prefix) { '/api/v2' } + let(:current_time) { Time.now } + let(:config) { { + config: {} + } } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, nil, nil) + end + + describe 'GET /token' do + context '(auth not configured)' do + let(:config) { { + config: {}, + auth: false + } } + + it 'returns a 404' do + get "#{prefix}/token" + expect_json(ok = false, http = 404) + end + end + + context '(auth configured)' do + let(:config) { + { + config: {}, + auth: { + 'provider' => 'dummy' + } + } + } + let(:username_str) { 'admin' } + let(:password_str) { 's3cr3t' } + + it 'returns a 401 if not authed' do + get "#{prefix}/token" + expect_json(ok = false, http = 401) + end + + it 'returns a list of tokens if authed' do + create_token "abc", "admin", current_time + + authorize 'admin', 's3cr3t' + get "#{prefix}/token" + expect_json(ok = true, http = 200) + + expect(JSON.parse(last_response.body)['abc']['created']).to eq(current_time.to_s) + end + end + end + + describe 'POST /token' do + context '(auth not configured)' do + let(:config) { { + config: {}, + auth: false + } } + + it 'returns a 404' do + post "#{prefix}/token" + expect_json(ok = false, http = 404) + end + end + + context '(auth configured)' do + let(:config) { + { + config: {}, + auth: { + 'provider' => 'dummy' + } + } + } + + it 'returns a 401 if not authed' do + post "#{prefix}/token" + expect_json(ok = false, http = 401) + end + + it 'returns a newly created token if authed' do + authorize 'admin', 's3cr3t' + post "#{prefix}/token" + expect_json(ok = true, http = 200) + + returned_token = JSON.parse(last_response.body)['token'] + expect(returned_token.length).to be(32) + expect(get_token_data(returned_token)['user']).to eq("admin") + end + end + end + end + + describe '/token/:token' do + let(:prefix) { '/api/v2' } + let(:current_time) { Time.now } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, nil, nil) + app.settings.set :config, config + app.settings.set :redis, redis + end + + def create_vm_for_token(token, pool, vm) + redis.sadd("vmpooler__running__#{pool}", vm) + redis.hset("vmpooler__vm__#{vm}", "token:token", token) + end + + describe 'GET /token/:token' do + context '(auth not configured)' do + let(:config) { { + config: {}, + auth: false + } } + + it 'returns a 404' do + get "#{prefix}/token/this" + expect_json(ok = false, http = 404) + end + end + + context '(auth configured)' do + let(:config) { { + config: {}, + auth: true, + pools: [ + {'name' => 'pool1', 'size' => 5} + ] + } } + + it 'returns a token' do + create_token "mytoken", "admin", current_time + create_vm_for_token "mytoken", "pool1", "vmhostname" + + get "#{prefix}/token/mytoken" + expect_json(ok = true, http = 200) + + expect(JSON.parse(last_response.body)['ok']).to eq(true) + expect(JSON.parse(last_response.body)['mytoken']['user']).to eq('admin') + expect(JSON.parse(last_response.body)['mytoken']['vms']['running']).to include('vmhostname') + end + end + end + + describe 'DELETE /token/:token' do + context '(auth not configured)' do + let(:config) { { + config: {}, + auth: false + } } + + it 'returns a 404' do + delete "#{prefix}/token/this" + expect_json(ok = false, http = 404) + end + end + + context '(auth configured)' do + let(:config) { + { + config: {}, + auth: { + 'provider' => 'dummy' + } + } + } + + it 'returns a 401 if not authed' do + delete "#{prefix}/token/this" + expect_json(ok = false, http = 401) + end + + it 'deletes a token if authed' do + create_token("mytoken", "admin", current_time) + authorize 'admin', 's3cr3t' + + delete "#{prefix}/token/mytoken" + expect_json(ok = true, http = 200) + end + + it 'fails if token does not exist' do + authorize 'admin', 's3cr3t' + + delete "#{prefix}/token/missingtoken" + expect_json(ok = false, http = 401) # TODO: should this be 404? + end + end + end + end +end diff --git a/spec/integration/api/v2/vm_hostname_spec.rb b/spec/integration/api/v2/vm_hostname_spec.rb new file mode 100644 index 0000000..b232cb2 --- /dev/null +++ b/spec/integration/api/v2/vm_hostname_spec.rb @@ -0,0 +1,352 @@ +require 'spec_helper' +require 'rack/test' + +def has_set_tag?(vm, tag, value) + value == redis.hget("vmpooler__vm__#{vm}", "tag:#{tag}") +end + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe '/vm/:hostname' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + + }, + pools: [ + {'name' => 'pool1', 'size' => 5}, + {'name' => 'pool2', 'size' => 10} + ], + alias: { 'poolone' => 'pool1' }, + auth: false + } + } + + let(:current_time) { Time.now } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'PUT /vm/:hostname' do + it 'allows tags to be set' do + create_vm('testhost', redis) + put "#{prefix}/vm/testhost", '{"tags":{"tested_by":"rspec"}}' + expect_json(ok = true, http = 200) + + expect has_set_tag?('testhost', 'tested_by', 'rspec') + end + + it 'skips empty tags' do + create_vm('testhost', redis) + put "#{prefix}/vm/testhost", '{"tags":{"tested_by":""}}' + expect_json(ok = true, http = 200) + + expect !has_set_tag?('testhost', 'tested_by', '') + end + + it 'does not set tags if request body format is invalid' do + create_vm('testhost', redis) + put "#{prefix}/vm/testhost", '{"tags":{"tested"}}' + expect_json(ok = false, http = 400) + + expect !has_set_tag?('testhost', 'tested', '') + end + + context '(allowed_tags configured)' do + it 'fails if specified tag is not in allowed_tags array' do + app.settings.set :config, + { :config => { 'allowed_tags' => ['created_by', 'project', 'url'] } } + + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"tags":{"created_by":"rspec","tested_by":"rspec"}}' + expect_json(ok = false, http = 400) + + expect !has_set_tag?('testhost', 'tested_by', 'rspec') + end + end + + context '(tagfilter configured)' do + before(:each) do + app.settings.set :config, tagfilter: { 'url' => '(.*)\/' } + end + + it 'correctly filters tags' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com/something.html"}}' + expect_json(ok = true, http = 200) + + expect has_set_tag?('testhost', 'url', 'foo.com') + end + + it "doesn't eat tags not matching filter" do + create_vm('testhost', redis) + put "#{prefix}/vm/testhost", '{"tags":{"url":"foo.com"}}' + expect_json(ok = true, http = 200) + + expect has_set_tag?('testhost', 'url', 'foo.com') + end + end + + context '(auth not configured)' do + before(:each) do + app.settings.set :config, auth: false + end + + it 'allows VM lifetime to be modified without a token' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"1"}' + expect_json(ok = true, http = 200) + + vm = fetch_vm('testhost') + expect(vm['lifetime'].to_i).to eq(1) + end + + it 'does not allow a lifetime to be 0' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"0"}' + expect_json(ok = false, http = 400) + + vm = fetch_vm('testhost') + expect(vm['lifetime']).to be_nil + end + + it 'does not enforce a lifetime' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"20000"}' + expect_json(ok = true, http = 200) + + vm = fetch_vm('testhost') + expect(vm['lifetime']).to eq("20000") + end + + it 'does not allow a lifetime to be initially past config max_lifetime_upper_limit' do + app.settings.set :config, + { :config => { 'max_lifetime_upper_limit' => 168 } } + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"200"}' + expect_json(ok = false, http = 400) + + vm = fetch_vm('testhost') + expect(vm['lifetime']).to be_nil + end + +# it 'does not allow a lifetime to be extended past config 168' do +# app.settings.set :config, +# { :config => { 'max_lifetime_upper_limit' => 168 } } +# create_vm('testhost', redis) +# +# set_vm_data('testhost', "checkout", (Time.now - (69*60*60)), redis) +# puts redis.hget("vmpooler__vm__testhost", 'checkout') +# put "#{prefix}/vm/testhost", '{"lifetime":"100"}' +# expect_json(ok = false, http = 400) +# +# vm = fetch_vm('testhost') +# expect(vm['lifetime']).to be_nil +# end + end + + context '(auth configured)' do + before(:each) do + app.settings.set :config, auth: true + end + + it 'allows VM lifetime to be modified with a token' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"1"}', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + vm = fetch_vm('testhost') + expect(vm['lifetime'].to_i).to eq(1) + end + + it 'does not allows VM lifetime to be modified without a token' do + create_vm('testhost', redis) + + put "#{prefix}/vm/testhost", '{"lifetime":"1"}' + expect_json(ok = false, http = 401) + end + end + end + + describe 'DELETE /vm/:hostname' do + context '(auth not configured)' do + it 'does not delete a non-existant VM' do + delete "#{prefix}/vm/testhost" + expect_json(ok = false, http = 404) + end + + it 'deletes an existing VM' do + create_running_vm('pool1', 'testhost', redis) + expect fetch_vm('testhost') + + delete "#{prefix}/vm/testhost" + expect_json(ok = true, http = 200) + expect !fetch_vm('testhost') + end + end + + context '(auth configured)' do + before(:each) do + app.settings.set :config, auth: true + end + + context '(checked-out without token)' do + it 'deletes a VM without supplying a token' do + create_running_vm('pool1', 'testhost', redis) + expect fetch_vm('testhost') + + delete "#{prefix}/vm/testhost" + expect_json(ok = true, http = 200) + expect !fetch_vm('testhost') + end + end + + context '(checked-out with token)' do + it 'fails to delete a VM without supplying a token' do + create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345') + expect fetch_vm('testhost') + + delete "#{prefix}/vm/testhost" + expect_json(ok = false, http = 401) + expect fetch_vm('testhost') + end + + it 'deletes a VM when token is supplied' do + create_running_vm('pool1', 'testhost', redis, 'abcdefghijklmnopqrstuvwxyz012345') + expect fetch_vm('testhost') + + delete "#{prefix}/vm/testhost", "", { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + expect !fetch_vm('testhost') + end + end + end + end + + describe 'POST /vm/:hostname/snapshot' do + context '(auth not configured)' do + it 'creates a snapshot' do + create_vm('testhost', redis) + post "#{prefix}/vm/testhost/snapshot" + expect_json(ok = true, http = 202) + expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32) + end + end + + context '(auth configured)' do + before(:each) do + app.settings.set :config, auth: true + end + + it 'returns a 401 if not authed' do + post "#{prefix}/vm/testhost/snapshot" + expect_json(ok = false, http = 401) + expect !has_vm_snapshot?('testhost', redis) + end + + it 'creates a snapshot if authed' do + create_vm('testhost', redis) + snapshot_vm('testhost', 'testsnapshot', redis) + + post "#{prefix}/vm/testhost/snapshot", "", { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 202) + expect(JSON.parse(last_response.body)['testhost']['snapshot'].length).to be(32) + expect has_vm_snapshot?('testhost', redis) + end + end + end + + describe 'POST /vm/:hostname/snapshot/:snapshot' do + context '(auth not configured)' do + it 'reverts to a snapshot' do + create_vm('testhost', redis) + snapshot_vm('testhost', 'testsnapshot', redis) + + post "#{prefix}/vm/testhost/snapshot/testsnapshot" + expect_json(ok = true, http = 202) + expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot') + end + + it 'fails if the specified snapshot does not exist' do + create_vm('testhost', redis) + + post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = false, http = 404) + expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot') + end + end + + context '(auth configured)' do + before(:each) do + app.settings.set :config, auth: true + end + + it 'returns a 401 if not authed' do + create_vm('testhost', redis) + snapshot_vm('testhost', 'testsnapshot', redis) + + post "#{prefix}/vm/testhost/snapshot/testsnapshot" + expect_json(ok = false, http = 401) + expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot') + end + + it 'fails if authed and the specified snapshot does not exist' do + create_vm('testhost', redis) + + post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = false, http = 404) + expect !vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot') + end + + it 'reverts to a snapshot if authed' do + create_vm('testhost', redis) + snapshot_vm('testhost', 'testsnapshot', redis) + + post "#{prefix}/vm/testhost/snapshot/testsnapshot", "", { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 202) + expect vm_reverted_to_snapshot?('testhost', redis, 'testsnapshot') + end + end + end + end +end diff --git a/spec/integration/api/v2/vm_spec.rb b/spec/integration/api/v2/vm_spec.rb new file mode 100644 index 0000000..5dd30bf --- /dev/null +++ b/spec/integration/api/v2/vm_spec.rb @@ -0,0 +1,444 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe '/vm' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2 + }, + pools: [ + {'name' => 'pool1', 'size' => 5, 'domain' => 'one.example.com'}, + {'name' => 'pool2', 'size' => 10, 'domain' => 'two.example.com'}, + {'name' => 'pool3', 'size' => 10, 'domain' => 'three.example.com'} + ], + statsd: { 'prefix' => 'stats_prefix'}, + alias: { 'poolone' => ['pool1'] }, + pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone', 'genericpool' ] + } + } + let(:current_time) { Time.now } + let(:vmname) { 'abcdefghijkl' } + let(:checkoutlock) { Mutex.new } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + app.settings.set :config, auth: false + app.settings.set :checkoutlock, checkoutlock + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'GET /vm/:hostname' do + it 'returns correct information on a running vm' do + create_running_vm 'pool1', vmname, redis + expect(TCPSocket).to receive(:gethostbyname).and_raise(RuntimeError) + get "#{prefix}/vm/#{vmname}" + expect_json(ok = true, http = 200) + response_body = (JSON.parse(last_response.body)[vmname]) + + expect(response_body["template"]).to eq("pool1") + expect(response_body["lifetime"]).to eq(0) + expect(response_body["running"]).to be >= 0 + expect(response_body["remaining"]).to be <= 0 + expect(response_body["start_time"]).to eq(current_time.to_datetime.rfc3339) + expect(response_body["end_time"]).to eq(current_time.to_datetime.rfc3339) + expect(response_body["state"]).to eq("running") + expect(response_body["ip"]).to eq("") + expect(response_body["host"]).to eq("host1") + end + end + + describe 'POST /vm' do + + let(:socket) { double('socket') } + it 'returns a single VM' do + create_ready_vm 'pool1', vmname, redis + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: "#{vmname}.one.example.com" + }, + domain: 'one.example.com' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns a single VM for an alias' do + create_ready_vm 'pool1', vmname, redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"poolone":"1"}' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + poolone: { + hostname: "#{vmname}.one.example.com" + }, + domain: 'one.example.com' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails on nonexistant pools' do + post "#{prefix}/vm", '{"poolpoolpool":"1"}' + expect_json(ok = false, http = 404) + end + + it 'returns 503 for empty pool when aliases are not defined' do + app.settings.config.delete(:alias) + app.settings.config[:pool_names] = ['pool1', 'pool2'] + + create_ready_vm 'pool1', vmname, redis + post "#{prefix}/vm/pool1" + post "#{prefix}/vm/pool1" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns 503 for empty pool referenced by alias' do + create_ready_vm 'pool1', vmname, redis + post "#{prefix}/vm/poolone" + post "#{prefix}/vm/poolone" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns multiple VMs' do + create_ready_vm 'pool1', vmname, redis + create_ready_vm 'pool2', 'qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: "#{vmname}.one.example.com" + }, + domain: 'two.example.com', + pool2: { + hostname: 'qrstuvwxyz012345.two.example.com' + } + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns multiple VMs even when multiple instances from the same pool are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool1', '2abcdefghijklmnop', redis + create_ready_vm 'pool2', 'qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: 'qrstuvwxyz012345' + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com') + expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345.two.example.com') + + expect_json(ok = true, http = 200) + end + + it 'returns multiple VMs even when multiple instances from multiple pools are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool1', '2abcdefghijklmnop', redis + create_ready_vm 'pool2', '1qrstuvwxyz012345', redis + create_ready_vm 'pool2', '2qrstuvwxyz012345', redis + create_ready_vm 'pool2', '3qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ] + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.one.example.com') + expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345.two.example.com', '2qrstuvwxyz012345.two.example.com', '3qrstuvwxyz012345.two.example.com') + + expect_json(ok = true, http = 200) + end + + it 'returns VMs from multiple backend pools requested by an alias' do + Vmpooler::API.settings.config[:alias]['genericpool'] = ['pool1', 'pool2', 'pool3'] + + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool2', '2abcdefghijklmnop', redis + create_ready_vm 'pool3', '1qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"genericpool":"3"}' + + expected = { + ok: true, + genericpool: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop', '1qrstuvwxyz012345' ] + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['genericpool']['hostname']).to include('1abcdefghijklmnop.one.example.com', '2abcdefghijklmnop.two.example.com', '1qrstuvwxyz012345.three.example.com') + + expect_json(ok = true, http = 200) + end + + it 'returns the first VM that was moved to the ready state when checking out a VM' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool1', '2abcdefghijklmnop', redis + create_ready_vm 'pool1', '3abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}' + + expected = { + ok: true, + "pool1": { + "hostname": "1abcdefghijklmnop.one.example.com" + }, + domain: 'one.example.com' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = true, http = 200) + end + + it 'fails when not all requested vms can be allocated' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + + post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool1', '2abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop', redis)).to eq(true) + expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop', redis)).to eq(true) + end + + it 'returns the second VM when the first fails to respond' do + create_ready_vm 'pool1', vmname, redis + create_ready_vm 'pool1', "2#{vmname}", redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with(vmname, 'one.example.com').and_raise('mockerror') + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with("2#{vmname}", 'one.example.com').and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: "2#{vmname}.one.example.com" + }, + domain: 'one.example.com' + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + expect(pool_has_ready_vm?('pool1', vmname, redis)).to be false + end + + context '(auth not configured)' do + it 'does not extend VM lifetime if auth token is provided' do + app.settings.set :config, auth: false + + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop.one.example.com' + }, + domain: 'one.example.com' + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime']).to be_nil + end + end + + context '(auth configured)' do + it 'extends VM lifetime if auth token is provided' do + app.settings.set :config, auth: true + + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop.one.example.com' + }, + domain: 'one.example.com' + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime'].to_i).to eq(2) + end + + it 'does not extend VM lifetime if auth token is not provided' do + app.settings.set :config, auth: true + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm", '{"pool1":"1"}' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop.one.example.com' + }, + domain: 'one.example.com' + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime']).to be_nil + end + end + end + end +end diff --git a/spec/integration/api/v2/vm_template_spec.rb b/spec/integration/api/v2/vm_template_spec.rb new file mode 100644 index 0000000..dc46c40 --- /dev/null +++ b/spec/integration/api/v2/vm_template_spec.rb @@ -0,0 +1,326 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V2 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + # Added to ensure no leakage in rack state from previous tests. + # Removes all routes, filters, middleware and extension hooks from the current class + # https://rubydoc.info/gems/sinatra/Sinatra/Base#reset!-class_method + before(:each) do + app.reset! + end + + describe '/vm/:template' do + let(:prefix) { '/api/v2' } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + }, + pools: [ + {'name' => 'pool1', 'size' => 5}, + {'name' => 'pool2', 'size' => 10}, + {'name' => 'poolone', 'size' => 0} + ], + statsd: { 'prefix' => 'stats_prefix'}, + alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] + } + } + + let(:current_time) { Time.now } + let(:socket) { double('socket') } + let(:checkoutlock) { Mutex.new } + + before(:each) do + expect(app).to receive(:run!).once + app.execute([:api], config, redis, metrics, nil) + app.settings.set :config, auth: false + app.settings.set :checkoutlock, checkoutlock + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'POST /vm/:template' do + it 'returns a single VM' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1", '' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop' + } + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns a single VM for an alias' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/poolone", '' + + expected = { + ok: true, + poolone: { + hostname: 'abcdefghijklmnop' + } + } + expect_json(ok = true, http = 200) + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails on nonexistant pools' do + post "#{prefix}/vm/poolpoolpool", '' + expect_json(ok = false, http = 404) + end + + it 'returns 503 for empty pool when aliases are not defined' do + app.settings.config.delete(:alias) + app.settings.config[:pool_names] = ['pool1', 'pool2'] + + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + post "#{prefix}/vm/pool1" + post "#{prefix}/vm/pool1" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns 503 for empty pool referenced by alias' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + post "#{prefix}/vm/poolone" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns multiple VMs' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool2', 'qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1+pool2", '' + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop' + }, + pool2: { + hostname: 'qrstuvwxyz012345' + } + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'returns multiple VMs even when multiple instances from multiple pools are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop', redis + create_ready_vm 'pool1', '2abcdefghijklmnop', redis + + create_ready_vm 'pool2', '1qrstuvwxyz012345', redis + create_ready_vm 'pool2', '2qrstuvwxyz012345', redis + create_ready_vm 'pool2', '3qrstuvwxyz012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ] + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop') + expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345') + expect_json(ok = true, http = 200) + end + + it 'fails when not all requested vms can be allocated' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + post "#{prefix}/vm/pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool1', '0123456789012345', redis + + post "#{prefix}/vm/pool1+pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool1', '0123456789012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1+pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true) + expect(pool_has_ready_vm?('pool1', '0123456789012345', redis)).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool2', '0123456789012345', redis + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool2', '0123456789012345', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop', redis)).to eq(true) + expect(pool_has_ready_vm?('pool2', '0123456789012345', redis)).to eq(true) + end + + context '(auth not configured)' do + it 'does not extend VM lifetime if auth token is provided' do + app.settings.set :config, auth: false + + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1", '', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop' + } + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime']).to be_nil + end + end + + context '(auth configured)' do + it 'extends VM lifetime if auth token is provided' do + app.settings.set :config, auth: true + + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1", '', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + expect_json(ok = true, http = 200) + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop' + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime'].to_i).to eq(2) + end + + it 'does not extend VM lifetime if auth token is not provided' do + app.settings.set :config, auth: true + create_ready_vm 'pool1', 'abcdefghijklmnop', redis + + allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).and_return(socket) + + post "#{prefix}/vm/pool1", '' + + expected = { + ok: true, + pool1: { + hostname: 'abcdefghijklmnop' + } + } + expect_json(ok = true, http = 200) + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + + vm = fetch_vm('abcdefghijklmnop') + expect(vm['lifetime']).to be_nil + end + end + end + end +end diff --git a/spec/unit/api/helpers_spec.rb b/spec/unit/api/helpers_spec.rb index 8325baa..5991fa3 100644 --- a/spec/unit/api/helpers_spec.rb +++ b/spec/unit/api/helpers_spec.rb @@ -16,12 +16,12 @@ describe Vmpooler::API::Helpers do describe '#hostname_shorten' do [ - ['example.com', 'not-example.com', 'example.com'], - ['example.com', 'example.com', 'example.com'], + ['example.com', 'not-example.com', 'example'], + ['example.com', 'example.com', 'example'], ['sub.example.com', 'example.com', 'sub'], ['adjective-noun.example.com', 'example.com', 'adjective-noun'], ['abc123.example.com', 'example.com', 'abc123'], - ['example.com', nil, 'example.com'] + ['example.com', nil, 'example'] ].each do |hostname, domain, expected| it { expect(subject.hostname_shorten(hostname, domain)).to eq expected } end diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 280e61f..f723376 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -1672,6 +1672,93 @@ EOT end end + describe '#get_domain_for_pool' do + let(:provider_name) { 'mock_provider' } + context 'No provider is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + EOT + )} + + it 'should return nil' do + result = subject.get_domain_for_pool(pool) + expect(result).to be_nil + end + end + context 'No domain is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return nil' do + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) + allow(provider).to receive(:name).and_return(provider_name) + result = subject.get_domain_for_pool(pool) + expect(result).to be_nil + end + end + + context 'Only a global domain is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + domain: example.com + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return the domain set in the config section' do + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) + allow(provider).to receive(:name).and_return(provider_name) + result = subject.get_domain_for_pool(pool) + expect(result).to_not be_nil + expect(result).to eq('example.com') + end + end + + context 'A provider specified a domain to use' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + domain: m.example.com + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return the domain set in the config section' do + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) + allow(provider).to receive(:name).and_return(provider_name) + result = subject.get_domain_for_pool(pool) + expect(result).to_not be_nil + expect(result).to eq('m.example.com') + end + end + end + describe '#check_disk_queue' do let(:threads) {[]} @@ -2708,6 +2795,7 @@ EOT allow(subject).to receive(:check_snapshot_queue) allow(subject).to receive(:check_pool) allow(subject).to receive(:check_ondemand_requests) + allow(subject).to receive(:get_domain_for_pool).and_return('example.com') allow(logger).to receive(:log) end diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index fcc22e6..1d8591e 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -440,7 +440,10 @@ # If set, restricts tags to those specified in this array. # # - domain -# If set, returns a top-level 'domain' JSON key in POST requests +# If set, returns a top-level 'domain' JSON key in POST requests. Note that domains can now be set per-pool +# and at the provider level. The domain key in the JSON response will correspond the last vm added to the +# response and should not be relied upon as the hostnames of the returned vms now include the domain. The +# domain key is only retained now for backwards compatibility and will be removed in the future. # # - prefix # If set, prefixes all created VMs with this string. This should include From e5c477254fdfa4539b19bb4ff33682519c09fc02 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Tue, 1 Feb 2022 16:39:57 -0600 Subject: [PATCH 02/20] initial review --- lib/vmpooler.rb | 2 + lib/vmpooler/api/v2.rb | 21 +++---- lib/vmpooler/pool_manager.rb | 29 ++------- lib/vmpooler/util/parsing.rb | 22 ++++++- spec/spec_helper.rb | 2 + spec/unit/pool_manager_spec.rb | 87 --------------------------- spec/unit/util/parsing_spec.rb | 104 +++++++++++++++++++++++++++++++++ 7 files changed, 141 insertions(+), 126 deletions(-) create mode 100644 spec/unit/util/parsing_spec.rb diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index d666943..1bf8e03 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -43,6 +43,7 @@ module Vmpooler if parsed_config[:config]['extra_config'] extra_configs = parsed_config[:config]['extra_config'].split(',') extra_configs.each do |config| + puts "loading extra_config file #{config}" extra_config = YAML.load_file(config) parsed_config.deep_merge(extra_config) end @@ -133,6 +134,7 @@ module Vmpooler # Create an index of pool aliases parsed_config[:pool_names] = Set.new unless parsed_config[:pools] + puts 'loading pools configuration from redis, since the config[:pools] is empty' redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password']) parsed_config[:pools] = load_pools_from_redis(redis) end diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index 9e52d27..4cd1a66 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -1,12 +1,17 @@ # frozen_string_literal: true require 'vmpooler/api/v1' + module Vmpooler class API class V2 < Vmpooler::API::V1 api_version = '2' api_prefix = "/api/v#{api_version}" + def full_config + Vmpooler::API.settings.config + end + def get_template_aliases(template) tracer.in_span("Vmpooler::API::V2.#{__method__}") do result = [] @@ -19,11 +24,6 @@ module Vmpooler end end - def get_domain_for_pool(poolname) - pool_index = pool_index(pools) - pools[pool_index[poolname]]['domain'] - end - # Fetch a single vm from a pool # # @param [String] template @@ -73,7 +73,7 @@ module Vmpooler next if vms.empty? vms.reverse.each do |vm| - vm_domain = get_domain_for_pool(template_backend) + vm_domain = Parsing.get_domain_for_pool(full_config, template_backend) ready = vm_ready?(vm, vm_domain) if ready smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) @@ -137,12 +137,11 @@ module Vmpooler else vm_names = [] vms.each do |(vmpool, vmname, vmtemplate)| - vmdomain = get_domain_for_pool(vmpool) + vmdomain = Parsing.get_domain_for_pool(full_config, vmpool) if vmdomain vmfqdn = "#{vmname}.#{vmdomain}" update_result_hosts(result, vmtemplate, vmfqdn) vm_names.append(vmfqdn) - result['domain'] = vmdomain else update_result_hosts(result, vmtemplate, vmname) vm_names.append(vmname) @@ -197,11 +196,7 @@ module Vmpooler platforms_with_aliases = [] requested_instances.each do |poolname, count| selection = evaluate_template_aliases(poolname, count) - selection.map do |selected_pool, selected_pool_count| - platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" - pool_domain = get_domain_for_pool(selected_pool) - result['domain'] = pool_domain if pool_domain - end + selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } end platforms_string = platforms_with_aliases.join(',') diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 2b9a07f..c38f436 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -60,7 +60,7 @@ module Vmpooler to_set[k] = pool[k] end to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias') - to_set['domain'] = get_domain_for_pool(pool['name']) + to_set['domain'] = Parsing.get_domain_for_pool(config, pool['name']) redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty? end previously_configured_pools.each do |pool| @@ -362,7 +362,7 @@ module Vmpooler max_hostname_retries = 3 while hostname_retries < max_hostname_retries hostname, hostname_available = generate_and_check_hostname - domain = get_domain_for_pool(pool_name) + domain = Parsing.get_domain_for_pool(config, pool_name) if domain fqdn = "#{hostname}.#{domain}" else @@ -384,7 +384,7 @@ module Vmpooler raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available - fqdn + hostname end # Query the DNS for the name we want to create and if it already exists, mark it unavailable @@ -402,7 +402,7 @@ module Vmpooler def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil) new_vmname = find_unique_hostname(pool_name) - pool_domain = get_domain_for_pool(pool_name) + pool_domain = Parsing.get_domain_for_pool(config, pool_name) mutex = vm_mutex(new_vmname) mutex.synchronize do @redis.with_metrics do |redis| @@ -665,27 +665,6 @@ module Vmpooler $providers[provider_name] end - # @param pool_name [String] - the name of the pool - # @return [String] - domain name for pool, if set - def get_domain_for_pool(pool_name) - pool = $config[:pools].find { |p| p['name'] == pool_name } - return nil unless pool - - provider_name = pool.fetch('provider', nil) - return nil unless provider_name - - $logger.log('d', "[*] [get_domain_for_pool] #{pool_name}'s provider is #{provider_name}") - if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain'] - domain = config[:providers][provider_name.to_sym]['domain'] - elsif config[:config] && config[:config]['domain'] - domain = config[:config]['domain'] - else - domain = nil - end - - domain - end - def check_disk_queue(maxloop = 0, loop_delay = 5) $logger.log('d', '[*] [disk_manager] starting worker thread') diff --git a/lib/vmpooler/util/parsing.rb b/lib/vmpooler/util/parsing.rb index 8a8273f..3949a55 100644 --- a/lib/vmpooler/util/parsing.rb +++ b/lib/vmpooler/util/parsing.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# utility class shared between apps +# utility class shared between apps api and pool_manager module Vmpooler class Parsing def self.get_platform_pool_count(requested, &_block) @@ -12,5 +12,25 @@ module Vmpooler yield platform_alias, pool, count end end + + # @param config [String] - the full config structure + # @param pool_name [String] - the name of the pool + # @return [String] - domain name for pool, if set in the provider for the pool or in the config block + def self.get_domain_for_pool(config, pool_name) + pool = config[:pools].find { |p| p['name'] == pool_name } + return nil unless pool + + provider_name = pool.fetch('provider', 'vsphere') # see vmpooler.yaml.example where it states defaulting to vsphere + + if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain'] + domain = config[:providers][provider_name.to_sym]['domain'] + elsif config[:config] && config[:config]['domain'] + domain = config[:config]['domain'] + else + domain = nil + end + + domain + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index dc94ee7..875596b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,9 @@ require 'simplecov' +=begin SimpleCov.start do add_filter '/spec/' end +=end require 'helpers' require 'rspec' require 'vmpooler' diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index f723376..6c1879c 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -1672,93 +1672,6 @@ EOT end end - describe '#get_domain_for_pool' do - let(:provider_name) { 'mock_provider' } - context 'No provider is set' do - let(:config) { YAML.load(<<~EOT - --- - :config: - :providers: - :mock_provider: - :pools: - - name: '#{pool}' - size: 1 - EOT - )} - - it 'should return nil' do - result = subject.get_domain_for_pool(pool) - expect(result).to be_nil - end - end - context 'No domain is set' do - let(:config) { YAML.load(<<~EOT - --- - :config: - :providers: - :mock_provider: - :pools: - - name: '#{pool}' - size: 1 - provider: #{provider_name} - EOT - )} - - it 'should return nil' do - allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) - allow(provider).to receive(:name).and_return(provider_name) - result = subject.get_domain_for_pool(pool) - expect(result).to be_nil - end - end - - context 'Only a global domain is set' do - let(:config) { YAML.load(<<~EOT - --- - :config: - domain: example.com - :providers: - :mock_provider: - :pools: - - name: '#{pool}' - size: 1 - provider: #{provider_name} - EOT - )} - - it 'should return the domain set in the config section' do - allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) - allow(provider).to receive(:name).and_return(provider_name) - result = subject.get_domain_for_pool(pool) - expect(result).to_not be_nil - expect(result).to eq('example.com') - end - end - - context 'A provider specified a domain to use' do - let(:config) { YAML.load(<<~EOT - --- - :config: - :providers: - :mock_provider: - domain: m.example.com - :pools: - - name: '#{pool}' - size: 1 - provider: #{provider_name} - EOT - )} - - it 'should return the domain set in the config section' do - allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) - allow(provider).to receive(:name).and_return(provider_name) - result = subject.get_domain_for_pool(pool) - expect(result).to_not be_nil - expect(result).to eq('m.example.com') - end - end - end - describe '#check_disk_queue' do let(:threads) {[]} diff --git a/spec/unit/util/parsing_spec.rb b/spec/unit/util/parsing_spec.rb new file mode 100644 index 0000000..3546600 --- /dev/null +++ b/spec/unit/util/parsing_spec.rb @@ -0,0 +1,104 @@ +require 'spec_helper' + +describe 'Parser' do + let(:pool) { 'pool1' } + subject { Vmpooler::Parsing } + describe '.get_domain_for_pool' do + let(:provider_name) { 'mock_provider' } + context 'No provider is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + EOT + )} + + it 'should return nil' do + result = subject.get_domain_for_pool(config, pool) + expect(result).to be_nil + end + end + context 'Provider is vsphere by default' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :vsphere: + domain: myown.com + :pools: + - name: '#{pool}' + size: 1 + EOT + )} + + it 'should return the domain set for vsphere' do + result = subject.get_domain_for_pool(config, pool) + expect(result).to eq('myown.com') + end + end + context 'No domain is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return nil' do + result = subject.get_domain_for_pool(config, pool) + expect(result).to be_nil + end + end + + context 'Only a global domain is set' do + let(:config) { YAML.load(<<~EOT + --- + :config: + domain: example.com + :providers: + :mock_provider: + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return the domain set in the config section' do + result = subject.get_domain_for_pool(config, pool) + expect(result).to_not be_nil + expect(result).to eq('example.com') + end + end + + context 'A provider specified a domain to use' do + let(:config) { YAML.load(<<~EOT + --- + :config: + :providers: + :mock_provider: + domain: m.example.com + :pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} + EOT + )} + + it 'should return the domain set in the config section' do + result = subject.get_domain_for_pool(config, pool) + expect(result).to_not be_nil + expect(result).to eq('m.example.com') + end + end + end +end From 66eb598e4ecf674170f8ede83f210cffe1f89f49 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 2 Feb 2022 13:30:54 -0600 Subject: [PATCH 03/20] tabling this PR for the next sprint --- lib/vmpooler/pool_manager.rb | 10 ++++- spec/integration/api/v2/ondemandvm_spec.rb | 1 - spec/integration/api/v2/vm_spec.rb | 33 +++++++------- spec/unit/pool_manager_spec.rb | 52 ++++++++++++++++++++++ 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index c38f436..993a4b9 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -368,7 +368,15 @@ module Vmpooler else fqdn = hostname end - dns_ip, dns_available = check_dns_available(fqdn) + + # skip dns check if the provider is set to skip_dns_check_before_creating_vm + provider = get_provider_for_pool(pool_name) + if config[:providers] && config[:providers][provider.name.to_sym] && config[:providers][provider.name.to_sym]['skip_dns_check_before_creating_vm'] + dns_available = true + else + dns_ip, dns_available = check_dns_available(fqdn) + end + break if hostname_available && dns_available hostname_retries += 1 diff --git a/spec/integration/api/v2/ondemandvm_spec.rb b/spec/integration/api/v2/ondemandvm_spec.rb index 9aeac3a..db0868b 100644 --- a/spec/integration/api/v2/ondemandvm_spec.rb +++ b/spec/integration/api/v2/ondemandvm_spec.rb @@ -125,7 +125,6 @@ describe Vmpooler::API::V2 do expected = { "ok": true, "request_id": uuid, - "domain": domain } expect(last_response.body).to eq(JSON.pretty_generate(expected)) end diff --git a/spec/integration/api/v2/vm_spec.rb b/spec/integration/api/v2/vm_spec.rb index 5dd30bf..d76f38a 100644 --- a/spec/integration/api/v2/vm_spec.rb +++ b/spec/integration/api/v2/vm_spec.rb @@ -24,10 +24,15 @@ describe Vmpooler::API::V2 do 'site_name' => 'test pooler', 'vm_lifetime_auth' => 2 }, + providers: { + vsphere: {'domain' => 'one.example.com'}, + gce: {'domain' => 'two.example.com'}, + foo: {'domain' => 'three.example.com'} + }, pools: [ - {'name' => 'pool1', 'size' => 5, 'domain' => 'one.example.com'}, - {'name' => 'pool2', 'size' => 10, 'domain' => 'two.example.com'}, - {'name' => 'pool3', 'size' => 10, 'domain' => 'three.example.com'} + {'name' => 'pool1', 'size' => 5, 'provider' => 'vsphere'}, + {'name' => 'pool2', 'size' => 10, 'provider' => 'gce'}, + {'name' => 'pool3', 'size' => 10, 'provider' => 'foo'} ], statsd: { 'prefix' => 'stats_prefix'}, alias: { 'poolone' => ['pool1'] }, @@ -80,8 +85,7 @@ describe Vmpooler::API::V2 do ok: true, pool1: { hostname: "#{vmname}.one.example.com" - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -99,8 +103,7 @@ describe Vmpooler::API::V2 do ok: true, poolone: { hostname: "#{vmname}.one.example.com" - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -150,7 +153,6 @@ describe Vmpooler::API::V2 do pool1: { hostname: "#{vmname}.one.example.com" }, - domain: 'two.example.com', pool2: { hostname: 'qrstuvwxyz012345.two.example.com' } @@ -253,8 +255,7 @@ describe Vmpooler::API::V2 do ok: true, "pool1": { "hostname": "1abcdefghijklmnop.one.example.com" - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -355,8 +356,7 @@ describe Vmpooler::API::V2 do ok: true, pool1: { hostname: "2#{vmname}.one.example.com" - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -381,8 +381,7 @@ describe Vmpooler::API::V2 do ok: true, pool1: { hostname: 'abcdefghijklmnop.one.example.com' - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -408,8 +407,7 @@ describe Vmpooler::API::V2 do ok: true, pool1: { hostname: 'abcdefghijklmnop.one.example.com' - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) @@ -430,8 +428,7 @@ describe Vmpooler::API::V2 do ok: true, pool1: { hostname: 'abcdefghijklmnop.one.example.com' - }, - domain: 'one.example.com' + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 6c1879c..df335fc 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -869,6 +869,58 @@ EOT end end + describe '#find_unique_hostname' do + # it should return the hostname + before do + allow(subject).to receive(:generate_and_check_hostname).and_return(["spicy-proton", true]) + allow(subject).to receive(:get_provider_for_pool).and_return(provider) + end + + context 'with a setting to skip dns check' do + let(:config) { + YAML.load(<<-EOT +--- +:providers: + :mock_provider: + skip_dns_check_before_creating_vm: true +:pools: + - name: '#{pool}' + size: 1 + EOT + ) + } + it 'should skip the dns check' do + #method is skipped + expect(subject).not_to receive(:check_dns_available) + expect(subject.find_unique_hostname(pool)).to eq("spicy-proton") + end + end + context 'without the setting to skip dns check' do + let(:config) { + YAML.load(<<-EOT +--- +:providers: + :mock_provider: +:pools: + - name: '#{pool}' + size: 1 + EOT + ) + } + it 'should run the dns check and pass' do + #method is skipped + expect(subject).to receive(:check_dns_available).and_return(["1.1.1.1",true]) + expect(subject.find_unique_hostname(pool)).to eq("spicy-proton") + end + + it 'should run the dns check and fail' do + #method is skipped + allow(subject).to receive(:check_dns_available).and_return(["1.1.1.1",false]).exactly(3).times + expect{subject.find_unique_hostname(pool)}.to raise_error(RuntimeError) + end + end + end + describe '#_clone_vm' do let (:pool_object) { { 'name' => pool } } let (:redis_ttl) { 1 } From dd375b20c3d06863d9a391819262009ce0d0b929 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Fri, 11 Feb 2022 13:10:08 -0600 Subject: [PATCH 04/20] document the new provider configuration skip_dns_check_before_creating_vm --- docs/configuration.md | 5 +---- lib/vmpooler/pool_manager.rb | 2 +- spec/integration/api/v1/vm_template_spec.rb | 1 + spec/integration/api/v2/vm_template_spec.rb | 1 + vmpooler.yaml.example | 14 ++++++++++---- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0bf331f..1c34c72 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,10 +21,7 @@ Path to a the file to use when loading the vmpooler configuration. This is only ### DOMAIN -If set, returns a top-level 'domain' JSON key in POST requests. Note that domains can now be set per-pool -and at the provider level. The domain key in the JSON response will correspond the last vm added to the -response and should not be relied upon as the hostnames of the returned vms now include the domain. The -domain key is only retained now for backwards compatibility and will be removed in the future. +This environment variable is to set the configuration config.domain (optional) ### REDIS\_SERVER diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 993a4b9..9ed2982 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -371,7 +371,7 @@ module Vmpooler # skip dns check if the provider is set to skip_dns_check_before_creating_vm provider = get_provider_for_pool(pool_name) - if config[:providers] && config[:providers][provider.name.to_sym] && config[:providers][provider.name.to_sym]['skip_dns_check_before_creating_vm'] + if provider && provider.provider_config['skip_dns_check_before_creating_vm'] dns_available = true else dns_ip, dns_available = check_dns_available(fqdn) diff --git a/spec/integration/api/v1/vm_template_spec.rb b/spec/integration/api/v1/vm_template_spec.rb index 25c0fbd..83097e8 100644 --- a/spec/integration/api/v1/vm_template_spec.rb +++ b/spec/integration/api/v1/vm_template_spec.rb @@ -24,6 +24,7 @@ describe Vmpooler::API::V1 do 'site_name' => 'test pooler', 'vm_lifetime_auth' => 2, }, + providers: { vsphere: {} }, pools: [ {'name' => 'pool1', 'size' => 5}, {'name' => 'pool2', 'size' => 10}, diff --git a/spec/integration/api/v2/vm_template_spec.rb b/spec/integration/api/v2/vm_template_spec.rb index dc46c40..1a487a4 100644 --- a/spec/integration/api/v2/vm_template_spec.rb +++ b/spec/integration/api/v2/vm_template_spec.rb @@ -24,6 +24,7 @@ describe Vmpooler::API::V2 do 'site_name' => 'test pooler', 'vm_lifetime_auth' => 2, }, + providers: { vsphere: {} }, pools: [ {'name' => 'pool1', 'size' => 5}, {'name' => 'pool2', 'size' => 10}, diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 1d8591e..a381bf9 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -28,6 +28,12 @@ # Expects an array of strings specifying the allowed items by name # (optional; default: nil) # +# - skip_dns_check_before_creating_vm +# Setting this configuration parameter in a provider will make vmpooler skip the check it normally does for a DNS +# record. The normal behavior is to check DNS and if a record already exists (conflict) to re-generate a new hostname. +# By using this configuration parameter, it will skip the check and continue with the same conflicting name. This is +# useful for providers that can handle that case, for instance by replacing the existing DNS record with a new one. +# # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify # the provider to use. @@ -440,10 +446,10 @@ # If set, restricts tags to those specified in this array. # # - domain -# If set, returns a top-level 'domain' JSON key in POST requests. Note that domains can now be set per-pool -# and at the provider level. The domain key in the JSON response will correspond the last vm added to the -# response and should not be relied upon as the hostnames of the returned vms now include the domain. The -# domain key is only retained now for backwards compatibility and will be removed in the future. +# If set, returns a top-level 'domain' JSON key in POST requests (v1). +# in the v2 API, it concatenates the hostname to include the domain, either set here or at the provider level +# With the introduction of vmpooler-provider-gce, domains can now be set at the provider level. The +# domain key is only retained in v1 for backwards compatibility and will be removed in the future. # # - prefix # If set, prefixes all created VMs with this string. This should include From a2d613782a5cc4b1c4c1307e1f59b9d6ce2c0b87 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Tue, 29 Mar 2022 12:01:17 -0500 Subject: [PATCH 05/20] remove 'last minute' tcp vm_ready check from the api for pooled vms the vm_ready methods should be implemented per provider and checking in the api creates issues since provider code is not available. Removing this for the V2 api --- lib/vmpooler/api/v2.rb | 17 +++++--------- spec/integration/api/v1/vm_spec.rb | 3 +-- spec/integration/api/v2/vm_spec.rb | 25 +-------------------- spec/integration/api/v2/vm_template_spec.rb | 4 ++-- 4 files changed, 9 insertions(+), 40 deletions(-) diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index 4cd1a66..d2c17f9 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -73,19 +73,12 @@ module Vmpooler next if vms.empty? vms.reverse.each do |vm| - vm_domain = Parsing.get_domain_for_pool(full_config, template_backend) - ready = vm_ready?(vm, vm_domain) - if ready - smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) - if smoved - return [vm, template_backend, template] - else - metrics.increment("checkout.smove.failed.#{template_backend}") - return [nil, nil, nil] - end + smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) + if smoved + return [vm, template_backend, template] else - backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) - metrics.increment("checkout.nonresponsive.#{template_backend}") + metrics.increment("checkout.smove.failed.#{template_backend}") + return [nil, nil, nil] end end end diff --git a/spec/integration/api/v1/vm_spec.rb b/spec/integration/api/v1/vm_spec.rb index 0d06585..88d8b57 100644 --- a/spec/integration/api/v1/vm_spec.rb +++ b/spec/integration/api/v1/vm_spec.rb @@ -124,8 +124,7 @@ describe Vmpooler::API::V1 do end it 'returns 503 for empty pool referenced by alias' do - create_ready_vm 'pool1', vmname, redis - post "#{prefix}/vm/poolone" + create_ready_vm 'pool2', vmname, redis post "#{prefix}/vm/poolone" expected = { ok: false } diff --git a/spec/integration/api/v2/vm_spec.rb b/spec/integration/api/v2/vm_spec.rb index d76f38a..b971ba8 100644 --- a/spec/integration/api/v2/vm_spec.rb +++ b/spec/integration/api/v2/vm_spec.rb @@ -129,8 +129,7 @@ describe Vmpooler::API::V2 do end it 'returns 503 for empty pool referenced by alias' do - create_ready_vm 'pool1', vmname, redis - post "#{prefix}/vm/poolone" + create_ready_vm 'pool2', vmname, redis post "#{prefix}/vm/poolone" expected = { ok: false } @@ -342,28 +341,6 @@ describe Vmpooler::API::V2 do expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop', redis)).to eq(true) end - it 'returns the second VM when the first fails to respond' do - create_ready_vm 'pool1', vmname, redis - create_ready_vm 'pool1', "2#{vmname}", redis - - allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with(vmname, 'one.example.com').and_raise('mockerror') - allow_any_instance_of(Vmpooler::API::Helpers).to receive(:open_socket).with("2#{vmname}", 'one.example.com').and_return(socket) - - post "#{prefix}/vm", '{"pool1":"1"}' - expect_json(ok = true, http = 200) - - expected = { - ok: true, - pool1: { - hostname: "2#{vmname}.one.example.com" - } - } - - expect(last_response.body).to eq(JSON.pretty_generate(expected)) - - expect(pool_has_ready_vm?('pool1', vmname, redis)).to be false - end - context '(auth not configured)' do it 'does not extend VM lifetime if auth token is provided' do app.settings.set :config, auth: false diff --git a/spec/integration/api/v2/vm_template_spec.rb b/spec/integration/api/v2/vm_template_spec.rb index 1a487a4..862df25 100644 --- a/spec/integration/api/v2/vm_template_spec.rb +++ b/spec/integration/api/v2/vm_template_spec.rb @@ -28,7 +28,7 @@ describe Vmpooler::API::V2 do pools: [ {'name' => 'pool1', 'size' => 5}, {'name' => 'pool2', 'size' => 10}, - {'name' => 'poolone', 'size' => 0} + {'name' => 'poolone', 'size' => 1} ], statsd: { 'prefix' => 'stats_prefix'}, alias: { 'poolone' => 'pool1' }, @@ -105,7 +105,7 @@ describe Vmpooler::API::V2 do end it 'returns 503 for empty pool referenced by alias' do - create_ready_vm 'pool1', 'abcdefghijklmnop', redis + create_ready_vm 'pool2', 'abcdefghijklmnop', redis post "#{prefix}/vm/poolone" expected = { ok: false } From 70b5bd297aabb6466071d7d5cec0c254199ddd2a Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Tue, 29 Mar 2022 12:45:36 -0500 Subject: [PATCH 06/20] add multi domain support to ondemand queries --- lib/vmpooler/api/v2.rb | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index d2c17f9..c8b01a3 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -329,6 +329,78 @@ module Vmpooler JSON.pretty_generate(result) end + get "#{api_prefix}/ondemandvm/:requestid/?" do + content_type :json + metrics.increment('http_requests_vm_total.get.ondemand.request') + + status 404 + result = check_ondemand_request(params[:requestid]) + + JSON.pretty_generate(result) + end + + def check_ondemand_request(request_id) + tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| + span.set_attribute('vmpooler.request_id', request_id) + result = { 'ok' => false } + request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") + if request_hash.empty? + e_message = "no request found for request_id '#{request_id}'" + result['message'] = e_message + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V2.check_ondemand_request', + 'error.message' => e_message + }) + return result + end + + result['request_id'] = request_id + result['ready'] = false + result['ok'] = true + status 202 + + case request_hash['status'] + when 'ready' + result['ready'] = true + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| + instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + domain = Parsing.get_domain_for_pool(full_config, pool) + instances.map!{ |instance| instance.concat(".#{domain}") } if domain + + if result.key?(platform_alias) + result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances + else + result[platform_alias] = { 'hostname': instances } + end + end + status 200 + when 'failed' + result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" + status 200 + when 'deleted' + result['message'] = 'The request has been deleted' + status 200 + else + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| + instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + instances_pending = count.to_i - instance_count.to_i + + if result.key?(platform_alias) && result[platform_alias].key?(:ready) + result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s + result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s + else + result[platform_alias] = { + 'ready': instance_count.to_s, + 'pending': instances_pending.to_s + } + end + end + end + + result + end + end + # Endpoints that only use bits from the V1 api are called here # Note that traces will be named based on the route used in the V1 api # but the http.url trace attribute will still have the actual requested url in it From 2e608b71964f6e8d0a4132c8f1d489ff888edc75 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Tue, 29 Mar 2022 13:32:23 -0500 Subject: [PATCH 07/20] spec test domain set at the config and provider levels --- spec/integration/api/v2/ondemandvm_spec.rb | 37 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/spec/integration/api/v2/ondemandvm_spec.rb b/spec/integration/api/v2/ondemandvm_spec.rb index db0868b..5d6d887 100644 --- a/spec/integration/api/v2/ondemandvm_spec.rb +++ b/spec/integration/api/v2/ondemandvm_spec.rb @@ -37,7 +37,12 @@ describe Vmpooler::API::V2 do 'poolone' => ['pool1'], 'pool2' => ['pool1'] }, - pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ] + pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ], + providers: { + :dummy => { + 'domain' => 'dummy.com' + } + } } } let(:current_time) { Time.now } @@ -116,7 +121,6 @@ describe Vmpooler::API::V2 do let(:domain) { 'example.com' } before(:each) do config[:config]['domain'] = domain - config[:pools][0]['domain'] = domain end it 'should include domain in the return reply' do @@ -275,7 +279,7 @@ describe Vmpooler::API::V2 do config[:config]['domain'] = domain end - it 'should include the domain in the result' do + it 'should include the domain in the hostname as fqdn, not a separate key unlike in v1' do get "#{prefix}/ondemandvm/#{uuid}" expected = { "ok": true, @@ -283,10 +287,31 @@ describe Vmpooler::API::V2 do "ready": true, "pool1": { "hostname": [ - vmname + "#{vmname}.#{domain}" ] - }, - "domain": domain + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + context 'with domain set in the provider' do + let(:domain) { 'dummy.com' } + before(:each) do + config[:pools][0]['provider'] = 'dummy' + end + + it 'should include the domain in the hostname as fqdn, not a separate key unlike in v1' do + get "#{prefix}/ondemandvm/#{uuid}" + expected = { + "ok": true, + "request_id": uuid, + "ready": true, + "pool1": { + "hostname": [ + "#{vmname}.#{domain}" + ] + } } expect(last_response.body).to eq(JSON.pretty_generate(expected)) end From 3809dac2d4900e701aabb3729f38c7672a52a551 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 30 Mar 2022 08:42:24 -0500 Subject: [PATCH 08/20] fix rubocop and fetch_single_vm logic, bump version --- lib/vmpooler/api/v2.rb | 15 +++++---------- lib/vmpooler/version.rb | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index c8b01a3..148488b 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -71,15 +71,10 @@ module Vmpooler template_backends.each do |template_backend| vms = backend.smembers("vmpooler__ready__#{template_backend}") next if vms.empty? - - vms.reverse.each do |vm| - smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) - if smoved - return [vm, template_backend, template] - else - metrics.increment("checkout.smove.failed.#{template_backend}") - return [nil, nil, nil] - end + vm = vms.pop + smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) + if smoved + return [vm, template_backend, template] end end [nil, nil, nil] @@ -365,7 +360,7 @@ module Vmpooler Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") domain = Parsing.get_domain_for_pool(full_config, pool) - instances.map!{ |instance| instance.concat(".#{domain}") } if domain + instances.map! { |instance| instance.concat(".#{domain}") } if domain if result.key?(platform_alias) result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances diff --git a/lib/vmpooler/version.rb b/lib/vmpooler/version.rb index 00af653..53179fc 100644 --- a/lib/vmpooler/version.rb +++ b/lib/vmpooler/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Vmpooler - VERSION = '2.2.0' + VERSION = '2.3.0' end From 7c5a3f802e7b1c7135ae4454354101a3339da5a7 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 30 Mar 2022 09:17:25 -0500 Subject: [PATCH 09/20] update gemfile.lock --- Gemfile.lock | 62 ++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b8fe64a..219973d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - vmpooler (2.2.0) + vmpooler (2.3.0) concurrent-ruby (~> 1.1) connection_pool (~> 2.2) deep_merge (~> 1.2) @@ -32,12 +32,12 @@ GEM builder (3.2.4) climate_control (1.0.1) coderay (1.1.3) - concurrent-ruby (1.1.9) + concurrent-ruby (1.1.10) connection_pool (2.2.5) deep_merge (1.2.2) diff-lcs (1.5.0) docile (1.4.0) - faraday (1.9.3) + faraday (1.10.0) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -61,10 +61,10 @@ GEM faraday-rack (1.0.0) faraday-retry (1.0.3) ffi (1.15.5-java) - google-cloud-env (1.5.0) - faraday (>= 0.17.3, < 2.0) + google-cloud-env (1.6.0) + faraday (>= 0.17.3, < 3.0) method_source (1.0.0) - mock_redis (0.29.0) + mock_redis (0.30.0) ruby2_keywords multipart-post (2.1.1) mustermann (1.1.1) @@ -108,8 +108,8 @@ GEM opentelemetry-semantic_conventions opentelemetry-semantic_conventions (1.8.0) opentelemetry-api (~> 1.0) - parallel (1.21.0) - parser (3.1.0.0) + parallel (1.22.1) + parser (3.1.1.0) ast (~> 2.4.1) pickup (0.0.11) prometheus-client (2.1.0) @@ -117,32 +117,32 @@ GEM coderay (~> 1.1) method_source (~> 1.0) spoon (~> 0.0) - puma (5.5.2-java) + puma (5.6.2-java) nio4r (~> 2.0) racc (1.6.0-java) rack (2.2.3) - rack-protection (2.1.0) + rack-protection (2.2.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) rainbow (3.1.1) rake (13.0.6) - redis (4.5.1) - regexp_parser (2.2.0) + redis (4.6.0) + regexp_parser (2.2.1) rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.2) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.3) + rspec-support (~> 3.11.0) + rspec-support (3.11.0) rubocop (1.1.0) parallel (~> 1.10) parser (>= 2.7.1.5) @@ -152,8 +152,8 @@ GEM rubocop-ast (>= 1.0.1) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.15.1) - parser (>= 3.0.1.1) + rubocop-ast (1.16.0) + parser (>= 3.1.1.0) ruby-progressbar (1.11.0) ruby2_keywords (0.0.5) simplecov (0.21.2) @@ -161,19 +161,19 @@ GEM simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.3) - sinatra (2.1.0) + simplecov_json_formatter (0.1.4) + sinatra (2.2.0) mustermann (~> 1.0) rack (~> 2.2) - rack-protection (= 2.1.0) + rack-protection (= 2.2.0) tilt (~> 2.0) - spicy-proton (2.1.13) + spicy-proton (2.1.14) bindata (~> 2.3) spoon (0.0.6) ffi statsd-ruby (1.5.0) thor (1.2.1) - thrift (0.15.0) + thrift (0.16.0) tilt (2.0.10) unicode-display_width (1.8.0) yarjuf (2.0.0) @@ -196,4 +196,4 @@ DEPENDENCIES yarjuf (>= 2.0) BUNDLED WITH - 2.3.5 + 2.3.10 From 1005a33ed2b220ec31886a93d49758a4d023907c Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 30 Mar 2022 09:33:29 -0500 Subject: [PATCH 10/20] fix rubocop --- lib/vmpooler/api/v2.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index 148488b..57f75ff 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -71,6 +71,7 @@ module Vmpooler template_backends.each do |template_backend| vms = backend.smembers("vmpooler__ready__#{template_backend}") next if vms.empty? + vm = vms.pop smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) if smoved From 9a9dfce31691aa45157723c883df0c4d25f43fec Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 30 Mar 2022 12:50:12 -0500 Subject: [PATCH 11/20] adding an api endpoint to print the current full config --- lib/vmpooler/api/v1.rb | 18 ++++++++++++++++++ lib/vmpooler/api/v2.rb | 2 +- spec/spec_helper.rb | 2 -- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index c11b2cc..e333afd 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -24,6 +24,10 @@ module Vmpooler Vmpooler::API.settings.config[:config] end + def full_config + Vmpooler::API.settings.config + end + def pools Vmpooler::API.settings.config[:pools] end @@ -1734,6 +1738,20 @@ module Vmpooler end JSON.pretty_generate(result) end + + get "#{api_prefix}/full_config/?" do + content_type :json + + result = { + full_config: full_config, + status: { + ok: true + } + } + + status 200 + JSON.pretty_generate(result) + end end end end diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index 57f75ff..b39722d 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -71,7 +71,7 @@ module Vmpooler template_backends.each do |template_backend| vms = backend.smembers("vmpooler__ready__#{template_backend}") next if vms.empty? - + vm = vms.pop smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) if smoved diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 875596b..dc94ee7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,9 +1,7 @@ require 'simplecov' -=begin SimpleCov.start do add_filter '/spec/' end -=end require 'helpers' require 'rspec' require 'vmpooler' From ef87fe8db53fa97a831de0adf36724cc3d80daa6 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Thu, 31 Mar 2022 13:10:38 -0500 Subject: [PATCH 12/20] adding deprecation warning to logs for reroute.rb --- lib/vmpooler/api/reroute.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/vmpooler/api/reroute.rb b/lib/vmpooler/api/reroute.rb index bc44106..a8c66fa 100644 --- a/lib/vmpooler/api/reroute.rb +++ b/lib/vmpooler/api/reroute.rb @@ -6,66 +6,82 @@ module Vmpooler api_version = '1' get '/status/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /status/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/status") end get '/summary/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/summary") end get '/summary/:route/?:key?/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/:route/?:key?/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}") end get '/token/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/token") end post '/token/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/token") end get '/token/:token/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}") end delete '/token/:token/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}") end get '/vm/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm") end post '/vm/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm") end post '/vm/:template/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:template/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}") end get '/vm/:hostname/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") end delete '/vm/:hostname/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") end put '/vm/:hostname/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") end post '/vm/:hostname/snapshot/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot") end post '/vm/:hostname/snapshot/:snapshot/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/:snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}") end put '/vm/:hostname/disk/:size/?' do + puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/disk/:size/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}") end end From 57542d775b3752829892f5e8401ec17fe5087fc2 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Thu, 31 Mar 2022 14:26:36 -0500 Subject: [PATCH 13/20] update docs, remove domain key --- docs/API-v2.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/API-v2.md b/docs/API-v2.md index 5b73230..54a83a6 100644 --- a/docs/API-v2.md +++ b/docs/API-v2.md @@ -12,6 +12,21 @@ vmpooler provides a REST API for VM management. The following examples use `curl` for communication. +## Major change in V2 versus V1 + +The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider, +the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each provider instead of only one. + +Clients using some of the direct API paths (without specifying api/v1 or api/v2) will still be redirected to v1, but this behavior is notw deprecated and will be changed to v2 in the next major version. + +### updavig clients from v1 to v2 + +Clients need to update their paths to using api/v2 instead of api/v1. Please note the API responses that used to return a "domain" key, will no longer have a separate "domain" key and now return +the FQDN (includes the domain in the hostname). + +One way to support both v1 and v2 in the client logic is to look for a "domain" and append it to the hostname if it exists (existing v1 behavior). If the "domain" key does not exist, you can use the hostname +as is since it is a FQDN (v2 behavor). + #### Token operations Token-based authentication can be used when requesting or modifying VMs. The `/token` route can be used to create, query, or delete tokens. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations. @@ -149,8 +164,7 @@ $ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.c }, "debian-7-x86_64": { "hostname": "y91qbrpbfj6d13q.example.com" - }, - "domain": "example.com" + } } ``` @@ -173,8 +187,7 @@ $ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386 "ok": true, "debian-7-i386": { "hostname": "fq6qlpjlsskycq6.example.com" - }, - "domain": "example.com" + } } ``` @@ -195,8 +208,7 @@ $ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386+debian-7-i386+debia }, "debian-7-x86_64": { "hostname": "zb91y9qbrbf6d3q.example.com" - }, - "domain": "example.com" + } } ``` @@ -227,7 +239,6 @@ $ curl --url vmpooler.example.com/api/v2/vm/pxpmtoonx7fiqg6 "user": "jdoe" }, "ip": "192.168.0.1", - "domain": "example.com", "host": "host1.example.com", "migrated": "true" } @@ -329,8 +340,7 @@ $ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 "state": "running", "disk": [ "+8gb" - ], - "domain": "delivery.puppetlabs.net" + ] } } @@ -374,8 +384,7 @@ $ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 "state": "running", "snapshots": [ "n4eb4kdtp7rwv4x158366vd9jhac8btq" - ], - "domain": "delivery.puppetlabs.net" + ] } } ```` From 09bc406c9f502750c566d314a4fe1fb50707e64f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:04:03 +0000 Subject: [PATCH 14/20] Update opentelemetry-instrumentation-http_client requirement from = 0.19.3 to = 0.19.4 Updates the requirements on [opentelemetry-instrumentation-http_client](https://github.com/open-telemetry/opentelemetry-ruby) to permit the latest version. - [Release notes](https://github.com/open-telemetry/opentelemetry-ruby/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-ruby/blob/main/instrumentation/http_client/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-ruby/compare/opentelemetry-instrumentation-http_client/v0.19.3...opentelemetry-instrumentation-http_client/v0.19.4) --- updated-dependencies: - dependency-name: opentelemetry-instrumentation-http_client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 14 ++++++++++++-- vmpooler.gemspec | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 219973d..a1ebf94 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ PATH nokogiri (~> 1.10) opentelemetry-exporter-jaeger (= 0.20.1) opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) - opentelemetry-instrumentation-http_client (= 0.19.3) + opentelemetry-instrumentation-http_client (= 0.19.4) opentelemetry-instrumentation-redis (= 0.21.2) opentelemetry-instrumentation-sinatra (= 0.19.3) opentelemetry-resource_detectors (= 0.19.1) @@ -70,9 +70,12 @@ GEM mustermann (1.1.1) ruby2_keywords (~> 0.0.1) net-ldap (0.17.0) + nio4r (2.5.8) nio4r (2.5.8-java) nokogiri (1.12.5-java) racc (~> 1.4) + nokogiri (1.12.5-x86_64-linux) + racc (~> 1.4) opentelemetry-api (1.0.1) opentelemetry-common (0.19.3) opentelemetry-api (~> 1.0) @@ -86,7 +89,7 @@ GEM opentelemetry-instrumentation-concurrent_ruby (0.19.2) opentelemetry-api (~> 1.0) opentelemetry-instrumentation-base (~> 0.19.0) - opentelemetry-instrumentation-http_client (0.19.3) + opentelemetry-instrumentation-http_client (0.19.4) opentelemetry-api (~> 1.0) opentelemetry-common (~> 0.19.3) opentelemetry-instrumentation-base (~> 0.19.0) @@ -113,12 +116,18 @@ GEM ast (~> 2.4.1) pickup (0.0.11) prometheus-client (2.1.0) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) pry (0.14.1-java) coderay (~> 1.1) method_source (~> 1.0) spoon (~> 0.0) + puma (5.6.2) + nio4r (~> 2.0) puma (5.6.2-java) nio4r (~> 2.0) + racc (1.6.0) racc (1.6.0-java) rack (2.2.3) rack-protection (2.2.0) @@ -182,6 +191,7 @@ GEM PLATFORMS universal-java-1.8 + x86_64-linux DEPENDENCIES climate_control (>= 0.2.0) diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 49e2375..3c903a9 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.add_dependency 'nokogiri', '~> 1.10' s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.20.1' s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.19.2' - s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.19.3' + s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.19.4' s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.21.2' s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.19.3' s.add_dependency 'opentelemetry-resource_detectors', '= 0.19.1' From a3a6cf05333a1e856bef01052e639b709961e802 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:04:15 +0000 Subject: [PATCH 15/20] Bump puma from 5.5.2 to 5.6.4 Bumps [puma](https://github.com/puma/puma) from 5.5.2 to 5.6.4. - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v5.5.2...v5.6.4) --- updated-dependencies: - dependency-name: puma dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 219973d..335c0ba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -70,9 +70,12 @@ GEM mustermann (1.1.1) ruby2_keywords (~> 0.0.1) net-ldap (0.17.0) + nio4r (2.5.8) nio4r (2.5.8-java) nokogiri (1.12.5-java) racc (~> 1.4) + nokogiri (1.12.5-x86_64-linux) + racc (~> 1.4) opentelemetry-api (1.0.1) opentelemetry-common (0.19.3) opentelemetry-api (~> 1.0) @@ -113,12 +116,18 @@ GEM ast (~> 2.4.1) pickup (0.0.11) prometheus-client (2.1.0) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) pry (0.14.1-java) coderay (~> 1.1) method_source (~> 1.0) spoon (~> 0.0) - puma (5.6.2-java) + puma (5.6.4) nio4r (~> 2.0) + puma (5.6.4-java) + nio4r (~> 2.0) + racc (1.6.0) racc (1.6.0-java) rack (2.2.3) rack-protection (2.2.0) @@ -182,6 +191,7 @@ GEM PLATFORMS universal-java-1.8 + x86_64-linux DEPENDENCIES climate_control (>= 0.2.0) From 6aa10151ca0debe68442cdf8adc74462d3c353ec Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 29 Jun 2022 13:57:48 -0500 Subject: [PATCH 16/20] (DIO-3138) vmpooler v2 api missing vm/hostname there was one API that was falling back on v1 and was returning a domain key. in order to make it consistent with the changes in v2, the domain is not returned anymore, and the fqdn is returned if it is available --- lib/vmpooler/api/helpers.rb | 9 +---- lib/vmpooler/api/v1.rb | 2 +- lib/vmpooler/api/v2.rb | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 6d1a347..cb983cb 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -181,14 +181,9 @@ module Vmpooler /^\d{4}-\d{2}-\d{2}$/ === date_str end + # note: domain is not needed here, so we should update the callers of this method def hostname_shorten(hostname, domain=nil) - if domain && hostname =~ /^[\w-]+\.#{domain}$/ - hostname = hostname[/[^.]+/] - elsif hostname =~ /^[\w-]+\..+$/ - hostname = hostname[/[^.]+/] - end - - hostname + hostname[/[^.]+/] end def get_task_times(backend, task, date_str) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index e333afd..c6e7d9b 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -1519,7 +1519,7 @@ module Vmpooler post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do content_type :json - metrics.increment('http_requests_vm_total.post.vm.disksize') + metrics.increment('http_requests_vm_total.post.vm.snapshot') need_token! if Vmpooler::API.settings.config[:auth] diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb index b39722d..44dad6e 100644 --- a/lib/vmpooler/api/v2.rb +++ b/lib/vmpooler/api/v2.rb @@ -258,6 +258,82 @@ module Vmpooler JSON.pretty_generate(result) end + get "#{api_prefix}/vm/:hostname/?" do + content_type :json + metrics.increment('http_requests_vm_total.get.vm.hostname') + + result = {} + + status 404 + result['ok'] = false + + params[:hostname] = hostname_shorten(params[:hostname], nil) + + rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") + unless rdata.empty? + status 200 + result['ok'] = true + + result[params[:hostname]] = {} + + result[params[:hostname]]['template'] = rdata['template'] + result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i + + if rdata['destroy'] + result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout'] + result[params[:hostname]]['state'] = 'destroyed' + elsif rdata['checkout'] + result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2) + result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2) + result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339 + result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339 + result[params[:hostname]]['state'] = 'running' + elsif rdata['check'] + result[params[:hostname]]['state'] = 'ready' + else + result[params[:hostname]]['state'] = 'pending' + end + + rdata.keys.each do |key| + if key.match('^tag\:(.+?)$') + result[params[:hostname]]['tags'] ||= {} + result[params[:hostname]]['tags'][$1] = rdata[key] + end + + if key.match('^snapshot\:(.+?)$') + result[params[:hostname]]['snapshots'] ||= [] + result[params[:hostname]]['snapshots'].push($1) + end + end + + if rdata['disk'] + result[params[:hostname]]['disk'] = rdata['disk'].split(':') + end + + # Look up IP address of the hostname + begin + ipAddress = TCPSocket.gethostbyname(params[:hostname])[3] + rescue StandardError + ipAddress = "" + end + + result[params[:hostname]]['ip'] = ipAddress + + if rdata['pool'] + vmdomain = Parsing.get_domain_for_pool(full_config, rdata['pool']) + if vmdomain + result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}" + end + end + + result[params[:hostname]]['host'] = rdata['host'] if rdata['host'] + result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated'] + + end + + JSON.pretty_generate(result) + end + post "#{api_prefix}/ondemandvm/?" do content_type :json metrics.increment('http_requests_vm_total.post.ondemand.requestid') From f5866d51b647c097d54bb22e55690341f24c3824 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 13:02:32 +0000 Subject: [PATCH 17/20] Update rubocop requirement from ~> 1.1.0 to ~> 1.28.2 Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.1.0...v1.28.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 20 ++++++++++---------- vmpooler.gemspec | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 335c0ba..f138766 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -112,7 +112,7 @@ GEM opentelemetry-semantic_conventions (1.8.0) opentelemetry-api (~> 1.0) parallel (1.22.1) - parser (3.1.1.0) + parser (3.1.2.0) ast (~> 2.4.1) pickup (0.0.11) prometheus-client (2.1.0) @@ -137,7 +137,7 @@ GEM rainbow (3.1.1) rake (13.0.6) redis (4.6.0) - regexp_parser (2.2.1) + regexp_parser (2.3.1) rexml (3.2.5) rspec (3.11.0) rspec-core (~> 3.11.0) @@ -152,16 +152,16 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.11.0) rspec-support (3.11.0) - rubocop (1.1.0) + rubocop (1.28.2) parallel (~> 1.10) - parser (>= 2.7.1.5) + parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8) + regexp_parser (>= 1.8, < 3.0) rexml - rubocop-ast (>= 1.0.1) + rubocop-ast (>= 1.17.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.16.0) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.17.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) ruby2_keywords (0.0.5) @@ -184,7 +184,7 @@ GEM thor (1.2.1) thrift (0.16.0) tilt (2.0.10) - unicode-display_width (1.8.0) + unicode-display_width (2.1.0) yarjuf (2.0.0) builder rspec (~> 3) @@ -199,7 +199,7 @@ DEPENDENCIES pry rack-test (>= 0.6) rspec (>= 3.2) - rubocop (~> 1.1.0) + rubocop (~> 1.28.2) simplecov (>= 0.11.2) thor (~> 1.0, >= 1.0.1) vmpooler! diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 49e2375..cee2a74 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -45,7 +45,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'pry' s.add_development_dependency 'rack-test', '>= 0.6' s.add_development_dependency 'rspec', '>= 3.2' - s.add_development_dependency 'rubocop', '~> 1.1.0' + s.add_development_dependency 'rubocop', '~> 1.28.2' s.add_development_dependency 'simplecov', '>= 0.11.2' s.add_development_dependency 'thor', '~> 1.0', '>= 1.0.1' s.add_development_dependency 'yarjuf', '>= 2.0' From c846e41780974c1f3f414386f33810814b2fec32 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Mon, 25 Jul 2022 08:59:12 -0500 Subject: [PATCH 18/20] fix rubocoop offences --- .rubocop.yml | 1 + lib/vmpooler/pool_manager.rb | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 2d50ca0..0fe1eff 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -66,6 +66,7 @@ Style/Next: Enabled: false Metrics/ParameterLists: Max: 10 + MaxOptionalParameters: 10 Style/GuardClause: Enabled: false diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 9ed2982..95922d5 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -1596,19 +1596,19 @@ module Vmpooler $config[:pools].each do |pool| provider_name = pool['provider'] # The provider_class parameter can be defined in the provider's data eg - #:providers: - # :vsphere: - # provider_class: 'vsphere' - # :another-vsphere: - # provider_class: 'vsphere' + # :providers: + # :vsphere: + # provider_class: 'vsphere' + # :another-vsphere: + # provider_class: 'vsphere' # the above would create two providers/vsphere.rb class objects named 'vsphere' and 'another-vsphere' # each pools would then define which provider definition to use: vsphere or another-vsphere # # if provider_class is not defined it will try to use the provider_name as the class, this is to be # backwards compatible for example when there is only one provider listed # :providers: - # :dummy: - # filename: 'db.txs' + # :dummy: + # filename: 'db.txs' # the above example would create an object based on the class providers/dummy.rb if $config[:providers].nil? || $config[:providers][provider_name.to_sym].nil? || $config[:providers][provider_name.to_sym]['provider_class'].nil? provider_class = provider_name From 7f1f8def8e05b1561ccbffa8e2206790e588f337 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Mon, 25 Jul 2022 09:12:12 -0500 Subject: [PATCH 19/20] fix comment offence --- lib/vmpooler/api/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index cb983cb..136b5f1 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -181,7 +181,7 @@ module Vmpooler /^\d{4}-\d{2}-\d{2}$/ === date_str end - # note: domain is not needed here, so we should update the callers of this method + # NOTE: domain is not needed here, so we should update the callers of this method def hostname_shorten(hostname, domain=nil) hostname[/[^.]+/] end From febca3a9b79c3e5b96ffce80cd4e001b2715d4cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:08:05 +0000 Subject: [PATCH 20/20] Bump mock_redis from 0.30.0 to 0.31.0 Bumps [mock_redis](https://github.com/sds/mock_redis) from 0.30.0 to 0.31.0. - [Release notes](https://github.com/sds/mock_redis/releases) - [Changelog](https://github.com/sds/mock_redis/blob/main/CHANGELOG.md) - [Commits](https://github.com/sds/mock_redis/compare/v0.30.0...v0.31.0) --- updated-dependencies: - dependency-name: mock_redis dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f138766..fae20e0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -64,7 +64,7 @@ GEM google-cloud-env (1.6.0) faraday (>= 0.17.3, < 3.0) method_source (1.0.0) - mock_redis (0.30.0) + mock_redis (0.31.0) ruby2_keywords multipart-post (2.1.1) mustermann (1.1.1)