diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 4f0d4d3..7046eb6 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -12,8 +12,7 @@ # SupportedStyles: with_first_parameter, with_fixed_indentation Layout/ParameterAlignment: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 9 # Cop supports --auto-correct. @@ -22,15 +21,13 @@ Layout/ParameterAlignment: Layout/CaseIndentation: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. Layout/ClosingParenthesisIndentation: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -61,16 +58,14 @@ Layout/EmptyLinesAroundModuleBody: Layout/FirstHashElementIndentation: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: Width, IgnoredPatterns. Layout/IndentationWidth: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -78,8 +73,7 @@ Layout/IndentationWidth: # SupportedStyles: symmetrical, new_line, same_line Layout/MultilineMethodCallBraceLayout: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -93,16 +87,14 @@ Layout/SpaceAroundEqualsInParameterDefault: # Cop supports --auto-correct. Layout/SpaceAroundKeyword: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: AllowForAlignment. Layout/SpaceAroundOperators: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 8 # Cop supports --auto-correct. @@ -117,16 +109,14 @@ Layout/SpaceInsideHashLiteralBraces: # Cop supports --auto-correct. Layout/SpaceInsideParens: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 2 # Configuration parameters: AllowSafeAssignment. Lint/AssignmentInCondition: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 2 Lint/SuppressedException: @@ -158,8 +148,7 @@ Lint/UselessAssignment: Style/AndOr: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 Style/CaseEquality: @@ -180,8 +169,7 @@ Style/For: Style/HashSyntax: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 4 # Cop supports --auto-correct. @@ -189,8 +177,7 @@ Style/HashSyntax: Style/IfUnlessModifier: Exclude: - 'lib/vmpooler/api/helpers.rb' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 3 # Cop supports --auto-correct. @@ -198,15 +185,13 @@ Style/IfUnlessModifier: # SupportedStyles: both, prefix, postfix Style/NegatedIf: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 3 # Cop supports --auto-correct. Style/Not: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -215,30 +200,26 @@ Style/Not: Style/NumericPredicate: Exclude: - 'spec/**/*' - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 2 # Cop supports --auto-correct. Style/ParallelAssignment: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: AllowSafeAssignment. Style/ParenthesesAroundCondition: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 2 # Cop supports --auto-correct. Style/PerlBackrefs: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist. @@ -254,8 +235,7 @@ Naming/PredicateName: # Cop supports --auto-correct. Style/RedundantParentheses: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 2 # Cop supports --auto-correct. @@ -276,8 +256,7 @@ Style/RedundantSelf: # SupportedStyles: single_quotes, double_quotes Style/StringLiterals: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -292,8 +271,7 @@ Style/TernaryParentheses: # SupportedStyles: snake_case, camelCase Naming/VariableName: Exclude: - - 'lib/vmpooler/api/v1.rb' - - 'lib/vmpooler/api/v2.rb' + - 'lib/vmpooler/api/v3.rb' # Offense count: 1 # Cop supports --auto-correct. diff --git a/README.md b/README.md index 08af5d3..03731f3 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ - [VMPooler](#vmpooler) - [Usage](#usage) + - [Migrating to v3](#migrating-to-v3) - [v2.0.0 note](#v200-note) - [Installation](#installation) - [Dependencies](#dependencies) @@ -30,6 +31,38 @@ VMPooler provides configurable 'pools' of instantly-available (pre-provisioned) At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. VMPooler manages the life cycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand. +### Migrating to v3 + +Starting with the v3.x release, management of DNS records is implemented as DNS plugins, similar to compute providers. This means each pool configuration should be pointing to a configuration object in `:dns_config` to determine it's method of record management. + +For those using the global `DOMAIN` environment variable or global `:config.domain` key, this means records were not previously being managed by VMPooler (presumably managed via dynamic dns), so it's value should be moved to `:dns_configs::domain` with the value for `dns_class` for the config set to `dynamic-dns`. + +For example, the following < v3.x configuration: + +```yaml +:config: + domain: 'example.com' +``` + +becomes: + +```yaml +:dns_configs: + :example: + dns_class: dynamic-dns + domain: 'example.com' +``` + +Then any pools that should have records created via the dns config above should now reference the named dns config in the `dns_plugin` key: + +```yaml +:pools: + - name: 'debian-8-x86_64' + dns_plugin: 'example' +``` + +For those using the GCE provider, [vmpooler-provider-gce](https://github.com/puppetlabs/vmpooler-provider-gce), as of version 1.x the DNS management has been decoupled. See + ### v2.0.0 note As of version 2.0.0, all providers other than the dummy one are now separate gems. Historically the vSphere provider was included within VMPooler itself. That code has been moved to the [puppetlabs/vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) repository and the `vmpooler-provider-vsphere` gem. To migrate from VMPooler 1.x to 2.0 you will need to ensure that `vmpooler-provider-vsphere` is installed along side the `vmpooler` gem. See the [Provider API](docs/PROVIDER_API.md) docs for more information. diff --git a/docs/API-v1.md b/docs/API-v1.md deleted file mode 100644 index ee6fa26..0000000 --- a/docs/API-v1.md +++ /dev/null @@ -1,936 +0,0 @@ -# Table of contents -1. [API](#API) -2. [Token operations](#token) -3. [VM operations](#vmops) -4. [Add disks](#adddisks) -5. [VM snapshots](#vmsnapshots) -6. [Status and metrics](#statusmetrics) -7. [Pool configuration](#poolconfig) -8. [Ondemand VM provisioning](#ondemandvm) - -### API - -vmpooler provides a REST API for VM management. The following examples use `curl` for communication. - -#### Token operations - -Token-based authentication can be used when requesting or modifying VMs. The `/token` route can be used to create, query, or delete tokens. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for information on configuring an authentication store to use when performing token operations. - -##### GET /token - -Get a list of issued tokens. - -Return codes: -* 200 OK -* 401 when not authorized -* 404 when config:auth not found or other error - -``` -$ curl -u jdoe --url vmpooler.example.com/api/v1/token -Enter host password for user 'jdoe': -``` -```json -{ - "ok": true, - "utpg2i2xswor6h8ttjhu3d47z53yy47y": { - "created": "2015-04-28 19:17:47 -0700" - } -} -``` - -##### POST /token - -Generate a new authentication token. - -Return codes: -* 200 OK -* 401 when not authorized -* 404 when config:auth not found - -``` -$ curl -X POST -u jdoe --url vmpooler.example.com/api/v1/token -Enter host password for user 'jdoe': -``` -```json -{ - "ok": true, - "token": "utpg2i2xswor6h8ttjhu3d47z53yy47y" -} -``` - -##### GET /token/<token> - -Get information about an existing token (including associated VMs). - -Return codes: -* 200 OK -* 404 when config:auth or token not found - -``` -$ curl --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y -``` -```json -{ - "ok": true, - "utpg2i2xswor6h8ttjhu3d47z53yy47y": { - "user": "jdoe", - "created": "2015-04-28 19:17:47 -0700", - "last": "2015-11-04 12:28:37 -0700", - "vms": { - "running": [ - "dqs4914g2wjyy5w", - "hul7ib0ssr0f4o0" - ] - } - } -} -``` - -##### DELETE /token/<token> - -Delete an authentication token. - -Return codes: -* 200 OK -* 401 when not authorized -* 404 when config:auth not found - -``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/token/utpg2i2xswor6h8ttjhu3d47z53yy47y -Enter host password for user 'jdoe': -``` -```json -{ - "ok": true -} -``` - -#### VM operations - -##### GET /vm - -Retrieve a list of available VM pools. - -Return codes: -* 200 OK - -``` -$ curl --url vmpooler.example.com/api/v1/vm -``` -```json -[ - "debian-7-i386", - "debian-7-x86_64" -] -``` - -##### POST /vm - -Useful for batch operations; post JSON (see format below), get back allocated VMs. - -If an authentication store is configured, an authentication token supplied via the `X-AUTH-TOKEN` HTTP header will modify a VM's default lifetime. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information. - -Return codes: -* 200 OK -* 404 when sending invalid JSON in the request body or requesting an invalid VM pool name -* 503 when the vm failed to allocate a vm, or the pool is empty - -``` -$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v1/vm -``` -```json -{ - "ok": true, - "debian-7-i386": { - "hostname": [ - "o41xtodlvnvu5cw", - "khirruvwfjlmx3y" - ] - }, - "debian-7-x86_64": { - "hostname": "y91qbrpbfj6d13q" - }, - "domain": "example.com" -} -``` - -**NOTE: Returns either all requested VMs or no VMs.** - -##### POST /vm/<pool> - -Check-out a VM or VMs. - -Return codes: -* 200 OK -* 404 when sending invalid JSON in the request body or requesting an invalid VM pool name -* 503 when the vm failed to allocate a vm, or the pool is empty - -``` -$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386 -``` -```json -{ - "ok": true, - "debian-7-i386": { - "hostname": "fq6qlpjlsskycq6" - }, - "domain": "example.com" -} -``` - -Multiple VMs can be requested by using multiple query parameters in the URL: - -``` -$ curl -d --url vmpooler.example.com/api/v1/vm/debian-7-i386+debian-7-i386+debian-7-x86_64 -``` - -```json -{ - "ok": true, - "debian-7-i386": { - "hostname": [ - "sc0o4xqtodlul5w", - "4m4dkhqiufnjmxy" - ] - }, - "debian-7-x86_64": { - "hostname": "zb91y9qbrbf6d3q" - }, - "domain": "example.com" -} -``` - -**NOTE: Returns either all requested VMs or no VMs.** - -##### GET /vm/<hostname> - -Query metadata information for a checked-out VM. - -Return codes: -* 200 OK -* 404 when requesting an invalid VM hostname - -``` -$ curl --url vmpooler.example.com/api/v1/vm/pxpmtoonx7fiqg6 -``` -```json -{ - "ok": true, - "pxpmtoonx7fiqg6": { - "template": "centos-6-x86_64", - "lifetime": 12, - "running": 3, - "remaining": 9, - "state": "running", - "tags": { - "department": "engineering", - "user": "jdoe" - }, - "ip": "192.168.0.1", - "domain": "example.com", - "host": "host1.example.com", - "migrated": "true" - } -} -``` - -##### PUT /vm/<hostname> - -Modify a checked-out VM. - -The following are valid PUT parameters and their required data structures: - -parameter | description | required structure ---------- | ----------- | ------------------ -*lifetime* | VM TTL (in hours) | integer -*tags* | free-form VM tagging | hash - -Any modifications can be verified using the [GET /vm/<hostname>](#get-vmhostname) endpoint. - -If an authentication store is configured, an authentication token is required (via the `X-AUTH-TOKEN` HTTP header) to access this route. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), and the 'token operations' section above for more information. - -Return codes: -* 200 OK -* 401 when you need an auth token -* 404 when requesting an invalid VM hostname -* 400 when supplied PUT parameters fail validation - -``` -$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6 -``` -```json -{ - "ok": true -} -``` - -``` -$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6 -``` -```json -{ - "ok": true -} -``` - -##### DELETE /vm/<hostname> - -Schedule a checked-out VM for deletion. - -Return codes: -* 200 OK -* 401 when you need an auth token -* 404 when requesting an invalid VM hostname - -``` -$ curl -X DELETE --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6 -``` -```json -{ - "ok": true -} -``` - -#### Adding additional disk(s) - -##### POST /vm/<hostname>/disk/<size> - -Add an additional disk to a running VM. - -Return codes: -* 202 OK -* 401 when you need an auth token -* 404 when requesting an invalid VM hostname or size is not an integer - -```` -$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/disk/8 -```` -````json -{ - "ok": true, - "fq6qlpjlsskycq6": { - "disk": "+8gb" - } -} -```` - -Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/` query: - -```` -$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6 -```` -````json -{ - "ok": true, - "fq6qlpjlsskycq6": { - "template": "debian-7-x86_64", - "lifetime": 2, - "running": 0.08, - "state": "running", - "disk": [ - "+8gb" - ], - "domain": "delivery.puppetlabs.net" - } -} - -```` - -#### VM snapshots - -##### POST /vm/<hostname>/snapshot - -Create a snapshot of a running VM. - -Return codes: -* 202 OK -* 401 when you need an auth token -* 404 when requesting an invalid VM hostname - -```` -$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot -```` -````json -{ - "ok": true, - "fq6qlpjlsskycq6": { - "snapshot": "n4eb4kdtp7rwv4x158366vd9jhac8btq" - } -} -```` - -Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/` query: - -```` -$ curl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6 -```` -````json -{ - "ok": true, - "fq6qlpjlsskycq6": { - "template": "debian-7-x86_64", - "lifetime": 2, - "running": 0.08, - "state": "running", - "snapshots": [ - "n4eb4kdtp7rwv4x158366vd9jhac8btq" - ], - "domain": "delivery.puppetlabs.net" - } -} -```` - -##### POST /vm/<hostname>/snapshot/<snapshot> - -Revert a VM back to a snapshot. - -Return codes: -* 202 OK -* 401 when you need an auth token -* 404 when requesting an invalid VM hostname or snapshot is not valid - -```` -$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v1/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq -```` -````json -{ - "ok": true -} -```` - -#### Status and metrics - -##### GET /status - -A "live" status endpoint, representing the current state of the service. - -``` -$ curl --url vmpooler.example.com/api/v1/status -``` -```json -{ - "capacity": { - "current": 716, - "total": 717, - "percent": 99.9 - }, - "clone": { - "duration": { - "average": 8.8, - "min": 2.79, - "max": 69.76 - }, - "count": { - "total": 1779 - } - }, - "queue": { - "pending": 1, - "cloning": 0, - "booting": 1, - "ready": 716, - "running": 142, - "completed": 0, - "total": 859 - }, - "status": { - "ok": true, - "message": "Battle station fully armed and operational." - } -} -``` - -If there are empty pools, the "status" section will convey this: - -```json - "status": { - "ok": false, - "message": "Found 2 empty pools.", - "empty": [ - "centos-6-x86_64", - "debian-7-x86_64" - ] - } -``` - -The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "status". -If the query parameter 'view' is provided, it will be used to select which top level -element to compute and return. Select them by specifying which one you want in a comma -separated list. -For example `vmpooler.example.com/api/v1/status?view=capacity,boot` - -##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]] - -Returns a summary, or report, for the timespan between `from` and `to` (inclusive) -parameters. The response includes both an overall and daily view of tracked -metrics, such as boot and cloning durations. - -Any omitted query parameter will default to now/today. A request without any -parameters will result in the current day's summary. - -Return codes: -* 200 OK -* 400 Invalid date format or range - - -``` -$ curl --url vmpooler.example.com/api/v1/summary -``` -```json -{ - "boot": { - "duration": { - "average": 106.6, - "min": 83.09, - "max": 121.06, - "total": 639.36, - }, - "count": { - "average": 6, - "min": 6, - "max": 6, - "total": 6, - } - }, - "clone": { - "duration": { - "average": 4.6, - "min": 2.78, - "max": 8.1, - "total": 63.94, - }, - "count": { - "average": 14, - "min": 14, - "max": 14, - "total": 14, - } - }, - "tag": { - "department": { - "engineering": 14, - "help desk": 10, - "IT": 44, - "total": 68 - }, - "user": { - "arodgers": 54, - "cmatthews": 10, - "jnelson": 4, - "total": 68 - } - }, - "daily": [ - { - "date": "2015-03-11", - "boot": { - "duration": { - "average": 106.6, - "min": 83.09, - "max": 121.06, - "total": 639.36 - }, - "count": { - "total": 6 - } - }, - "clone": { - "duration": { - "average": 4.6, - "min": 2.78, - "max": 8.1, - "total": 63.94 - }, - "count": { - "total": 14 - } - }, - "tag": { - "department": { - "engineering": 14, - "help desk": 10, - "IT": 44, - "total": 68 - }, - "user": { - "arodgers": 54, - "cmatthews": 10, - "jnelson": 4, - "total": 68 - } - } - } - ] -} -``` - - -``` -$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v1/summary -``` -```json -{ - "boot": {...}, - "clone": {...}, - "daily": [ - { - "date": "2015-03-10", - "boot": { - "duration": { - "average": 0, - "min": 0, - "max": 0, - "total": 0 - }, - "count": { - "total": 0 - } - }, - "clone": { - "duration": { - "average": 0, - "min": 0, - "max": 0, - "total": 0 - }, - "count": { - "total": 0 - } - }, - "tag": { } - }, - { - "date": "2015-03-11", - "boot": { - "duration": { - "average": 106.6, - "min": 83.09, - "max": 121.06, - "total": 639.36 - }, - "count": { - "total": 6 - } - }, - "clone": { - "duration": { - "average": 4.6, - "min": 2.78, - "max": 8.1, - "total": 63.94 - }, - "count": { - "total": 14 - } - }, - "tag": { } - } - ] -} -``` - -You can also query only the specific top level section you want by including it after `summary/`. -The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v1/summary/boot/`. -You can further drill-down the data by specifying the second level parameter to query eg -`vmpooler.example.com/api/v1/summary/tag/created_by` - -##### GET /poolstat?pool=FOO - -For parameter `pool`, containing a comma separated list of pool names to query, this endpoint returns -each of the pool's ready, max and alias information. It can be used to get a fast response for -the required pools instead of using the /status API endpoint - -Return codes -* 200 OK - -``` -$ curl https://vmpooler.example.com/api/v1/poolstat?pool=centos-6-x86_64 -``` -```json -{ - "pools": { - "centos-6-x86_64": { - "ready": 25, - "max": 25, - "alias": [ - "centos-6-64", - "centos-6-amd64" - ] - } - } -} -``` - -##### GET /totalrunning - -Fast endpoint to return the total number of VMs in a 'running' state - -Return codes -* 200 OK - -``` -$ curl https://vmpooler.example.com/api/v1/totalrunning -``` - -```json -{ - - "running": 362 - -} -``` - -#### Managing pool configuration via API - -##### GET /config - -Returns the running pool configuration - -Return codes -* 200 OK -* 400 No configuration found - -``` -$ curl https://vmpooler.example.com/api/v1/config -``` -```json -{ - "pool_configuration": [ - { - "name": "redhat-7-x86_64", - "template": "templates/redhat-7.2-x86_64-0.0.3", - "folder": "vmpooler/redhat-7-x86_64", - "datastore": "stor1", - "size": 1, - "datacenter": "dc1", - "provider": "vsphere", - "capacity": 1, - "major": "redhat", - "template_ready": true - } - ], - "status": { - "ok": true - } -} -``` - -Note: to enable poolsize and pooltemplate config endpoints it is necessary to set 'experimental_features: true' in your vmpooler configuration. A 405 is returned when you attempt to interact with these endpoints when this configuration option is not set. - -##### POST /config/poolsize - -Change pool size without having to restart the service. - -All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned - -When a pool size is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned. - -Pool size configuration changes persist through application restarts, and take precedence over a pool size value configured in the pool configuration provided when the application starts. This persistence is dependent on redis. So, if the redis data is lost then the configuration updates revert to those provided at startup at the next application start. - -An authentication token is required in order to change pool configuration when authentication is configured. -Responses: -* 200 - No changes required -* 201 - Changes made on at least one pool with changes requested -* 400 - An invalid configuration was provided causing requested changes to fail -* 404 - An unknown error occurred -* 405 - The endpoint is disabled because experimental features are disabled -``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v1/config/poolsize -``` -```json -{ - "ok": true -} -``` - -##### DELETE /config/poolsize/<pool> - -Delete an overridden pool size. This results in the values from VMPooler's config being used. - -Return codes: -* 200 - when nothing was changed but no error occurred -* 201 - size reset successful -* 401 - when not authorized -* 404 - pool does not exist -* 405 - The endpoint is disabled because experimental features are disabled - -``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/poolsize/almalinux-8-x86_64 -``` -```json -{ - "ok": true, - "pool_size_before_overrides": 2, - "pool_size_before_reset": 4 -} -``` - -##### POST /config/pooltemplate - -Change the template configured for a pool, and replenish the pool with instances built from the new template. - -All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned - -When a pool template is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned. - -A pool template being updated will cause the following actions, which are logged in vmpooler.log: -* Destroy all instances for the pool template being updated that are in the ready and pending state -* Halt repopulating the pool while creating template deltas for the newly configured template -* Unblock pool population and let the pool replenish with instances based on the newly configured template - -Pool template changes persist through application restarts, and take precedence over a pool template configured in the pool configuration provided when the application starts. This persistence is dependent on redis. As a result, if the redis data is lost then the configuration values revert to those provided at startup at the next application start. - -An authentication token is required in order to change pool configuration when authentication is configured. - -Responses: -* 200 - No changes required -* 201 - Changes made on at least one pool with changes requested -* 400 - An invalid configuration was provided causing requested changes to fail -* 404 - An unknown error occurred -* 405 - The endpoint is disabled because experimental features are disabled -``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v1/config/pooltemplate -``` -```json -{ - "ok": true -} -``` - -##### DELETE /config/pooltemplate/<pool> - -Delete an overridden pool template. This results in the values from VMPooler's config being used. - -Return codes: -* 200 - when nothing was changed but no error occurred -* 201 - template reset successful -* 401 - when not authorized -* 404 - pool does not exist -* 405 - The endpoint is disabled because experimental features are disabled - -``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v1/pooltemplate/almalinux-8-x86_64 -``` -```json -{ - "ok": true, - "template_before_overrides": "templates/almalinux-8-x86_64-0.0.2", - "template_before_reset": "templates/almalinux-8-x86_64-0.0.3-beta" -} -``` - -##### POST /poolreset - -Clear all pending and ready instances in a pool, and deploy replacements - -All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned. - -When a pool reset is requested a 201 status will be returned. - -A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances. - -For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured. - -Responses: -* 201 - Pool reset requested received -* 400 - An invalid configuration was provided causing requested changes to fail -* 404 - An unknown error occurred -* 405 - The endpoint is disabled because experimental features are disabled -``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v1/poolreset -``` -```json -{ - "ok": true -} -``` - -#### Ondemand VM operations - -Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated. - -##### POST /ondemandvm - -All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned - -When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status. - -An authentication token is required in order to request instances on demand when authentication is configured. - -Responses: -* 201 - Provisioning request accepted -* 400 - Payload contains invalid JSON and cannot be parsed -* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled -* 403 - Request exceeds the configured per pool maximum -* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred. -* 409 - A request of the matching ID has already been created -``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v1/ondemandvm -``` -```json -{ - "ok": true, - "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a" -} -``` - -##### GET /ondemandvm - -Get the status of an ondemandvm request that has already been posted. - -When the request is ready the ready status will change to 'true'. - -The number of instances pending vs ready will be reflected in the API response. - -Responses: -* 200 - The API request was successful and the status is ok -* 202 - The request is not ready yet -* 404 - The request can not be found, or an unknown error occurred -``` -$ curl https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a -``` -```json -{ - "ok": true, - "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", - "ready": false, - "debian-7-i386": { - "ready": "3", - "pending": "1" - } -} -``` -```json -{ - "ok": true, - "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", - "ready": true, - "debian-7-i386": { - "hostname": [ - "vm1", - "vm2", - "vm3", - "vm4" - ] - } -} -``` - -##### DELETE /ondemandvm - -Delete a ondemand request - -Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted. - -Responses: -* 200 - The API request was sucessful. A message will indicate if the request has already been deleted. -* 401 - No auth token provided, or provided auth token is not valid, and auth is enabled -* 404 - The request can not be found, or an unknown error occurred. -``` -$ curl -X DELETE https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a -``` -```json -{ - "ok": true -} -``` diff --git a/docs/API-v2.md b/docs/API-v3.md similarity index 89% rename from docs/API-v2.md rename to docs/API-v3.md index 54a83a6..ec4feaf 100644 --- a/docs/API-v2.md +++ b/docs/API-v3.md @@ -12,6 +12,15 @@ vmpooler provides a REST API for VM management. The following examples use `curl` for communication. +## Major change in V3 versus V2 + +The api/v1 and api/v2 endpoints have been removed. Additionally, the generic api endpoint that reroutes to a versioned endpoint has been removed. + +The api/v3 endpoint removes the deprecated "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider, +the hostname now returns an FQDN including that domain. That is to say, we can now have multiple, different domains for each pool instead of only a single domain for all pools, or a domain restricted to a particular provider. + +Clients using some of the direct API paths (without specifying api/v1 or api/v2) will now now need to specify the versioned endpoint (api/v3). + ## Major change in V2 versus V1 The api/v2 endpoint removes a separate "domain" key returned in some of the operations like getting a VM, etc. If there is a "domain" configured in the top level configuration or for a specific provider, @@ -41,7 +50,7 @@ Return codes: * 404 when config:auth not found or other error ``` -$ curl -u jdoe --url vmpooler.example.com/api/v2/token +$ curl -u jdoe --url vmpooler.example.com/api/v3/token Enter host password for user 'jdoe': ``` ```json @@ -63,7 +72,7 @@ Return codes: * 404 when config:auth not found ``` -$ curl -X POST -u jdoe --url vmpooler.example.com/api/v2/token +$ curl -X POST -u jdoe --url vmpooler.example.com/api/v3/token Enter host password for user 'jdoe': ``` ```json @@ -82,7 +91,7 @@ Return codes: * 404 when config:auth or token not found ``` -$ curl --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y +$ curl --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y ``` ```json { @@ -111,7 +120,7 @@ Return codes: * 404 when config:auth not found ``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/token/utpg2i2xswor6h8ttjhu3d47z53yy47y Enter host password for user 'jdoe': ``` ```json @@ -130,7 +139,7 @@ Return codes: * 200 OK ``` -$ curl --url vmpooler.example.com/api/v2/vm +$ curl --url vmpooler.example.com/api/v3/vm ``` ```json [ @@ -151,7 +160,7 @@ Return codes: * 503 when the vm failed to allocate a vm, or the pool is empty ``` -$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v2/vm +$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.example.com/api/v3/vm ``` ```json { @@ -180,7 +189,7 @@ Return codes: * 503 when the vm failed to allocate a vm, or the pool is empty ``` -$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386 +$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386 ``` ```json { @@ -194,7 +203,7 @@ $ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386 Multiple VMs can be requested by using multiple query parameters in the URL: ``` -$ curl -d --url vmpooler.example.com/api/v2/vm/debian-7-i386+debian-7-i386+debian-7-x86_64 +$ curl -d --url vmpooler.example.com/api/v3/vm/debian-7-i386+debian-7-i386+debian-7-x86_64 ``` ```json @@ -223,7 +232,7 @@ Return codes: * 404 when requesting an invalid VM hostname ``` -$ curl --url vmpooler.example.com/api/v2/vm/pxpmtoonx7fiqg6 +$ curl --url vmpooler.example.com/api/v3/vm/pxpmtoonx7fiqg6 ``` ```json { @@ -267,7 +276,7 @@ Return codes: * 400 when supplied PUT parameters fail validation ``` -$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +$ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6 ``` ```json { @@ -276,7 +285,7 @@ $ curl -X PUT -d '{"lifetime":"2"}' --url vmpooler.example.com/api/v2/vm/fq6qlpj ``` ``` -$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +$ curl -X PUT -d '{"tags":{"department":"engineering","user":"jdoe"}}' --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6 ``` ```json { @@ -294,7 +303,7 @@ Return codes: * 404 when requesting an invalid VM hostname ``` -$ curl -X DELETE --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +$ curl -X DELETE --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6 ``` ```json { @@ -314,7 +323,7 @@ Return codes: * 404 when requesting an invalid VM hostname or size is not an integer ```` -$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/disk/8 +$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/disk/8 ```` ````json { @@ -328,7 +337,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e Provisioning and attaching disks can take a moment, but once the task completes it will be reflected in a `GET /vm/` query: ```` -$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6 ```` ````json { @@ -358,7 +367,7 @@ Return codes: * 404 when requesting an invalid VM hostname ```` -$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot +$ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot ```` ````json { @@ -372,7 +381,7 @@ $ curl -X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.e Snapshotting a live VM can take a moment, but once the snapshot task completes it will be reflected in a `GET /vm/` query: ```` -$ curl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6 +$ curl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6 ```` ````json { @@ -399,7 +408,7 @@ Return codes: * 404 when requesting an invalid VM hostname or snapshot is not valid ```` -$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v2/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq +$ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.example.com/api/v3/vm/fq6qlpjlsskycq6/snapshot/n4eb4kdtp7rwv4x158366vd9jhac8btq ```` ````json { @@ -414,7 +423,7 @@ $ curl X POST -H X-AUTH-TOKEN:a9znth9dn01t416hrguu56ze37t790bl --url vmpooler.ex A "live" status endpoint, representing the current state of the service. ``` -$ curl --url vmpooler.example.com/api/v2/status +$ curl --url vmpooler.example.com/api/v3/status ``` ```json { @@ -466,7 +475,7 @@ The top level sections are: "capacity", "queue", "clone", "boot", "pools" and "s If the query parameter 'view' is provided, it will be used to select which top level element to compute and return. Select them by specifying which one you want in a comma separated list. -For example `vmpooler.example.com/api/v2/status?view=capacity,boot` +For example `vmpooler.example.com/api/v3/status?view=capacity,boot` ##### GET /summary[?from=YYYY-MM-DD[&to=YYYY-MM-DD]] @@ -483,7 +492,7 @@ Return codes: ``` -$ curl --url vmpooler.example.com/api/v2/summary +$ curl --url vmpooler.example.com/api/v3/summary ``` ```json { @@ -575,7 +584,7 @@ $ curl --url vmpooler.example.com/api/v2/summary ``` -$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v2/summary +$ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api/v3/summary ``` ```json { @@ -639,9 +648,9 @@ $ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.example.com/api ``` You can also query only the specific top level section you want by including it after `summary/`. -The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v2/summary/boot/`. +The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v3/summary/boot/`. You can further drill-down the data by specifying the second level parameter to query eg -`vmpooler.example.com/api/v2/summary/tag/created_by` +`vmpooler.example.com/api/v3/summary/tag/created_by` ##### GET /poolstat?pool=FOO @@ -653,7 +662,7 @@ Return codes * 200 OK ``` -$ curl https://vmpooler.example.com/api/v2/poolstat?pool=centos-6-x86_64 +$ curl https://vmpooler.example.com/api/v3/poolstat?pool=centos-6-x86_64 ``` ```json { @@ -678,7 +687,7 @@ Return codes * 200 OK ``` -$ curl https://vmpooler.example.com/api/v2/totalrunning +$ curl https://vmpooler.example.com/api/v3/totalrunning ``` ```json @@ -700,7 +709,7 @@ Return codes * 400 No configuration found ``` -$ curl https://vmpooler.example.com/api/v2/config +$ curl https://vmpooler.example.com/api/v3/config ``` ```json { @@ -744,7 +753,7 @@ Responses: * 404 - An unknown error occurred * 405 - The endpoint is disabled because experimental features are disabled ``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v2/config/poolsize +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.example.com/api/v3/config/poolsize ``` ```json { @@ -764,7 +773,7 @@ Return codes: * 405 - The endpoint is disabled because experimental features are disabled ``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/poolsize/almalinux-8-x86_64 +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/poolsize/almalinux-8-x86_64 ``` ```json { @@ -798,7 +807,7 @@ Responses: * 404 - An unknown error occurred * 405 - The endpoint is disabled because experimental features are disabled ``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v2/config/pooltemplate +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.example.com/api/v3/config/pooltemplate ``` ```json { @@ -818,7 +827,7 @@ Return codes: * 405 - The endpoint is disabled because experimental features are disabled ``` -$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v2/pooltemplate/almalinux-8-x86_64 +$ curl -X DELETE -u jdoe --url vmpooler.example.com/api/v3/pooltemplate/almalinux-8-x86_64 ``` ```json { @@ -846,7 +855,7 @@ Responses: * 404 - An unknown error occurred * 405 - The endpoint is disabled because experimental features are disabled ``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v2/poolreset +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v3/poolreset ``` ```json { @@ -874,7 +883,7 @@ Responses: * 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred. * 409 - A request of the matching ID has already been created ``` -$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v2/ondemandvm +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v3/ondemandvm ``` ```json { @@ -896,7 +905,7 @@ Responses: * 202 - The request is not ready yet * 404 - The request can not be found, or an unknown error occurred ``` -$ curl https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +$ curl https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a ``` ```json { @@ -936,7 +945,7 @@ Responses: * 401 - No auth token provided, or provided auth token is not valid, and auth is enabled * 404 - The request can not be found, or an unknown error occurred. ``` -$ curl -X DELETE https://vmpooler.example.com/api/v2/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +$ curl -X DELETE https://vmpooler.example.com/api/v3/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a ``` ```json { diff --git a/docs/configuration.md b/docs/configuration.md index 1c34c72..e577025 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,11 +19,6 @@ Provide the entire configuration as a blob of yaml. Individual parameters passed Path to a the file to use when loading the vmpooler configuration. This is only evaluated if `VMPOOLER_CONFIG` has not been specified. -### DOMAIN - -This environment variable is to set the configuration config.domain -(optional) - ### REDIS\_SERVER The redis server to use for vmpooler. diff --git a/examples/vmpooler.yaml.dummy-example.aliasedpools b/examples/vmpooler.yaml.dummy-example.aliasedpools index efe0ce2..ebece50 100644 --- a/examples/vmpooler.yaml.dummy-example.aliasedpools +++ b/examples/vmpooler.yaml.dummy-example.aliasedpools @@ -23,9 +23,13 @@ allowed_tags: - 'created_by' - 'project' - domain: 'example.com' prefix: 'poolvm-' +:dns_configs: + :example: + dns_class: dynamic-dns + domain: 'example.com' + :pools: - name: 'debian-7-i386' alias: [ 'debian-7-32' ] @@ -36,6 +40,7 @@ timeout: 15 ready_ttl: 1440 provider: dummy + dns_plugin: 'example' - name: 'debian-7-i386-stringalias' alias: 'debian-7-32-stringalias' template: 'Templates/debian-7-i386' @@ -45,6 +50,7 @@ timeout: 15 ready_ttl: 1440 provider: dummy + dns_plugin: 'example' - name: 'debian-7-x86_64' alias: [ 'debian-7-64', 'debian-7-amd64' ] template: 'Templates/debian-7-x86_64' @@ -54,6 +60,7 @@ timeout: 15 ready_ttl: 1440 provider: dummy + dns_plugin: 'example' - name: 'debian-7-i386-noalias' template: 'Templates/debian-7-i386' folder: 'Pooled VMs/debian-7-i386' @@ -62,6 +69,7 @@ timeout: 15 ready_ttl: 1440 provider: dummy + dns_plugin: 'example' - name: 'debian-7-x86_64-alias-otherpool-extended' alias: [ 'debian-7-x86_64' ] template: 'Templates/debian-7-x86_64' @@ -71,4 +79,4 @@ timeout: 15 ready_ttl: 1440 provider: dummy - + dns_plugin: 'example' diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 516843f..54c4ffc 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -76,7 +76,10 @@ module Vmpooler parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || '' parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE'] parsed_config[:config]['site_name'] = ENV['SITE_NAME'] if ENV['SITE_NAME'] - parsed_config[:config]['domain'] = ENV['DOMAIN'] if ENV['DOMAIN'] + if !parsed_config[:config]['domain'].nil? || !ENV['DOMAIN'].nil? + puts '[!] [error] The "domain" config setting has been removed in v3. Please see the docs for migrating the domain config to use a dns plugin at https://github.com/puppetlabs/vmpooler/blob/main/README.md#migrating-to-v3' + exit 1 + end parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET'] parsed_config[:config]['timeout'] = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT'] parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH'] diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index a7eea30..3b0d9de 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -3,7 +3,7 @@ module Vmpooler class API < Sinatra::Base # Load API components - %w[helpers dashboard reroute v1 v2 request_logger healthcheck].each do |lib| + %w[helpers dashboard v3 request_logger healthcheck].each do |lib| require "vmpooler/api/#{lib}" end # Load dashboard components @@ -52,9 +52,7 @@ module Vmpooler use Vmpooler::Dashboard use Vmpooler::API::Dashboard - use Vmpooler::API::Reroute - use Vmpooler::API::V1 - use Vmpooler::API::V2 + use Vmpooler::API::V3 end # Get thee started O WebServer diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 93d571b..e393466 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -181,8 +181,7 @@ module Vmpooler /^\d{4}-\d{2}-\d{2}$/ === date_str end - # NOTE: domain is not needed here, so we should update the callers of this method - def hostname_shorten(hostname, domain=nil) + def hostname_shorten(hostname) hostname[/[^.]+/] end diff --git a/lib/vmpooler/api/reroute.rb b/lib/vmpooler/api/reroute.rb deleted file mode 100644 index a8c66fa..0000000 --- a/lib/vmpooler/api/reroute.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module Vmpooler - class API - class Reroute < Sinatra::Base - api_version = '1' - - get '/status/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /status/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/status") - end - - get '/summary/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/summary") - end - - get '/summary/:route/?:key?/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /summary/:route/?:key?/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}") - end - - get '/token/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/token") - end - - post '/token/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/token") - end - - get '/token/:token/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}") - end - - delete '/token/:token/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /token/:token/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/token/#{params[:token]}") - end - - get '/vm/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm") - end - - post '/vm/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm") - end - - post '/vm/:template/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:template/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:template]}") - end - - get '/vm/:hostname/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") - end - - delete '/vm/:hostname/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called delete /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") - end - - put '/vm/:hostname/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}") - end - - post '/vm/:hostname/snapshot/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot") - end - - post '/vm/:hostname/snapshot/:snapshot/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called post /vm/:hostname/snapshot/:snapshot/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/snapshot/#{params[:snapshot]}") - end - - put '/vm/:hostname/disk/:size/?' do - puts "DEPRECATION WARNING a client (#{request.user_agent}) called put /vm/:hostname/disk/:size/? and got redirected to api_version=1, this behavior will change in the next major version, please modify the client to use v2 in advance" - call env.merge('PATH_INFO' => "/api/v#{api_version}/vm/#{params[:hostname]}/disk/#{params[:size]}") - end - end - end -end diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb deleted file mode 100644 index c6e7d9b..0000000 --- a/lib/vmpooler/api/v1.rb +++ /dev/null @@ -1,1757 +0,0 @@ -# frozen_string_literal: true - -require 'vmpooler/util/parsing' - -module Vmpooler - class API - class V1 < Sinatra::Base - api_version = '1' - api_prefix = "/api/v#{api_version}" - - helpers do - include Vmpooler::API::Helpers - end - - def backend - Vmpooler::API.settings.redis - end - - def metrics - Vmpooler::API.settings.metrics - end - - def config - Vmpooler::API.settings.config[:config] - end - - def full_config - Vmpooler::API.settings.config - end - - def pools - Vmpooler::API.settings.config[:pools] - end - - def pools_at_startup - Vmpooler::API.settings.config[:pools_at_startup] - end - - def pool_exists?(template) - Vmpooler::API.settings.config[:pool_names].include?(template) - end - - def need_auth! - validate_auth(backend) - end - - def need_token! - validate_token(backend) - end - - def checkoutlock - Vmpooler::API.settings.checkoutlock - end - - def get_template_aliases(template) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = [] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - result += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) - end - result - end - end - - def get_pool_weights(template_backends) - pool_index = pool_index(pools) - weighted_pools = {} - template_backends.each do |t| - next unless pool_index.key? t - - index = pool_index[t] - clone_target = pools[index]['clone_target'] || config['clone_target'] - next unless config.key?('backend_weight') - - weight = config['backend_weight'][clone_target] - if weight - weighted_pools[t] = weight - end - end - weighted_pools - end - - def count_selection(selection) - result = {} - selection.uniq.each do |poolname| - result[poolname] = selection.count(poolname) - end - result - end - - def evaluate_template_aliases(template, count) - template_backends = [] - template_backends << template if backend.sismember('vmpooler__pools', template) - selection = [] - aliases = get_template_aliases(template) - if aliases - template_backends += aliases - weighted_pools = get_pool_weights(template_backends) - - if weighted_pools.count > 1 && weighted_pools.count == template_backends.count - pickup = Pickup.new(weighted_pools) - count.to_i.times do - selection << pickup.pick - end - else - count.to_i.times do - selection << template_backends.sample - end - end - end - - count_selection(selection) - end - - def fetch_single_vm(template) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - template_backends = [template] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - template_backends += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) - pool_index = pool_index(pools) - weighted_pools = {} - template_backends.each do |t| - next unless pool_index.key? t - - index = pool_index[t] - clone_target = pools[index]['clone_target'] || config['clone_target'] - next unless config.key?('backend_weight') - - weight = config['backend_weight'][clone_target] - if weight - weighted_pools[t] = weight - end - end - - if weighted_pools.count == template_backends.count - pickup = Pickup.new(weighted_pools) - selection = pickup.pick - template_backends.delete(selection) - template_backends.unshift(selection) - else - first = template_backends.sample - template_backends.delete(first) - template_backends.unshift(first) - end - end - - checkoutlock.synchronize do - template_backends.each do |template_backend| - vms = backend.smembers("vmpooler__ready__#{template_backend}") - next if vms.empty? - - vms.reverse.each do |vm| - ready = vm_ready?(vm, config['domain']) - if ready - smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) - if smoved - return [vm, template_backend, template] - else - metrics.increment("checkout.smove.failed.#{template_backend}") - return [nil, nil, nil] - end - else - backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) - metrics.increment("checkout.nonresponsive.#{template_backend}") - end - end - end - [nil, nil, nil] - end - end - end - - def return_vm_to_ready_state(template, vm) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - backend.srem("vmpooler__migrating__#{template}", vm) - backend.hdel("vmpooler__active__#{template}", vm) - backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user') - backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm) - end - end - - def account_for_starting_vm(template, vm) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user') - span.set_attribute('enduser.id', user) - has_token_result = has_token? - backend.sadd("vmpooler__migrating__#{template}", vm) - backend.hset("vmpooler__active__#{template}", vm, Time.now) - backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) - - if Vmpooler::API.settings.config[:auth] and has_token_result - backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) - backend.hset("vmpooler__vm__#{vm}", 'token:user', user) - - if config['vm_lifetime_auth'].to_i > 0 - backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i) - end - end - end - end - - def update_result_hosts(result, template, vm) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result[template] ||= {} - if result[template]['hostname'] - result[template]['hostname'] = Array(result[template]['hostname']) - result[template]['hostname'].push(vm) - else - result[template]['hostname'] = vm - end - end - end - - def atomically_allocate_vms(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - result = { 'ok' => false } - failed = false - vms = [] - - validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? - - payload.each do |requested, count| - count.to_i.times do |_i| - vmname, vmpool, vmtemplate = fetch_single_vm(requested) - if vmname - account_for_starting_vm(vmpool, vmname) - vms << [vmpool, vmname, vmtemplate] - metrics.increment("checkout.success.#{vmpool}") - update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] - else - failed = true - metrics.increment("checkout.empty.#{requested}") - break - end - end - end - - if failed - vms.each do |(vmpool, vmname, _vmtemplate)| - return_vm_to_ready_state(vmpool, vmname) - end - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V1.atomically_allocate_vms', - 'error.message' => '503 due to failing to allocate one or more vms' - }) - status 503 - else - vm_names = [] - vms.each do |(_vmpool, vmname, vmtemplate)| - update_result_hosts(result, vmtemplate, vmname) - vm_names.append(vmname) - end - - span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty? - - result['ok'] = true - result['domain'] = config['domain'] if config['domain'] - end - - result - end - end - - def component_to_test(match, labels_string) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - return if labels_string.nil? - - labels_string_parts = labels_string.split(',') - labels_string_parts.each do |part| - key, value = part.split('=') - next if value.nil? - return value if key == match - end - 'none' - end - end - - def update_user_metrics(operation, vmname) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - begin - backend.multi - backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url') - backend.hget("vmpooler__vm__#{vmname}", 'token:user') - backend.hget("vmpooler__vm__#{vmname}", 'template') - jenkins_build_url, user, poolname = backend.exec - poolname = poolname.gsub('.', '_') - - if user - user = user.gsub('.', '_') - else - user = 'unauthenticated' - end - metrics.increment("user.#{user}.#{operation}.#{poolname}") - - if jenkins_build_url - if jenkins_build_url.include? 'litmus' - # Very simple filter for Litmus jobs - just count them coming through for the moment. - metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}") - else - url_parts = jenkins_build_url.split('/')[2..-1] - jenkins_instance = url_parts[0].gsub('.', '_') - value_stream_parts = url_parts[2].split('_') - value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') } - value_stream = value_stream_parts.shift - branch = value_stream_parts.pop - project = value_stream_parts.shift - job_name = value_stream_parts.join('_') - build_metadata_parts = url_parts[3] - component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) - - metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") - metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") - metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") - end - end - rescue StandardError => e - puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" - span.record_exception(e) - span.status = OpenTelemetry::Trace::Status.error(e.to_s) - span.add_event('log', attributes: { - 'log.severity' => 'debug', - 'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" - }) - end - end - end - - def reset_pool_size(poolname) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - pool_index = pool_index(pools) - - pools_updated = 0 - sync_pool_sizes - - pool_size_now = pools[pool_index[poolname]]['size'].to_i - pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i - result['pool_size_before_reset'] = pool_size_now - result['pool_size_before_overrides'] = pool_size_original - - unless pool_size_now == pool_size_original - pools[pool_index[poolname]]['size'] = pool_size_original - backend.hdel('vmpooler__config__poolsize', poolname) - backend.sadd('vmpooler__pool__undo_size_override', poolname) - pools_updated += 1 - status 201 - end - - status 200 unless pools_updated > 0 - result['ok'] = true - result - end - end - - def update_pool_size(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - pool_index = pool_index(pools) - pools_updated = 0 - sync_pool_sizes - - payload.each do |poolname, size| - unless pools[pool_index[poolname]]['size'] == size.to_i - pools[pool_index[poolname]]['size'] = size.to_i - backend.hset('vmpooler__config__poolsize', poolname, size) - pools_updated += 1 - status 201 - end - end - status 200 unless pools_updated > 0 - result['ok'] = true - result - end - end - - def reset_pool_template(poolname) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - pool_index_live = pool_index(pools) - pool_index_original = pool_index(pools_at_startup) - - pools_updated = 0 - sync_pool_templates - - template_now = pools[pool_index_live[poolname]]['template'] - template_original = pools_at_startup[pool_index_original[poolname]]['template'] - result['template_before_reset'] = template_now - result['template_before_overrides'] = template_original - - unless template_now == template_original - pools[pool_index_live[poolname]]['template'] = template_original - backend.hdel('vmpooler__config__template', poolname) - backend.sadd('vmpooler__pool__undo_template_override', poolname) - pools_updated += 1 - status 201 - end - - status 200 unless pools_updated > 0 - result['ok'] = true - result - end - end - - def update_pool_template(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - pool_index = pool_index(pools) - pools_updated = 0 - sync_pool_templates - - payload.each do |poolname, template| - unless pools[pool_index[poolname]]['template'] == template - pools[pool_index[poolname]]['template'] = template - backend.hset('vmpooler__config__template', poolname, template) - pools_updated += 1 - status 201 - end - end - status 200 unless pools_updated > 0 - result['ok'] = true - result - end - end - - def reset_pool(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - payload.each do |poolname, _count| - backend.sadd('vmpooler__poolreset', poolname) - end - status 201 - result['ok'] = true - result - end - end - - def update_clone_target(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - result = { 'ok' => false } - - pool_index = pool_index(pools) - pools_updated = 0 - sync_clone_targets - - payload.each do |poolname, clone_target| - unless pools[pool_index[poolname]]['clone_target'] == clone_target - pools[pool_index[poolname]]['clone_target'] = clone_target - backend.hset('vmpooler__config__clone_target', poolname, clone_target) - pools_updated += 1 - status 201 - end - end - status 200 unless pools_updated > 0 - result['ok'] = true - result - end - end - - def sync_pool_templates - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - pool_index = pool_index(pools) - template_configs = backend.hgetall('vmpooler__config__template') - template_configs&.each do |poolname, template| - next unless pool_index.include? poolname - - pools[pool_index[poolname]]['template'] = template - end - end - end - - def sync_pool_sizes - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - pool_index = pool_index(pools) - poolsize_configs = backend.hgetall('vmpooler__config__poolsize') - poolsize_configs&.each do |poolname, size| - next unless pool_index.include? poolname - - pools[pool_index[poolname]]['size'] = size.to_i - end - end - end - - def sync_clone_targets - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - pool_index = pool_index(pools) - clone_target_configs = backend.hgetall('vmpooler__config__clone_target') - clone_target_configs&.each do |poolname, clone_target| - next unless pool_index.include? poolname - - pools[pool_index[poolname]]['clone_target'] = clone_target - end - end - end - - def too_many_requested?(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - payload&.each do |poolname, count| - next unless count.to_i > config['max_ondemand_instances_per_request'] - - metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}") - return true - end - false - end - end - - def generate_ondemand_request(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - result = { 'ok': false } - - requested_instances = payload.reject { |k, _v| k == 'request_id' } - if too_many_requested?(requested_instances) - e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" - result['message'] = e_message - status 403 - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request', - 'error.message' => "403 due to #{e_message}" - }) - return result - end - - score = Time.now.to_i - request_id = payload['request_id'] - request_id ||= generate_request_id - result['request_id'] = request_id - span.set_attribute('vmpooler.request_id', request_id) - - if backend.exists?("vmpooler__odrequest__#{request_id}") - e_message = "request_id '#{request_id}' has already been created" - result['message'] = e_message - status 409 - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request', - 'error.message' => "409 due to #{e_message}" - }) - metrics.increment('ondemandrequest_generate.duplicaterequests') - return result - end - - status 201 - - platforms_with_aliases = [] - requested_instances.each do |poolname, count| - selection = evaluate_template_aliases(poolname, count) - selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } - end - platforms_string = platforms_with_aliases.join(',') - - return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) - - backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) - if Vmpooler::API.settings.config[:auth] and has_token? - token_token = request.env['HTTP_X_AUTH_TOKEN'] - token_user = backend.hget("vmpooler__token__#{token_token}", 'user') - backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token) - backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user) - span.set_attribute('enduser.id', token_user) - end - - result['domain'] = config['domain'] if config['domain'] - result[:ok] = true - metrics.increment('ondemandrequest_generate.success') - result - end - end - - def generate_request_id - SecureRandom.uuid - end - - get '/' do - sync_pool_sizes - redirect to('/dashboard/') - end - - # Provide run-time statistics - # - # Example: - # - # { - # "boot": { - # "duration": { - # "average": 163.6, - # "min": 65.49, - # "max": 830.07, - # "total": 247744.71000000002 - # }, - # "count": { - # "total": 1514 - # } - # }, - # "capacity": { - # "current": 968, - # "total": 975, - # "percent": 99.3 - # }, - # "clone": { - # "duration": { - # "average": 17.0, - # "min": 4.66, - # "max": 637.96, - # "total": 25634.15 - # }, - # "count": { - # "total": 1507 - # } - # }, - # "queue": { - # "pending": 12, - # "cloning": 0, - # "booting": 12, - # "ready": 968, - # "running": 367, - # "completed": 0, - # "total": 1347 - # }, - # "pools": { - # "ready": 100, - # "running": 120, - # "pending": 5, - # "max": 250, - # } - # "status": { - # "ok": true, - # "message": "Battle station fully armed and operational.", - # "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools - # "redhat-7-x86_64", - # "ubuntu-1404-i386" - # ], - # "uptime": 179585.9 - # } - # - # If the query parameter 'view' is provided, it will be used to select which top level - # element to compute and return. Select them by specifying them in a comma separated list. - # For example /status?view=capacity,boot - # would return only the "capacity" and "boot" statistics. "status" is always returned - - get "#{api_prefix}/status/?" do - content_type :json - - if params[:view] - views = params[:view].split(",") - end - - result = { - status: { - ok: true, - message: 'Battle station fully armed and operational.' - } - } - - sync_pool_sizes - - result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity") - result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue") - result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone") - result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot") - - # Check for empty pools - result[:pools] = {} unless views and not views.include?("pools") - ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) - running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend) - pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend) - lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend) - - unless views and not views.include?("pools") - pools.each do |pool| - # REMIND: move this out of the API and into the back-end - ready = ready_hash[pool['name']] - running = running_hash[pool['name']] - pending = pending_hash[pool['name']] - max = pool['size'] - lastBoot = lastBoot_hash[pool['name']] - aka = pool['alias'] - - result[:pools][pool['name']] = { - ready: ready, - running: running, - pending: pending, - max: max, - lastBoot: lastBoot - } - - if aka - result[:pools][pool['name']][:alias] = aka - end - - # for backwards compatibility, include separate "empty" stats in "status" block - if ready == 0 && max != 0 - result[:status][:empty] ||= [] - result[:status][:empty].push(pool['name']) - - result[:status][:ok] = false - result[:status][:message] = "Found #{result[:status][:empty].length} empty pools." - end - end - end - - result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime] - - JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) - end - - # request statistics for specific pools by passing parameter 'pool' - # with a coma separated list of pools we want to query ?pool=ABC,DEF - # returns the ready, max numbers and the aliases (if set) - get "#{api_prefix}/poolstat/?" do - content_type :json - - result = {} - - poolscopy = [] - - if params[:pool] - subpool = params[:pool].split(",") - poolscopy = pools.select do |p| - if subpool.include?(p['name']) - true - elsif !p['alias'].nil? - if p['alias'].instance_of?(Array) - (p['alias'] & subpool).any? - elsif p['alias'].instance_of?(String) - subpool.include?(p['alias']) - end - end - end - end - - result[:pools] = {} - - poolscopy.each do |pool| - result[:pools][pool['name']] = {} - - max = pool['size'] - aka = pool['alias'] - - result[:pools][pool['name']][:max] = max - - if aka - result[:pools][pool['name']][:alias] = aka - end - end - - ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend) - - ready_hash.each { |k, v| result[:pools][k][:ready] = v } - - JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) - end - - # requests the total number of running VMs - get "#{api_prefix}/totalrunning/?" do - content_type :json - queue = { - running: 0 - } - - queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) - - JSON.pretty_generate(queue) - end - - get "#{api_prefix}/summary/?" do - content_type :json - - result = { - daily: [] - } - - from_param = params[:from] || Date.today.to_s - to_param = params[:to] || Date.today.to_s - - # Validate date formats - [from_param, to_param].each do |param| - if !validate_date_str(param.to_s) - halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." - end - end - - from_date, to_date = Date.parse(from_param), Date.parse(to_param) - - if to_date < from_date - halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' - elsif from_date > Date.today - halt 400, 'Date range is invalid, \'from\' must be in the past.' - end - - boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true) - clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true) - tag = get_tag_summary(backend, from_date, to_date) - - result[:boot] = boot[:boot] - result[:clone] = clone[:clone] - result[:tag] = tag[:tag] - - daily = {} - - boot[:daily].each do |day| - daily[day[:date]] ||= {} - daily[day[:date]][:boot] = day[:boot] - end - - clone[:daily].each do |day| - daily[day[:date]] ||= {} - daily[day[:date]][:clone] = day[:clone] - end - - tag[:daily].each do |day| - daily[day[:date]] ||= {} - daily[day[:date]][:tag] = day[:tag] - end - - daily.each_key do |day| - result[:daily].push({ - date: day, - boot: daily[day][:boot], - clone: daily[day][:clone], - tag: daily[day][:tag] - }) - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/summary/:route/?:key?/?" do - content_type :json - - result = {} - - from_param = params[:from] || Date.today.to_s - to_param = params[:to] || Date.today.to_s - - # Validate date formats - [from_param, to_param].each do |param| - if !validate_date_str(param.to_s) - halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." - end - end - - from_date, to_date = Date.parse(from_param), Date.parse(to_param) - - if to_date < from_date - halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' - elsif from_date > Date.today - halt 400, 'Date range is invalid, \'from\' must be in the past.' - end - - case params[:route] - when 'boot' - result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key]) - when 'clone' - result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key]) - when 'tag' - result = get_tag_summary(backend, from_date, to_date, :only => params[:key]) - else - halt 404, JSON.pretty_generate({ 'ok' => false }) - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/token/?" do - content_type :json - - status 404 - result = { 'ok' => false } - - if Vmpooler::API.settings.config[:auth] - status 401 - - need_auth! - - backend.keys('vmpooler__token__*').each do |key| - data = backend.hgetall(key) - - if data['user'] == Rack::Auth::Basic::Request.new(request.env).username - span = OpenTelemetry::Trace.current_span - span.set_attribute('enduser.id', data['user']) - token = key.split('__').last - - result[token] ||= {} - - result[token]['created'] = data['created'] - result[token]['last'] = data['last'] || 'never' - - result['ok'] = true - end - end - - if result['ok'] - status 200 - else - status 404 - end - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/token/:token/?" do - content_type :json - - status 404 - result = { 'ok' => false } - - if Vmpooler::API.settings.config[:auth] - token = backend.hgetall("vmpooler__token__#{params[:token]}") - - if not token.nil? and not token.empty? - status 200 - - pools.each do |pool| - backend.smembers("vmpooler__running__#{pool['name']}").each do |vm| - if backend.hget("vmpooler__vm__#{vm}", 'token:token') == params[:token] - token['vms'] ||= {} - token['vms']['running'] ||= [] - token['vms']['running'].push(vm) - end - end - end - - result = { 'ok' => true, params[:token] => token } - end - end - - JSON.pretty_generate(result) - end - - delete "#{api_prefix}/token/:token/?" do - content_type :json - - status 404 - result = { 'ok' => false } - - if Vmpooler::API.settings.config[:auth] - status 401 - - need_auth! - - if backend.del("vmpooler__token__#{params[:token]}").to_i > 0 - status 200 - result['ok'] = true - end - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/token" do - content_type :json - - status 404 - result = { 'ok' => false } - - if Vmpooler::API.settings.config[:auth] - status 401 - - need_auth! - - o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join - - backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username) - backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now) - span = OpenTelemetry::Trace.current_span - span.set_attribute('enduser.id', @auth.username) - - status 200 - result['ok'] = true - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/vm/?" do - content_type :json - - result = [] - - pools.each do |pool| - result.push(pool['name']) - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/ondemandvm/?" do - content_type :json - metrics.increment('http_requests_vm_total.post.ondemand.requestid') - - need_token! if Vmpooler::API.settings.config[:auth] - - result = { 'ok' => false } - - begin - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) - if invalid.empty? - result = generate_ondemand_request(payload) - else - result[:bad_templates] = invalid - invalid.each do |bad_template| - metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('ondemandrequest_fail.invalid.unknown') - status 404 - end - rescue JSON::ParserError - span = OpenTelemetry::Trace.current_span - span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') - status 400 - result = { - 'ok' => false, - 'message' => 'JSON payload could not be parsed' - } - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/ondemandvm/:template/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.delete.ondemand.template') - - need_token! if Vmpooler::API.settings.config[:auth] - - payload = extract_templates_from_query_params(params[:template]) - - if payload - invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) - if invalid.empty? - result = generate_ondemand_request(payload) - else - result[:bad_templates] = invalid - invalid.each do |bad_template| - metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('ondemandrequest_fail.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/ondemandvm/:requestid/?" do - content_type :json - metrics.increment('http_requests_vm_total.get.ondemand.request') - - status 404 - result = check_ondemand_request(params[:requestid]) - - JSON.pretty_generate(result) - end - - delete "#{api_prefix}/ondemandvm/:requestid/?" do - content_type :json - need_token! if Vmpooler::API.settings.config[:auth] - metrics.increment('http_requests_vm_total.delete.ondemand.request') - - status 404 - result = delete_ondemand_request(params[:requestid]) - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/vm/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.post.vm.checkout') - - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_templates(payload) - if invalid.empty? - result = atomically_allocate_vms(payload) - else - invalid.each do |bad_template| - metrics.increment("checkout.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('checkout.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - def extract_templates_from_query_params(params) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - payload = {} - - params.split('+').each do |template| - payload[template] ||= 0 - payload[template] += 1 - end - - payload - end - end - - def invalid_templates(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - invalid = [] - payload.keys.each do |template| - invalid << template unless pool_exists?(template) - end - invalid - end - end - - def invalid_template_or_size(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - invalid = [] - payload.each do |pool, size| - invalid << pool unless pool_exists?(pool) - unless is_integer?(size) - invalid << pool - next - end - invalid << pool unless Integer(size) >= 0 - end - invalid - end - end - - def invalid_template_or_path(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - invalid = [] - payload.each do |pool, template| - invalid << pool unless pool_exists?(pool) - invalid << pool unless template.include? '/' - invalid << pool if template[0] == '/' - invalid << pool if template[-1] == '/' - end - invalid - end - end - - def invalid_pool(payload) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do - invalid = [] - payload.each do |pool, _clone_target| - invalid << pool unless pool_exists?(pool) - end - invalid - end - end - - def check_ondemand_request(request_id) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - span.set_attribute('vmpooler.request_id', request_id) - result = { 'ok' => false } - request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") - if request_hash.empty? - e_message = "no request found for request_id '#{request_id}'" - result['message'] = e_message - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V1.check_ondemand_request', - 'error.message' => e_message - }) - return result - end - - result['request_id'] = request_id - result['ready'] = false - result['ok'] = true - status 202 - - case request_hash['status'] - when 'ready' - result['ready'] = true - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| - instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - - if result.key?(platform_alias) - result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances - else - result[platform_alias] = { 'hostname': instances } - end - end - result['domain'] = config['domain'] if config['domain'] - status 200 - when 'failed' - result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" - status 200 - when 'deleted' - result['message'] = 'The request has been deleted' - status 200 - else - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| - instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - instances_pending = count.to_i - instance_count.to_i - - if result.key?(platform_alias) && result[platform_alias].key?(:ready) - result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s - result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s - else - result[platform_alias] = { - 'ready': instance_count.to_s, - 'pending': instances_pending.to_s - } - end - end - end - - result - end - end - - def delete_ondemand_request(request_id) - tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| - span.set_attribute('vmpooler.request_id', request_id) - result = { 'ok' => false } - - platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') - unless platforms - e_message = "no request found for request_id '#{request_id}'" - result['message'] = e_message - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V1.delete_ondemand_request', - 'error.message' => e_message - }) - return result - end - - if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' - result['message'] = 'the request has already been deleted' - else - backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') - - Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count| - backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm| - backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) - end - backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - end - backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) - end - status 200 - result['ok'] = true - result - end - end - - post "#{api_prefix}/vm/:template/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.get.vm.template') - - payload = extract_templates_from_query_params(params[:template]) - - if payload - invalid = invalid_templates(payload) - if invalid.empty? - result = atomically_allocate_vms(payload) - else - invalid.each do |bad_template| - metrics.increment("checkout.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('checkout.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/vm/:hostname/?" do - content_type :json - metrics.increment('http_requests_vm_total.get.vm.hostname') - - result = {} - - status 404 - result['ok'] = false - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") - unless rdata.empty? - status 200 - result['ok'] = true - - result[params[:hostname]] = {} - - result[params[:hostname]]['template'] = rdata['template'] - result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i - - if rdata['destroy'] - result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout'] - result[params[:hostname]]['state'] = 'destroyed' - elsif rdata['checkout'] - result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2) - result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2) - result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339 - result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339 - result[params[:hostname]]['state'] = 'running' - elsif rdata['check'] - result[params[:hostname]]['state'] = 'ready' - else - result[params[:hostname]]['state'] = 'pending' - end - - rdata.keys.each do |key| - if key.match('^tag\:(.+?)$') - result[params[:hostname]]['tags'] ||= {} - result[params[:hostname]]['tags'][$1] = rdata[key] - end - - if key.match('^snapshot\:(.+?)$') - result[params[:hostname]]['snapshots'] ||= [] - result[params[:hostname]]['snapshots'].push($1) - end - end - - if rdata['disk'] - result[params[:hostname]]['disk'] = rdata['disk'].split(':') - end - - # Look up IP address of the hostname - begin - ipAddress = TCPSocket.gethostbyname(params[:hostname])[3] - rescue StandardError - ipAddress = "" - end - - result[params[:hostname]]['ip'] = ipAddress - - if config['domain'] - result[params[:hostname]]['domain'] = config['domain'] - end - - result[params[:hostname]]['host'] = rdata['host'] if rdata['host'] - result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated'] - - end - - JSON.pretty_generate(result) - end - - delete "#{api_prefix}/vm/:hostname/?" do - content_type :json - metrics.increment('http_requests_vm_total.delete.vm.hostname') - - result = {} - - status 404 - result['ok'] = false - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") - unless rdata.empty? - need_token! if rdata['token:token'] - - if backend.srem("vmpooler__running__#{rdata['template']}", params[:hostname]) - backend.sadd("vmpooler__completed__#{rdata['template']}", params[:hostname]) - - status 200 - result['ok'] = true - metrics.increment('delete.success') - update_user_metrics('destroy', params[:hostname]) if Vmpooler::API.settings.config[:config]['usage_stats'] - else - metrics.increment('delete.failed') - end - end - - JSON.pretty_generate(result) - end - - put "#{api_prefix}/vm/:hostname/?" do - content_type :json - metrics.increment('http_requests_vm_total.put.vm.modify') - - status 404 - result = { 'ok' => false } - - failure = [] - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - if backend.exists?("vmpooler__vm__#{params[:hostname]}") - begin - jdata = JSON.parse(request.body.read) - rescue StandardError => e - span = OpenTelemetry::Trace.current_span - span.record_exception(e) - span.status = OpenTelemetry::Trace::Status.error(e.to_s) - halt 400, JSON.pretty_generate(result) - end - - # Validate data payload - jdata.each do |param, arg| - case param - when 'lifetime' - need_token! if Vmpooler::API.settings.config[:auth] - - # in hours, defaults to one week - max_lifetime_upper_limit = config['max_lifetime_upper_limit'] - if max_lifetime_upper_limit - max_lifetime_upper_limit = max_lifetime_upper_limit.to_i - if arg.to_i >= max_lifetime_upper_limit - failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.") - end - end - - # validate lifetime is within boundaries - unless arg.to_i > 0 - failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.") - end - - when 'tags' - failure.push("You provided tags (#{arg}) as something other than a hash.") unless arg.is_a?(Hash) - failure.push("You provided unsuppored tags (#{arg}).") if config['allowed_tags'] && !(arg.keys - config['allowed_tags']).empty? - else - failure.push("Unknown argument #{arg}.") - end - end - - if !failure.empty? - status 400 - result['failure'] = failure - else - jdata.each do |param, arg| - case param - when 'lifetime' - need_token! if Vmpooler::API.settings.config[:auth] - - arg = arg.to_i - - backend.hset("vmpooler__vm__#{params[:hostname]}", param, arg) - when 'tags' - filter_tags(arg) - export_tags(backend, params[:hostname], arg) - end - end - - status 200 - result['ok'] = true - end - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/vm/:hostname/disk/:size/?" do - content_type :json - metrics.increment('http_requests_vm_total.post.vm.disksize') - - need_token! if Vmpooler::API.settings.config[:auth] - - status 404 - result = { 'ok' => false } - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - if ((params[:size].to_i > 0 )and (backend.exists?("vmpooler__vm__#{params[:hostname]}"))) - result[params[:hostname]] = {} - result[params[:hostname]]['disk'] = "+#{params[:size]}gb" - - backend.sadd('vmpooler__tasks__disk', "#{params[:hostname]}:#{params[:size]}") - - status 202 - result['ok'] = true - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/vm/:hostname/snapshot/?" do - content_type :json - metrics.increment('http_requests_vm_total.post.vm.snapshot') - - need_token! if Vmpooler::API.settings.config[:auth] - - status 404 - result = { 'ok' => false } - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - if backend.exists?("vmpooler__vm__#{params[:hostname]}") - result[params[:hostname]] = {} - - o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join - - backend.sadd('vmpooler__tasks__snapshot', "#{params[:hostname]}:#{result[params[:hostname]]['snapshot']}") - - status 202 - result['ok'] = true - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do - content_type :json - metrics.increment('http_requests_vm_total.post.vm.snapshot') - - need_token! if Vmpooler::API.settings.config[:auth] - - status 404 - result = { 'ok' => false } - - params[:hostname] = hostname_shorten(params[:hostname], config['domain']) - - unless backend.hget("vmpooler__vm__#{params[:hostname]}", "snapshot:#{params[:snapshot]}").to_i.zero? - backend.sadd('vmpooler__tasks__snapshot-revert', "#{params[:hostname]}:#{params[:snapshot]}") - - status 202 - result['ok'] = true - end - - JSON.pretty_generate(result) - end - - delete "#{api_prefix}/config/poolsize/:pool/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - if pool_exists?(params[:pool]) - result = reset_pool_size(params[:pool]) - else - metrics.increment('config.invalid.unknown') - status 404 - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/config/poolsize/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_template_or_size(payload) - if invalid.empty? - result = update_pool_size(payload) - else - invalid.each do |bad_template| - metrics.increment("config.invalid.#{bad_template}") - end - result[:not_configured] = invalid - status 400 - end - else - metrics.increment('config.invalid.unknown') - status 404 - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - delete "#{api_prefix}/config/pooltemplate/:pool/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - if pool_exists?(params[:pool]) - result = reset_pool_template(params[:pool]) - else - metrics.increment('config.invalid.unknown') - status 404 - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/config/pooltemplate/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_template_or_path(payload) - if invalid.empty? - result = update_pool_template(payload) - else - invalid.each do |bad_template| - metrics.increment("config.invalid.#{bad_template}") - end - result[:bad_templates] = invalid - status 400 - end - else - metrics.increment('config.invalid.unknown') - status 404 - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/poolreset/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - begin - payload = JSON.parse(request.body.read) - if payload - invalid = invalid_templates(payload) - if invalid.empty? - result = reset_pool(payload) - else - invalid.each do |bad_pool| - metrics.increment("poolreset.invalid.#{bad_pool}") - end - result[:bad_pools] = invalid - status 400 - end - else - metrics.increment('poolreset.invalid.unknown') - status 404 - end - rescue JSON::ParserError - span = OpenTelemetry::Trace.current_span - span.record_exception(e) - span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') - status 400 - result = { - 'ok' => false, - 'message' => 'JSON payload could not be parsed' - } - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/config/clonetarget/?" do - content_type :json - result = { 'ok' => false } - - if config['experimental_features'] - need_token! if Vmpooler::API.settings.config[:auth] - - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_pool(payload) - if invalid.empty? - result = update_clone_target(payload) - else - invalid.each do |bad_template| - metrics.increment("config.invalid.#{bad_template}") - end - result[:bad_templates] = invalid - status 400 - end - else - metrics.increment('config.invalid.unknown') - status 404 - end - else - status 405 - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/config/?" do - content_type :json - result = { 'ok' => false } - status 404 - - if pools - sync_pool_sizes - sync_pool_templates - - pool_configuration = [] - pools.each do |pool| - pool['template_ready'] = template_ready?(pool, backend) - pool_configuration << pool - end - - result = { - pool_configuration: pool_configuration, - status: { - ok: true - } - } - - status 200 - end - JSON.pretty_generate(result) - end - - get "#{api_prefix}/full_config/?" do - content_type :json - - result = { - full_config: full_config, - status: { - ok: true - } - } - - status 200 - JSON.pretty_generate(result) - end - end - end -end diff --git a/lib/vmpooler/api/v2.rb b/lib/vmpooler/api/v2.rb deleted file mode 100644 index 44dad6e..0000000 --- a/lib/vmpooler/api/v2.rb +++ /dev/null @@ -1,505 +0,0 @@ -# frozen_string_literal: true - -require 'vmpooler/api/v1' - -module Vmpooler - class API - class V2 < Vmpooler::API::V1 - api_version = '2' - api_prefix = "/api/v#{api_version}" - - def full_config - Vmpooler::API.settings.config - end - - def get_template_aliases(template) - tracer.in_span("Vmpooler::API::V2.#{__method__}") do - result = [] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - result += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) - end - result - end - end - - # Fetch a single vm from a pool - # - # @param [String] template - # The template that the vm should be created from - # - # @return [Tuple] vmname, vmpool, vmtemplate - # Returns a tuple containing the vm's name, the pool it came from, and - # what template was used, if successful. Otherwise the tuple contains. - # nil values. - def fetch_single_vm(template) - tracer.in_span("Vmpooler::API::V2.#{__method__}") do - template_backends = [template] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - template_backends += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) - pool_index = pool_index(pools) - weighted_pools = {} - template_backends.each do |t| - next unless pool_index.key? t - - index = pool_index[t] - clone_target = pools[index]['clone_target'] || config['clone_target'] - next unless config.key?('backend_weight') - - weight = config['backend_weight'][clone_target] - if weight - weighted_pools[t] = weight - end - end - - if weighted_pools.count == template_backends.count - pickup = Pickup.new(weighted_pools) - selection = pickup.pick - template_backends.delete(selection) - template_backends.unshift(selection) - else - first = template_backends.sample - template_backends.delete(first) - template_backends.unshift(first) - end - end - - checkoutlock.synchronize do - template_backends.each do |template_backend| - vms = backend.smembers("vmpooler__ready__#{template_backend}") - next if vms.empty? - - vm = vms.pop - smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) - if smoved - return [vm, template_backend, template] - end - end - [nil, nil, nil] - end - end - end - - # The domain in the result body will be set to the one associated with the - # last vm added. The part of the response is only being retained for - # backwards compatibility as the hostnames are now fqdn's instead of bare - # hostnames. This change is a result of now being able to specify a domain - # per pool. If no vm's in the result had a domain sepcified then the - # domain key will be omitted similar to how it was previously omitted if - # the global option domain wasn't specified. - def atomically_allocate_vms(payload) - tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| - result = { 'ok' => false } - failed = false - vms = [] # vmpool, vmname, vmtemplate - - validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? - - payload.each do |requested, count| - count.to_i.times do |_i| - vmname, vmpool, vmtemplate = fetch_single_vm(requested) - if vmname - account_for_starting_vm(vmpool, vmname) - vms << [vmpool, vmname, vmtemplate] - metrics.increment("checkout.success.#{vmpool}") - update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] - else - failed = true - metrics.increment("checkout.empty.#{requested}") - break - end - end - end - - if failed - vms.each do |(vmpool, vmname, _vmtemplate)| - return_vm_to_ready_state(vmpool, vmname) - end - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V2.atomically_allocate_vms', - 'error.message' => '503 due to failing to allocate one or more vms' - }) - status 503 - else - vm_names = [] - vms.each do |(vmpool, vmname, vmtemplate)| - vmdomain = Parsing.get_domain_for_pool(full_config, vmpool) - if vmdomain - vmfqdn = "#{vmname}.#{vmdomain}" - update_result_hosts(result, vmtemplate, vmfqdn) - vm_names.append(vmfqdn) - else - update_result_hosts(result, vmtemplate, vmname) - vm_names.append(vmname) - end - end - - span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty? - - result['ok'] = true - end - - result - end - end - - def generate_ondemand_request(payload) - tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| - result = { 'ok': false } - - requested_instances = payload.reject { |k, _v| k == 'request_id' } - if too_many_requested?(requested_instances) - e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" - result['message'] = e_message - status 403 - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request', - 'error.message' => "403 due to #{e_message}" - }) - return result - end - - score = Time.now.to_i - request_id = payload['request_id'] - request_id ||= generate_request_id - result['request_id'] = request_id - span.set_attribute('vmpooler.request_id', request_id) - - if backend.exists?("vmpooler__odrequest__#{request_id}") - e_message = "request_id '#{request_id}' has already been created" - result['message'] = e_message - status 409 - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request', - 'error.message' => "409 due to #{e_message}" - }) - metrics.increment('ondemandrequest_generate.duplicaterequests') - return result - end - - status 201 - - platforms_with_aliases = [] - requested_instances.each do |poolname, count| - selection = evaluate_template_aliases(poolname, count) - selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } - end - platforms_string = platforms_with_aliases.join(',') - - return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) - - backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) - if Vmpooler::API.settings.config[:auth] and has_token? - token_token = request.env['HTTP_X_AUTH_TOKEN'] - token_user = backend.hget("vmpooler__token__#{token_token}", 'user') - backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token) - backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user) - span.set_attribute('enduser.id', token_user) - end - - result[:ok] = true - metrics.increment('ondemandrequest_generate.success') - result - end - end - - # Endpoints that use overridden methods - - post "#{api_prefix}/vm/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.post.vm.checkout') - - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_templates(payload) - if invalid.empty? - result = atomically_allocate_vms(payload) - else - invalid.each do |bad_template| - metrics.increment("checkout.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('checkout.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/vm/:template/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.get.vm.template') - - payload = extract_templates_from_query_params(params[:template]) - - if payload - invalid = invalid_templates(payload) - if invalid.empty? - result = atomically_allocate_vms(payload) - else - invalid.each do |bad_template| - metrics.increment("checkout.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('checkout.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/vm/:hostname/?" do - content_type :json - metrics.increment('http_requests_vm_total.get.vm.hostname') - - result = {} - - status 404 - result['ok'] = false - - params[:hostname] = hostname_shorten(params[:hostname], nil) - - rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") - unless rdata.empty? - status 200 - result['ok'] = true - - result[params[:hostname]] = {} - - result[params[:hostname]]['template'] = rdata['template'] - result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i - - if rdata['destroy'] - result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout'] - result[params[:hostname]]['state'] = 'destroyed' - elsif rdata['checkout'] - result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2) - result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2) - result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339 - result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339 - result[params[:hostname]]['state'] = 'running' - elsif rdata['check'] - result[params[:hostname]]['state'] = 'ready' - else - result[params[:hostname]]['state'] = 'pending' - end - - rdata.keys.each do |key| - if key.match('^tag\:(.+?)$') - result[params[:hostname]]['tags'] ||= {} - result[params[:hostname]]['tags'][$1] = rdata[key] - end - - if key.match('^snapshot\:(.+?)$') - result[params[:hostname]]['snapshots'] ||= [] - result[params[:hostname]]['snapshots'].push($1) - end - end - - if rdata['disk'] - result[params[:hostname]]['disk'] = rdata['disk'].split(':') - end - - # Look up IP address of the hostname - begin - ipAddress = TCPSocket.gethostbyname(params[:hostname])[3] - rescue StandardError - ipAddress = "" - end - - result[params[:hostname]]['ip'] = ipAddress - - if rdata['pool'] - vmdomain = Parsing.get_domain_for_pool(full_config, rdata['pool']) - if vmdomain - result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}" - end - end - - result[params[:hostname]]['host'] = rdata['host'] if rdata['host'] - result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated'] - - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/ondemandvm/?" do - content_type :json - metrics.increment('http_requests_vm_total.post.ondemand.requestid') - - need_token! if Vmpooler::API.settings.config[:auth] - - result = { 'ok' => false } - - begin - payload = JSON.parse(request.body.read) - - if payload - invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) - if invalid.empty? - result = generate_ondemand_request(payload) - else - result[:bad_templates] = invalid - invalid.each do |bad_template| - metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('ondemandrequest_fail.invalid.unknown') - status 404 - end - rescue JSON::ParserError - span = OpenTelemetry::Trace.current_span - span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') - status 400 - result = { - 'ok' => false, - 'message' => 'JSON payload could not be parsed' - } - end - - JSON.pretty_generate(result) - end - - post "#{api_prefix}/ondemandvm/:template/?" do - content_type :json - result = { 'ok' => false } - metrics.increment('http_requests_vm_total.delete.ondemand.template') - - need_token! if Vmpooler::API.settings.config[:auth] - - payload = extract_templates_from_query_params(params[:template]) - - if payload - invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) - if invalid.empty? - result = generate_ondemand_request(payload) - else - result[:bad_templates] = invalid - invalid.each do |bad_template| - metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") - end - status 404 - end - else - metrics.increment('ondemandrequest_fail.invalid.unknown') - status 404 - end - - JSON.pretty_generate(result) - end - - get "#{api_prefix}/ondemandvm/:requestid/?" do - content_type :json - metrics.increment('http_requests_vm_total.get.ondemand.request') - - status 404 - result = check_ondemand_request(params[:requestid]) - - JSON.pretty_generate(result) - end - - def check_ondemand_request(request_id) - tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span| - span.set_attribute('vmpooler.request_id', request_id) - result = { 'ok' => false } - request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") - if request_hash.empty? - e_message = "no request found for request_id '#{request_id}'" - result['message'] = e_message - span.add_event('error', attributes: { - 'error.type' => 'Vmpooler::API::V2.check_ondemand_request', - 'error.message' => e_message - }) - return result - end - - result['request_id'] = request_id - result['ready'] = false - result['ok'] = true - status 202 - - case request_hash['status'] - when 'ready' - result['ready'] = true - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| - instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - domain = Parsing.get_domain_for_pool(full_config, pool) - instances.map! { |instance| instance.concat(".#{domain}") } if domain - - if result.key?(platform_alias) - result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances - else - result[platform_alias] = { 'hostname': instances } - end - end - status 200 - when 'failed' - result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" - status 200 - when 'deleted' - result['message'] = 'The request has been deleted' - status 200 - else - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| - instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - instances_pending = count.to_i - instance_count.to_i - - if result.key?(platform_alias) && result[platform_alias].key?(:ready) - result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s - result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s - else - result[platform_alias] = { - 'ready': instance_count.to_s, - 'pending': instances_pending.to_s - } - end - end - end - - result - end - end - - # Endpoints that only use bits from the V1 api are called here - # Note that traces will be named based on the route used in the V1 api - # but the http.url trace attribute will still have the actual requested url in it - - delete "#{api_prefix}/*" do - versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") - request.path_info = "/api/v1/#{versionless_path_info}" - call env - end - - get "#{api_prefix}/*" do - versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") - request.path_info = "/api/v1/#{versionless_path_info}" - call env - end - - post "#{api_prefix}/*" do - versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") - request.path_info = "/api/v1/#{versionless_path_info}" - call env - end - - put "#{api_prefix}/*" do - versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/") - request.path_info = "/api/v1/#{versionless_path_info}" - call env - end - end - end -end diff --git a/lib/vmpooler/api/v3.rb b/lib/vmpooler/api/v3.rb new file mode 100644 index 0000000..4e8f6e9 --- /dev/null +++ b/lib/vmpooler/api/v3.rb @@ -0,0 +1,1761 @@ +# frozen_string_literal: true + +require 'vmpooler/util/parsing' +require 'vmpooler/dns' + +module Vmpooler + class API + class V3 < Sinatra::Base + api_version = '3' + api_prefix = "/api/v#{api_version}" + + helpers do + include Vmpooler::API::Helpers + end + + def backend + Vmpooler::API.settings.redis + end + + def metrics + Vmpooler::API.settings.metrics + end + + def config + Vmpooler::API.settings.config[:config] + end + + def full_config + Vmpooler::API.settings.config + end + + def pools + Vmpooler::API.settings.config[:pools] + end + + def pools_at_startup + Vmpooler::API.settings.config[:pools_at_startup] + end + + def pool_exists?(template) + Vmpooler::API.settings.config[:pool_names].include?(template) + end + + def need_auth! + validate_auth(backend) + end + + def need_token! + validate_token(backend) + end + + def checkoutlock + Vmpooler::API.settings.checkoutlock + end + + def get_template_aliases(template) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = [] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + result += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + end + result + end + end + + def get_pool_weights(template_backends) + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + weighted_pools + end + + def count_selection(selection) + result = {} + selection.uniq.each do |poolname| + result[poolname] = selection.count(poolname) + end + result + end + + def evaluate_template_aliases(template, count) + template_backends = [] + template_backends << template if backend.sismember('vmpooler__pools', template) + selection = [] + aliases = get_template_aliases(template) + if aliases + template_backends += aliases + weighted_pools = get_pool_weights(template_backends) + + if weighted_pools.count > 1 && weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + count.to_i.times do + selection << pickup.pick + end + else + count.to_i.times do + selection << template_backends.sample + end + end + end + + count_selection(selection) + end + + # Fetch a single vm from a pool + # + # @param [String] template + # The template that the vm should be created from + # + # @return [Tuple] vmname, vmpool, vmtemplate + # Returns a tuple containing the vm's name, the pool it came from, and + # what template was used, if successful. Otherwise the tuple contains. + # nil values. + def fetch_single_vm(template) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + template_backends = [template] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + template_backends += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + + if weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + selection = pickup.pick + template_backends.delete(selection) + template_backends.unshift(selection) + else + first = template_backends.sample + template_backends.delete(first) + template_backends.unshift(first) + end + end + + checkoutlock.synchronize do + template_backends.each do |template_backend| + vms = backend.smembers("vmpooler__ready__#{template_backend}") + next if vms.empty? + + vm = vms.pop + smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) + if smoved + return [vm, template_backend, template] + end + end + [nil, nil, nil] + end + end + end + + def return_vm_to_ready_state(template, vm) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + backend.srem("vmpooler__migrating__#{template}", vm) + backend.hdel("vmpooler__active__#{template}", vm) + backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user') + backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm) + end + end + + def account_for_starting_vm(template, vm) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user') + span.set_attribute('enduser.id', user) + has_token_result = has_token? + backend.sadd("vmpooler__migrating__#{template}", vm) + backend.hset("vmpooler__active__#{template}", vm, Time.now) + backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) + + if Vmpooler::API.settings.config[:auth] and has_token_result + backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) + backend.hset("vmpooler__vm__#{vm}", 'token:user', user) + + if config['vm_lifetime_auth'].to_i > 0 + backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i) + end + end + end + end + + def update_result_hosts(result, template, vm) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result[template] ||= {} + if result[template]['hostname'] + result[template]['hostname'] = Array(result[template]['hostname']) + result[template]['hostname'].push(vm) + else + result[template]['hostname'] = vm + end + end + end + + def atomically_allocate_vms(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + result = { 'ok' => false } + failed = false + vms = [] # vmpool, vmname, vmtemplate + + validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? + + payload.each do |requested, count| + count.to_i.times do |_i| + vmname, vmpool, vmtemplate = fetch_single_vm(requested) + if vmname + account_for_starting_vm(vmpool, vmname) + vms << [vmpool, vmname, vmtemplate] + metrics.increment("checkout.success.#{vmpool}") + update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] + else + failed = true + metrics.increment("checkout.empty.#{requested}") + break + end + end + end + + if failed + vms.each do |(vmpool, vmname, _vmtemplate)| + return_vm_to_ready_state(vmpool, vmname) + end + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V3.atomically_allocate_vms', + 'error.message' => '503 due to failing to allocate one or more vms' + }) + status 503 + else + vm_names = [] + vms.each do |(vmpool, vmname, vmtemplate)| + vmdomain = Dns.get_domain_for_pool(full_config, vmpool) + vmfqdn = "#{vmname}.#{vmdomain}" + update_result_hosts(result, vmtemplate, vmfqdn) + vm_names.append(vmfqdn) + end + + span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty? + + result['ok'] = true + end + + result + end + end + + def component_to_test(match, labels_string) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + return if labels_string.nil? + + labels_string_parts = labels_string.split(',') + labels_string_parts.each do |part| + key, value = part.split('=') + next if value.nil? + return value if key == match + end + 'none' + end + end + + def update_user_metrics(operation, vmname) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + begin + backend.multi + backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url') + backend.hget("vmpooler__vm__#{vmname}", 'token:user') + backend.hget("vmpooler__vm__#{vmname}", 'template') + jenkins_build_url, user, poolname = backend.exec + poolname = poolname.gsub('.', '_') + + if user + user = user.gsub('.', '_') + else + user = 'unauthenticated' + end + metrics.increment("user.#{user}.#{operation}.#{poolname}") + + if jenkins_build_url + if jenkins_build_url.include? 'litmus' + # Very simple filter for Litmus jobs - just count them coming through for the moment. + metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}") + else + url_parts = jenkins_build_url.split('/')[2..-1] + jenkins_instance = url_parts[0].gsub('.', '_') + value_stream_parts = url_parts[2].split('_') + value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') } + value_stream = value_stream_parts.shift + branch = value_stream_parts.pop + project = value_stream_parts.shift + job_name = value_stream_parts.join('_') + build_metadata_parts = url_parts[3] + component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) + + metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") + metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") + metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") + end + end + rescue StandardError => e + puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error(e.to_s) + span.add_event('log', attributes: { + 'log.severity' => 'debug', + 'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" + }) + end + end + end + + def reset_pool_size(poolname) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + pool_index = pool_index(pools) + + pools_updated = 0 + sync_pool_sizes + + pool_size_now = pools[pool_index[poolname]]['size'].to_i + pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i + result['pool_size_before_reset'] = pool_size_now + result['pool_size_before_overrides'] = pool_size_original + + unless pool_size_now == pool_size_original + pools[pool_index[poolname]]['size'] = pool_size_original + backend.hdel('vmpooler__config__poolsize', poolname) + backend.sadd('vmpooler__pool__undo_size_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + end + + def update_pool_size(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + pool_index = pool_index(pools) + pools_updated = 0 + sync_pool_sizes + + payload.each do |poolname, size| + unless pools[pool_index[poolname]]['size'] == size.to_i + pools[pool_index[poolname]]['size'] = size.to_i + backend.hset('vmpooler__config__poolsize', poolname, size) + pools_updated += 1 + status 201 + end + end + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + end + + def reset_pool_template(poolname) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + pool_index_live = pool_index(pools) + pool_index_original = pool_index(pools_at_startup) + + pools_updated = 0 + sync_pool_templates + + template_now = pools[pool_index_live[poolname]]['template'] + template_original = pools_at_startup[pool_index_original[poolname]]['template'] + result['template_before_reset'] = template_now + result['template_before_overrides'] = template_original + + unless template_now == template_original + pools[pool_index_live[poolname]]['template'] = template_original + backend.hdel('vmpooler__config__template', poolname) + backend.sadd('vmpooler__pool__undo_template_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + end + + def update_pool_template(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + pool_index = pool_index(pools) + pools_updated = 0 + sync_pool_templates + + payload.each do |poolname, template| + unless pools[pool_index[poolname]]['template'] == template + pools[pool_index[poolname]]['template'] = template + backend.hset('vmpooler__config__template', poolname, template) + pools_updated += 1 + status 201 + end + end + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + end + + def reset_pool(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + payload.each do |poolname, _count| + backend.sadd('vmpooler__poolreset', poolname) + end + status 201 + result['ok'] = true + result + end + end + + def update_clone_target(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + result = { 'ok' => false } + + pool_index = pool_index(pools) + pools_updated = 0 + sync_clone_targets + + payload.each do |poolname, clone_target| + unless pools[pool_index[poolname]]['clone_target'] == clone_target + pools[pool_index[poolname]]['clone_target'] = clone_target + backend.hset('vmpooler__config__clone_target', poolname, clone_target) + pools_updated += 1 + status 201 + end + end + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + end + + def sync_pool_templates + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + pool_index = pool_index(pools) + template_configs = backend.hgetall('vmpooler__config__template') + template_configs&.each do |poolname, template| + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['template'] = template + end + end + end + + def sync_pool_sizes + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + pool_index = pool_index(pools) + poolsize_configs = backend.hgetall('vmpooler__config__poolsize') + poolsize_configs&.each do |poolname, size| + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['size'] = size.to_i + end + end + end + + def sync_clone_targets + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + pool_index = pool_index(pools) + clone_target_configs = backend.hgetall('vmpooler__config__clone_target') + clone_target_configs&.each do |poolname, clone_target| + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['clone_target'] = clone_target + end + end + end + + def too_many_requested?(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + payload&.each do |poolname, count| + next unless count.to_i > config['max_ondemand_instances_per_request'] + + metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}") + return true + end + false + end + end + + def generate_ondemand_request(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + result = { 'ok': false } + + requested_instances = payload.reject { |k, _v| k == 'request_id' } + if too_many_requested?(requested_instances) + e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" + result['message'] = e_message + status 403 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V3.generate_ondemand_request', + 'error.message' => "403 due to #{e_message}" + }) + return result + end + + score = Time.now.to_i + request_id = payload['request_id'] + request_id ||= generate_request_id + result['request_id'] = request_id + span.set_attribute('vmpooler.request_id', request_id) + + if backend.exists?("vmpooler__odrequest__#{request_id}") + e_message = "request_id '#{request_id}' has already been created" + result['message'] = e_message + status 409 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V3.generate_ondemand_request', + 'error.message' => "409 due to #{e_message}" + }) + metrics.increment('ondemandrequest_generate.duplicaterequests') + return result + end + + status 201 + + platforms_with_aliases = [] + requested_instances.each do |poolname, count| + selection = evaluate_template_aliases(poolname, count) + selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } + end + platforms_string = platforms_with_aliases.join(',') + + return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) + + backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) + if Vmpooler::API.settings.config[:auth] and has_token? + token_token = request.env['HTTP_X_AUTH_TOKEN'] + token_user = backend.hget("vmpooler__token__#{token_token}", 'user') + backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token) + backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user) + span.set_attribute('enduser.id', token_user) + end + + result[:ok] = true + metrics.increment('ondemandrequest_generate.success') + result + end + end + + def generate_request_id + SecureRandom.uuid + end + + get '/' do + sync_pool_sizes + redirect to('/dashboard/') + end + + # Provide run-time statistics + # + # Example: + # + # { + # "boot": { + # "duration": { + # "average": 163.6, + # "min": 65.49, + # "max": 830.07, + # "total": 247744.71000000002 + # }, + # "count": { + # "total": 1514 + # } + # }, + # "capacity": { + # "current": 968, + # "total": 975, + # "percent": 99.3 + # }, + # "clone": { + # "duration": { + # "average": 17.0, + # "min": 4.66, + # "max": 637.96, + # "total": 25634.15 + # }, + # "count": { + # "total": 1507 + # } + # }, + # "queue": { + # "pending": 12, + # "cloning": 0, + # "booting": 12, + # "ready": 968, + # "running": 367, + # "completed": 0, + # "total": 1347 + # }, + # "pools": { + # "ready": 100, + # "running": 120, + # "pending": 5, + # "max": 250, + # } + # "status": { + # "ok": true, + # "message": "Battle station fully armed and operational.", + # "empty": [ # NOTE: would not have 'ok: true' w/ "empty" pools + # "redhat-7-x86_64", + # "ubuntu-1404-i386" + # ], + # "uptime": 179585.9 + # } + # + # If the query parameter 'view' is provided, it will be used to select which top level + # element to compute and return. Select them by specifying them in a comma separated list. + # For example /status?view=capacity,boot + # would return only the "capacity" and "boot" statistics. "status" is always returned + + get "#{api_prefix}/status/?" do + content_type :json + + if params[:view] + views = params[:view].split(",") + end + + result = { + status: { + ok: true, + message: 'Battle station fully armed and operational.' + } + } + + sync_pool_sizes + + result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity") + result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue") + result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone") + result[:boot] = get_task_metrics(backend, 'boot', Date.today.to_s) unless views and not views.include?("boot") + + # Check for empty pools + result[:pools] = {} unless views and not views.include?("pools") + ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) + running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend) + pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend) + lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend) + + unless views and not views.include?("pools") + pools.each do |pool| + # REMIND: move this out of the API and into the back-end + ready = ready_hash[pool['name']] + running = running_hash[pool['name']] + pending = pending_hash[pool['name']] + max = pool['size'] + lastBoot = lastBoot_hash[pool['name']] + aka = pool['alias'] + + result[:pools][pool['name']] = { + ready: ready, + running: running, + pending: pending, + max: max, + lastBoot: lastBoot + } + + if aka + result[:pools][pool['name']][:alias] = aka + end + + # for backwards compatibility, include separate "empty" stats in "status" block + if ready == 0 && max != 0 + result[:status][:empty] ||= [] + result[:status][:empty].push(pool['name']) + + result[:status][:ok] = false + result[:status][:message] = "Found #{result[:status][:empty].length} empty pools." + end + end + end + + result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime] + + JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) + end + + # request statistics for specific pools by passing parameter 'pool' + # with a coma separated list of pools we want to query ?pool=ABC,DEF + # returns the ready, max numbers and the aliases (if set) + get "#{api_prefix}/poolstat/?" do + content_type :json + + result = {} + + poolscopy = [] + + if params[:pool] + subpool = params[:pool].split(",") + poolscopy = pools.select do |p| + if subpool.include?(p['name']) + true + elsif !p['alias'].nil? + if p['alias'].instance_of?(Array) + (p['alias'] & subpool).any? + elsif p['alias'].instance_of?(String) + subpool.include?(p['alias']) + end + end + end + end + + result[:pools] = {} + + poolscopy.each do |pool| + result[:pools][pool['name']] = {} + + max = pool['size'] + aka = pool['alias'] + + result[:pools][pool['name']][:max] = max + + if aka + result[:pools][pool['name']][:alias] = aka + end + end + + ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend) + + ready_hash.each { |k, v| result[:pools][k][:ready] = v } + + JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) + end + + # requests the total number of running VMs + get "#{api_prefix}/totalrunning/?" do + content_type :json + queue = { + running: 0 + } + + queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) + + JSON.pretty_generate(queue) + end + + get "#{api_prefix}/summary/?" do + content_type :json + + result = { + daily: [] + } + + from_param = params[:from] || Date.today.to_s + to_param = params[:to] || Date.today.to_s + + # Validate date formats + [from_param, to_param].each do |param| + if !validate_date_str(param.to_s) + halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." + end + end + + from_date, to_date = Date.parse(from_param), Date.parse(to_param) + + if to_date < from_date + halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' + elsif from_date > Date.today + halt 400, 'Date range is invalid, \'from\' must be in the past.' + end + + boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true) + clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true) + tag = get_tag_summary(backend, from_date, to_date) + + result[:boot] = boot[:boot] + result[:clone] = clone[:clone] + result[:tag] = tag[:tag] + + daily = {} + + boot[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:boot] = day[:boot] + end + + clone[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:clone] = day[:clone] + end + + tag[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:tag] = day[:tag] + end + + daily.each_key do |day| + result[:daily].push({ + date: day, + boot: daily[day][:boot], + clone: daily[day][:clone], + tag: daily[day][:tag] + }) + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/summary/:route/?:key?/?" do + content_type :json + + result = {} + + from_param = params[:from] || Date.today.to_s + to_param = params[:to] || Date.today.to_s + + # Validate date formats + [from_param, to_param].each do |param| + if !validate_date_str(param.to_s) + halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." + end + end + + from_date, to_date = Date.parse(from_param), Date.parse(to_param) + + if to_date < from_date + halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' + elsif from_date > Date.today + halt 400, 'Date range is invalid, \'from\' must be in the past.' + end + + case params[:route] + when 'boot' + result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key]) + when 'clone' + result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key]) + when 'tag' + result = get_tag_summary(backend, from_date, to_date, :only => params[:key]) + else + halt 404, JSON.pretty_generate({ 'ok' => false }) + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/token/?" do + content_type :json + + status 404 + result = { 'ok' => false } + + if Vmpooler::API.settings.config[:auth] + status 401 + + need_auth! + + backend.keys('vmpooler__token__*').each do |key| + data = backend.hgetall(key) + + if data['user'] == Rack::Auth::Basic::Request.new(request.env).username + span = OpenTelemetry::Trace.current_span + span.set_attribute('enduser.id', data['user']) + token = key.split('__').last + + result[token] ||= {} + + result[token]['created'] = data['created'] + result[token]['last'] = data['last'] || 'never' + + result['ok'] = true + end + end + + if result['ok'] + status 200 + else + status 404 + end + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/token/:token/?" do + content_type :json + + status 404 + result = { 'ok' => false } + + if Vmpooler::API.settings.config[:auth] + token = backend.hgetall("vmpooler__token__#{params[:token]}") + + if not token.nil? and not token.empty? + status 200 + + pools.each do |pool| + backend.smembers("vmpooler__running__#{pool['name']}").each do |vm| + if backend.hget("vmpooler__vm__#{vm}", 'token:token') == params[:token] + token['vms'] ||= {} + token['vms']['running'] ||= [] + token['vms']['running'].push(vm) + end + end + end + + result = { 'ok' => true, params[:token] => token } + end + end + + JSON.pretty_generate(result) + end + + delete "#{api_prefix}/token/:token/?" do + content_type :json + + status 404 + result = { 'ok' => false } + + if Vmpooler::API.settings.config[:auth] + status 401 + + need_auth! + + if backend.del("vmpooler__token__#{params[:token]}").to_i > 0 + status 200 + result['ok'] = true + end + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/token" do + content_type :json + + status 404 + result = { 'ok' => false } + + if Vmpooler::API.settings.config[:auth] + status 401 + + need_auth! + + o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + result['token'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join + + backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username) + backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now) + span = OpenTelemetry::Trace.current_span + span.set_attribute('enduser.id', @auth.username) + + status 200 + result['ok'] = true + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/vm/?" do + content_type :json + + result = [] + + pools.each do |pool| + result.push(pool['name']) + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/ondemandvm/?" do + content_type :json + metrics.increment('http_requests_vm_total.post.ondemand.requestid') + + need_token! if Vmpooler::API.settings.config[:auth] + + result = { 'ok' => false } + + begin + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('ondemandrequest_fail.invalid.unknown') + status 404 + end + rescue JSON::ParserError + span = OpenTelemetry::Trace.current_span + span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/ondemandvm/:template/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.delete.ondemand.template') + + need_token! if Vmpooler::API.settings.config[:auth] + + payload = extract_templates_from_query_params(params[:template]) + + if payload + invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment("ondemandrequest_fail.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('ondemandrequest_fail.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/ondemandvm/:requestid/?" do + content_type :json + metrics.increment('http_requests_vm_total.get.ondemand.request') + + status 404 + result = check_ondemand_request(params[:requestid]) + + JSON.pretty_generate(result) + end + + delete "#{api_prefix}/ondemandvm/:requestid/?" do + content_type :json + need_token! if Vmpooler::API.settings.config[:auth] + metrics.increment('http_requests_vm_total.delete.ondemand.request') + + status 404 + result = delete_ondemand_request(params[:requestid]) + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/vm/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.post.vm.checkout') + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment("checkout.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('checkout.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + def extract_templates_from_query_params(params) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + payload = {} + + params.split('+').each do |template| + payload[template] ||= 0 + payload[template] += 1 + end + + payload + end + end + + def invalid_templates(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + invalid = [] + payload.keys.each do |template| + invalid << template unless pool_exists?(template) + end + invalid + end + end + + def invalid_template_or_size(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + invalid = [] + payload.each do |pool, size| + invalid << pool unless pool_exists?(pool) + unless is_integer?(size) + invalid << pool + next + end + invalid << pool unless Integer(size) >= 0 + end + invalid + end + end + + def invalid_template_or_path(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + invalid = [] + payload.each do |pool, template| + invalid << pool unless pool_exists?(pool) + invalid << pool unless template.include? '/' + invalid << pool if template[0] == '/' + invalid << pool if template[-1] == '/' + end + invalid + end + end + + def invalid_pool(payload) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do + invalid = [] + payload.each do |pool, _clone_target| + invalid << pool unless pool_exists?(pool) + end + invalid + end + end + + def delete_ondemand_request(request_id) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + span.set_attribute('vmpooler.request_id', request_id) + result = { 'ok' => false } + + platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') + unless platforms + e_message = "no request found for request_id '#{request_id}'" + result['message'] = e_message + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V3.delete_ondemand_request', + 'error.message' => e_message + }) + return result + end + + if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' + result['message'] = 'the request has already been deleted' + else + backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') + + Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count| + backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm| + backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) + end + backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + end + backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) + end + status 200 + result['ok'] = true + result + end + end + + post "#{api_prefix}/vm/:template/?" do + content_type :json + result = { 'ok' => false } + metrics.increment('http_requests_vm_total.get.vm.template') + + payload = extract_templates_from_query_params(params[:template]) + + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment("checkout.invalid.#{bad_template}") + end + status 404 + end + else + metrics.increment('checkout.invalid.unknown') + status 404 + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/vm/:hostname/?" do + content_type :json + metrics.increment('http_requests_vm_total.get.vm.hostname') + + result = {} + + status 404 + result['ok'] = false + + params[:hostname] = hostname_shorten(params[:hostname]) + + rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") + unless rdata.empty? + status 200 + result['ok'] = true + + result[params[:hostname]] = {} + + result[params[:hostname]]['template'] = rdata['template'] + result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i + + if rdata['destroy'] + result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout'] + result[params[:hostname]]['state'] = 'destroyed' + elsif rdata['checkout'] + result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2) + result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2) + result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339 + result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339 + result[params[:hostname]]['state'] = 'running' + elsif rdata['check'] + result[params[:hostname]]['state'] = 'ready' + else + result[params[:hostname]]['state'] = 'pending' + end + + rdata.keys.each do |key| + if key.match('^tag\:(.+?)$') + result[params[:hostname]]['tags'] ||= {} + result[params[:hostname]]['tags'][$1] = rdata[key] + end + + if key.match('^snapshot\:(.+?)$') + result[params[:hostname]]['snapshots'] ||= [] + result[params[:hostname]]['snapshots'].push($1) + end + end + + if rdata['disk'] + result[params[:hostname]]['disk'] = rdata['disk'].split(':') + end + + # Look up IP address of the hostname + begin + ipAddress = TCPSocket.gethostbyname(params[:hostname])[3] + rescue StandardError + ipAddress = "" + end + + result[params[:hostname]]['ip'] = ipAddress + + if rdata['pool'] + vmdomain = Dns.get_domain_for_pool(full_config, rdata['pool']) + if vmdomain + result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}" + end + end + + result[params[:hostname]]['host'] = rdata['host'] if rdata['host'] + result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated'] + + end + + JSON.pretty_generate(result) + end + + def check_ondemand_request(request_id) + tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span| + span.set_attribute('vmpooler.request_id', request_id) + result = { 'ok' => false } + request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") + if request_hash.empty? + e_message = "no request found for request_id '#{request_id}'" + result['message'] = e_message + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V3.check_ondemand_request', + 'error.message' => e_message + }) + return result + end + + result['request_id'] = request_id + result['ready'] = false + result['ok'] = true + status 202 + + case request_hash['status'] + when 'ready' + result['ready'] = true + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| + instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + domain = Dns.get_domain_for_pool(full_config, pool) + instances.map! { |instance| instance.concat(".#{domain}") } + + if result.key?(platform_alias) + result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances + else + result[platform_alias] = { 'hostname': instances } + end + end + status 200 + when 'failed' + result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" + status 200 + when 'deleted' + result['message'] = 'The request has been deleted' + status 200 + else + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| + instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + instances_pending = count.to_i - instance_count.to_i + + if result.key?(platform_alias) && result[platform_alias].key?(:ready) + result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s + result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s + else + result[platform_alias] = { + 'ready': instance_count.to_s, + 'pending': instances_pending.to_s + } + end + end + end + + result + end + end + + delete "#{api_prefix}/vm/:hostname/?" do + content_type :json + metrics.increment('http_requests_vm_total.delete.vm.hostname') + + result = {} + + status 404 + result['ok'] = false + + params[:hostname] = hostname_shorten(params[:hostname]) + + rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}") + unless rdata.empty? + need_token! if rdata['token:token'] + + if backend.srem("vmpooler__running__#{rdata['template']}", params[:hostname]) + backend.sadd("vmpooler__completed__#{rdata['template']}", params[:hostname]) + + status 200 + result['ok'] = true + metrics.increment('delete.success') + update_user_metrics('destroy', params[:hostname]) if Vmpooler::API.settings.config[:config]['usage_stats'] + else + metrics.increment('delete.failed') + end + end + + JSON.pretty_generate(result) + end + + put "#{api_prefix}/vm/:hostname/?" do + content_type :json + metrics.increment('http_requests_vm_total.put.vm.modify') + + status 404 + result = { 'ok' => false } + + failure = [] + + params[:hostname] = hostname_shorten(params[:hostname]) + + if backend.exists?("vmpooler__vm__#{params[:hostname]}") + begin + jdata = JSON.parse(request.body.read) + rescue StandardError => e + span = OpenTelemetry::Trace.current_span + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error(e.to_s) + halt 400, JSON.pretty_generate(result) + end + + # Validate data payload + jdata.each do |param, arg| + case param + when 'lifetime' + need_token! if Vmpooler::API.settings.config[:auth] + + # in hours, defaults to one week + max_lifetime_upper_limit = config['max_lifetime_upper_limit'] + if max_lifetime_upper_limit + max_lifetime_upper_limit = max_lifetime_upper_limit.to_i + if arg.to_i >= max_lifetime_upper_limit + failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.") + end + end + + # validate lifetime is within boundaries + unless arg.to_i > 0 + failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.") + end + + when 'tags' + failure.push("You provided tags (#{arg}) as something other than a hash.") unless arg.is_a?(Hash) + failure.push("You provided unsuppored tags (#{arg}).") if config['allowed_tags'] && !(arg.keys - config['allowed_tags']).empty? + else + failure.push("Unknown argument #{arg}.") + end + end + + if !failure.empty? + status 400 + result['failure'] = failure + else + jdata.each do |param, arg| + case param + when 'lifetime' + need_token! if Vmpooler::API.settings.config[:auth] + + arg = arg.to_i + + backend.hset("vmpooler__vm__#{params[:hostname]}", param, arg) + when 'tags' + filter_tags(arg) + export_tags(backend, params[:hostname], arg) + end + end + + status 200 + result['ok'] = true + end + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/vm/:hostname/disk/:size/?" do + content_type :json + metrics.increment('http_requests_vm_total.post.vm.disksize') + + need_token! if Vmpooler::API.settings.config[:auth] + + status 404 + result = { 'ok' => false } + + params[:hostname] = hostname_shorten(params[:hostname]) + + if ((params[:size].to_i > 0 )and (backend.exists?("vmpooler__vm__#{params[:hostname]}"))) + result[params[:hostname]] = {} + result[params[:hostname]]['disk'] = "+#{params[:size]}gb" + + backend.sadd('vmpooler__tasks__disk', "#{params[:hostname]}:#{params[:size]}") + + status 202 + result['ok'] = true + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/vm/:hostname/snapshot/?" do + content_type :json + metrics.increment('http_requests_vm_total.post.vm.snapshot') + + need_token! if Vmpooler::API.settings.config[:auth] + + status 404 + result = { 'ok' => false } + + params[:hostname] = hostname_shorten(params[:hostname]) + + if backend.exists?("vmpooler__vm__#{params[:hostname]}") + result[params[:hostname]] = {} + + o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + result[params[:hostname]]['snapshot'] = o[rand(25)] + (0...31).map { o[rand(o.length)] }.join + + backend.sadd('vmpooler__tasks__snapshot', "#{params[:hostname]}:#{result[params[:hostname]]['snapshot']}") + + status 202 + result['ok'] = true + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do + content_type :json + metrics.increment('http_requests_vm_total.post.vm.snapshot') + + need_token! if Vmpooler::API.settings.config[:auth] + + status 404 + result = { 'ok' => false } + + params[:hostname] = hostname_shorten(params[:hostname]) + + unless backend.hget("vmpooler__vm__#{params[:hostname]}", "snapshot:#{params[:snapshot]}").to_i.zero? + backend.sadd('vmpooler__tasks__snapshot-revert', "#{params[:hostname]}:#{params[:snapshot]}") + + status 202 + result['ok'] = true + end + + JSON.pretty_generate(result) + end + + delete "#{api_prefix}/config/poolsize/:pool/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + if pool_exists?(params[:pool]) + result = reset_pool_size(params[:pool]) + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/config/poolsize/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_template_or_size(payload) + if invalid.empty? + result = update_pool_size(payload) + else + invalid.each do |bad_template| + metrics.increment("config.invalid.#{bad_template}") + end + result[:not_configured] = invalid + status 400 + end + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + delete "#{api_prefix}/config/pooltemplate/:pool/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + if pool_exists?(params[:pool]) + result = reset_pool_template(params[:pool]) + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/config/pooltemplate/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_template_or_path(payload) + if invalid.empty? + result = update_pool_template(payload) + else + invalid.each do |bad_template| + metrics.increment("config.invalid.#{bad_template}") + end + result[:bad_templates] = invalid + status 400 + end + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/poolreset/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + begin + payload = JSON.parse(request.body.read) + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = reset_pool(payload) + else + invalid.each do |bad_pool| + metrics.increment("poolreset.invalid.#{bad_pool}") + end + result[:bad_pools] = invalid + status 400 + end + else + metrics.increment('poolreset.invalid.unknown') + status 404 + end + rescue JSON::ParserError + span = OpenTelemetry::Trace.current_span + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + post "#{api_prefix}/config/clonetarget/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_pool(payload) + if invalid.empty? + result = update_clone_target(payload) + else + invalid.each do |bad_template| + metrics.increment("config.invalid.#{bad_template}") + end + result[:bad_templates] = invalid + status 400 + end + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/config/?" do + content_type :json + result = { 'ok' => false } + status 404 + + if pools + sync_pool_sizes + sync_pool_templates + + pool_configuration = [] + pools.each do |pool| + pool['template_ready'] = template_ready?(pool, backend) + pool_configuration << pool + end + + result = { + pool_configuration: pool_configuration, + status: { + ok: true + } + } + + status 200 + end + JSON.pretty_generate(result) + end + + get "#{api_prefix}/full_config/?" do + content_type :json + + result = { + full_config: full_config, + status: { + ok: true + } + } + + status 200 + JSON.pretty_generate(result) + end + end + end +end diff --git a/lib/vmpooler/dns.rb b/lib/vmpooler/dns.rb new file mode 100644 index 0000000..823fe17 --- /dev/null +++ b/lib/vmpooler/dns.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'pathname' + +module Vmpooler + class Dns + # Load one or more VMPooler DNS plugin gems by name + # + # @param names [Array] The list of gem names to load + def self.load_by_name(names) + names = Array(names) + instance = new + names.map { |name| instance.load_from_gems(name) }.flatten + end + + # Returns the plugin class for the specified dns config by name + # + # @param config [Object] The entire VMPooler config object + # @param name [Symbol] The name of the dns config key to get the dns class + # @return [String] The plugin class for the specifid dns config + def self.get_dns_plugin_class_by_name(config, name) + dns_configs = config[:dns_configs].keys + plugin_class = '' + + dns_configs.map do |dns_config_name| + plugin_class = config[:dns_configs][dns_config_name]['dns_class'] if dns_config_name.to_s == name + end + + plugin_class + end + + # Returns the domain for the specified pool + # + # @param config [String] - the full config structure + # @param pool_name [String] - the name of the pool + # @return [String] - domain name for pool, which is set via reference to the dns_configs block + def self.get_domain_for_pool(config, pool_name) + pool = config[:pools].find { |p| p['name'] == pool_name } + pool_dns_config = pool['dns_plugin'] + dns_configs = config[:dns_configs].keys + dns_configs.map do |dns_config_name| + return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == pool_dns_config + end + end + + # Returns the plugin domain for the specified dns config by name + # + # @param config [Object] The entire VMPooler config object + # @param name [Symbol] The name of the dns config key to get the dns domain + # @return [String] The domain for the specifid dns config + def self.get_dns_plugin_domain_by_name(config, name) + dns_configs = config[:dns_configs].keys + dns_configs.map do |dns_config_name| + return config[:dns_configs][dns_config_name]['domain'] if dns_config_name.to_s == name + end + end + + # Returns a list of DNS plugin classes specified in the vmpooler configuration + # + # @param config [Object] The entire VMPooler config object + # @return nil || [Array] A list of DNS plugin classes + def self.get_dns_plugin_config_classes(config) + return nil unless config[:dns_configs] + + dns_configs = config[:dns_configs].keys + dns_plugins = dns_configs.map do |dns_config_name| + if config[:dns_configs][dns_config_name] && config[:dns_configs][dns_config_name]['dns_class'] + config[:dns_configs][dns_config_name]['dns_class'].to_s + else + dns_config_name.to_s + end + end.compact.uniq + + # dynamic-dns is not actually a class, it's just used as a value to denote + # that dynamic dns is used so no loading or record management is needed + dns_plugins.delete('dynamic-dns') + + dns_plugins + end + + # Load a single DNS plugin gem by name + # + # @param name [String] The name of the DNS plugin gem to load + # @return [String] The full require path to the specified gem + def load_from_gems(name = nil) + require_path = "vmpooler/dns/#{name.gsub('-', '/')}" + require require_path + require_path + end + end +end diff --git a/lib/vmpooler/dns/base.rb b/lib/vmpooler/dns/base.rb new file mode 100644 index 0000000..61403ef --- /dev/null +++ b/lib/vmpooler/dns/base.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Vmpooler + class PoolManager + class Dns + class Base + # These defs must be overidden in child classes + + # Helper Methods + # Global Logger object + attr_reader :logger + # Global Metrics object + attr_reader :metrics + # Provider options passed in during initialization + attr_reader :dns_options + + def initialize(config, logger, metrics, redis_connection_pool, name, options) + @config = config + @logger = logger + @metrics = metrics + @redis = redis_connection_pool + @dns_plugin_name = name + + @dns_options = options + + logger.log('s', "[!] Creating dns plugin '#{name}'") + end + + def pool_config(pool_name) + # Get the configuration of a specific pool + @config[:pools].each do |pool| + return pool if pool['name'] == pool_name + end + + nil + end + + # Returns this dns plugin's configuration + # + # @returns [Hashtable] This dns plugins's configuration from the config file. Returns nil if the dns plugin config does not exist + def dns_config + @config[:dns_configs].each do |dns| + # Convert the symbol from the config into a string for comparison + return (dns[1].nil? ? {} : dns[1]) if dns[0].to_s == @dns_plugin_name + end + + nil + end + + def global_config + # This entire VM Pooler config + @config + end + + def name + @dns_plugin_name + end + + def get_ip(vm_name) + @redis.with_metrics do |redis| + redis.hget("vmpooler__vm__#{vm_name}", 'ip') + end + end + + # returns + # Array[String] : Array of pool names this provider services + def provided_pools + @config[:pools].select { |pool| pool['dns_config'] == name }.map { |pool| pool['name'] } + end + + def create_or_replace_record(hostname) + raise("#{self.class.name} does not implement create_or_replace_record #{hostname}") + end + + def delete_record(hostname) + raise("#{self.class.name} does not implement delete_record for #{hostname}") + end + end + end + end +end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 0d908d3..ae1024a 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'vmpooler/dns' require 'vmpooler/providers' require 'vmpooler/util/parsing' require 'spicy-proton' @@ -26,6 +27,9 @@ module Vmpooler # VM Provider objects $providers = Concurrent::Hash.new + # VM DNS objects + $dns_plugins = Concurrent::Hash.new + # Our thread-tracker object $threads = Concurrent::Hash.new @@ -39,6 +43,9 @@ module Vmpooler # load specified providers from config file load_used_providers + + # load specified dns plugins from config file + load_used_dns_plugins end def config @@ -60,7 +67,7 @@ module Vmpooler to_set[k] = pool[k] end to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias') - to_set['domain'] = Parsing.get_domain_for_pool(config, pool['name']) + to_set['domain'] = Vmpooler::Dns.get_domain_for_pool(config, pool['name']) redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty? end previously_configured_pools.each do |pool| @@ -106,6 +113,11 @@ module Vmpooler def remove_nonexistent_vm(vm, pool, redis) redis.srem("vmpooler__pending__#{pool}", vm) + dns_plugin = get_dns_plugin_class_for_pool(pool) + dns_plugin_class_name = get_dns_plugin_class_name_for_pool(pool) + domain = get_dns_plugin_domain_for_pool(pool) + fqdn = "#{vm}.#{domain}" + dns_plugin.delete_record(fqdn) unless dns_plugin_class_name == 'dynamic-dns' $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") end @@ -323,10 +335,10 @@ module Vmpooler end # Clone a VM - def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil) + def clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil) Thread.new do begin - _clone_vm(pool_name, provider, request_id, pool_alias) + _clone_vm(pool_name, provider, dns_plugin, request_id, pool_alias) rescue StandardError => e if request_id $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}") @@ -368,12 +380,8 @@ module Vmpooler max_hostname_retries = 3 while hostname_retries < max_hostname_retries hostname, hostname_available = generate_and_check_hostname - domain = Parsing.get_domain_for_pool(config, pool_name) - if domain - fqdn = "#{hostname}.#{domain}" - else - fqdn = hostname - end + domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name) + fqdn = "#{hostname}.#{domain}" # skip dns check if the provider is set to skip_dns_check_before_creating_vm provider = get_provider_for_pool(pool_name) @@ -414,9 +422,9 @@ module Vmpooler [dns_ip, false] end - def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil) + def _clone_vm(pool_name, provider, dns_plugin, request_id = nil, pool_alias = nil) new_vmname = find_unique_hostname(pool_name) - pool_domain = Parsing.get_domain_for_pool(config, pool_name) + pool_domain = Vmpooler::Dns.get_domain_for_pool(config, pool_name) mutex = vm_mutex(new_vmname) mutex.synchronize do @redis.with_metrics do |redis| @@ -426,7 +434,7 @@ module Vmpooler redis.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now) redis.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool. redis.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name) - redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) if pool_domain + redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) redis.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id redis.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias redis.exec @@ -437,16 +445,25 @@ module Vmpooler start = Time.now provider.create_vm(pool_name, new_vmname) finish = format('%