diff --git a/Gemfile b/Gemfile index d301aba..e97d7cb 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,7 @@ gem 'statsd-ruby', '~> 1.4.0', :require => 'statsd' gem 'connection_pool', '~> 2.2' gem 'nokogiri', '~> 1.10' gem 'spicy-proton', '~> 2.1' +gem 'concurrent-ruby', '~> 1.1' group :development do gem 'pry' diff --git a/README.md b/README.md index 1a77553..d322131 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ vmpooler provides configurable 'pools' of instantly-available (running) virtual ## Usage -At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. Dynamic cloning of VM templates initially worked fine for this, but added several seconds to each test run and was unable to account for failed clone tasks. By pushing these operations to a backend service, we were able to both speed up tests and eliminate test failures due to underlying infrastructure failures. - +At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of disposable VMs every day. Vmpooler manages the lifecycle of these VMs from request through deletion, with options available to pool ready instances, and provision on demand. ## Installation @@ -85,7 +84,7 @@ docker run -it vmpooler manager ### docker-compose -A docker-compose file is provided to support running vmpooler easily via docker-compose. +A docker-compose file is provided to support running vmpooler easily via docker-compose. This is useful for development because your local code is used to build the gem used in the docker-compose environment. ``` docker-compose -f docker/docker-compose.yml up @@ -113,7 +112,6 @@ A dashboard is provided to offer real-time statistics and historical graphs. It ## Command-line Utility -- The [vmpooler_client.py](https://github.com/puppetlabs/vmpooler-client) CLI utility provides easy access to the vmpooler service. The tool is cross-platform and written in Python. - [vmfloaty](https://github.com/briancain/vmfloaty) is a ruby based CLI tool and scripting library written in ruby. ## Vagrant plugin diff --git a/bin/vmpooler b/bin/vmpooler index 2f0f98f..3730c1f 100755 --- a/bin/vmpooler +++ b/bin/vmpooler @@ -7,6 +7,8 @@ config = Vmpooler.config redis_host = config[:redis]['server'] redis_port = config[:redis]['port'] redis_password = config[:redis]['password'] +redis_connection_pool_size = config[:redis]['connection_pool_size'] +redis_connection_pool_timeout = config[:redis]['connection_pool_timeout'] logger_file = config[:config]['logfile'] metrics = Vmpooler.new_metrics(config) @@ -36,7 +38,7 @@ if torun.include? 'manager' Vmpooler::PoolManager.new( config, Vmpooler.new_logger(logger_file), - Vmpooler.new_redis(redis_host, redis_port, redis_password), + Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics), metrics ).execute! end diff --git a/docker/Dockerfile b/docker/Dockerfile index eb2ae6b..e879f1f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,9 +8,9 @@ # RUN: # docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler -FROM jruby:9.2.9-jdk +FROM jruby:9.2-jdk -ARG vmpooler_version=0.5.0 +ARG vmpooler_version=0.11.3 COPY docker/docker-entrypoint.sh /usr/local/bin/ diff --git a/docker/Dockerfile_local b/docker/Dockerfile_local index 0f6ed55..7a47516 100644 --- a/docker/Dockerfile_local +++ b/docker/Dockerfile_local @@ -8,7 +8,7 @@ # RUN: # docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler -FROM jruby:9.2.9-jdk +FROM jruby:9.2-jdk COPY docker/docker-entrypoint.sh /usr/local/bin/ COPY ./ ./ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4ef776c..dd00a99 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,7 +17,7 @@ services: - VMPOOLER_DEBUG=true # for use of dummy auth - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - REDIS_SERVER=redislocal - - LOGFILE=/dev/stdout + - LOGFILE=/dev/null image: vmpooler-local depends_on: - redislocal diff --git a/docs/API.md b/docs/API.md index d30b329..b4675ed 100644 --- a/docs/API.md +++ b/docs/API.md @@ -6,6 +6,7 @@ 5. [VM snapshots](#vmsnapshots) 6. [Status and metrics](#statusmetrics) 7. [Pool configuration](#poolconfig) +8. [Ondemand VM provisioning](#ondemandvm) ### API @@ -799,3 +800,91 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' -- "ok": true } ``` + +#### Ondemand VM operations + +Ondemand VM operations offer a user an option to directly request instances to be allocated for use. This can be very useful when supporting a wide range of images because idle instances can be eliminated. + +##### POST /ondemandvm + +All instance types requested must match a pool name or alias in the running application configuration, or a 404 code will be returned + +When a provisioning request is accepted the API will return an indication that the request is successful. You may then poll /ondemandvm to monitor request status. + +An authentication token is required in order to request instances on demand when authentication is configured. + +Responses: +* 201 - Provisioning request accepted +* 400 - Payload contains invalid JSON and cannot be parsed +* 403 - Request exceeds the configured per pool maximum +* 404 - A pool was requested, which is not available in the running configuration, or an unknown error occurred. +* 409 - A request of the matching ID has already been created +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"4"}' --url https://vmpooler.example.com/api/v1/ondemandvm +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a" +} +``` + +##### GET /ondemandvm + +Get the status of an ondemandvm request that has already been posted. + +When the request is ready the ready status will change to 'true'. + +The number of instances pending vs ready will be reflected in the API response. + +Responses: +* 200 - The API request was successful and the status is ok +* 202 - The request is not ready yet +* 404 - The request can not be found, or an unknown error occurred +``` +$ curl https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", + "ready": false, + "debian-7-i386": { + "ready": "3", + "pending": "1" + } +} +``` +```json +{ + "ok": true, + "request_id": "e3ff6271-d201-4f31-a315-d17f4e15863a", + "ready": true, + "debian-7-i386": { + "hostname": [ + "vm1", + "vm2", + "vm3", + "vm4" + ] + } +} +``` + +##### DELETE /ondemandvm + +Delete a ondemand request + +Deleting a ondemand request will delete any instances created for the request and mark the backend data for expiration in two weeks. Any subsequent attempts to retrieve request data will indicate it has been deleted. + +Responses: +* 200 - The API request was sucessful. A message will indicate if the request has already been deleted. +* 404 - The request can not be found, or an unknown error occurred. +``` +$ curl -X DELETE https://vmpooler.example.com/api/v1/ondemandvm/e3ff6271-d201-4f31-a315-d17f4e15863a +``` +```json +{ + "ok": true +} +``` diff --git a/docs/configuration.md b/docs/configuration.md index 26d485b..4846b94 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,6 +74,16 @@ The prefix to use while storing Graphite data. The TCP port to communicate with the graphite server. (optional; default: 2003) +### MAX\_ONDEMAND\_INSTANCES\_PER\_REQUEST + +The maximum number of instances any individual ondemand request may contain per pool. +(default: 10) + +### ONDEMAND\_REQUEST\_TTL + +The amount of time (in minutes) to give for a ondemand request to be fulfilled before considering it to have failed. +(default: 5) + ## Manager options ### TASK\_LIMIT @@ -123,6 +133,11 @@ The target cluster VMs are cloned into (host with least VMs chosen) How long (in minutes) before marking a clone as 'failed' and retrying. (optional; default: 15) +### READY\_TTL + +How long (in minutes) a ready VM should stay in the ready queue. +(default: 60) + ### MAX\_TRIES Set the max number of times a connection should retry in VM providers. This optional setting allows a user to dial in retry limits to suit your environment. @@ -130,7 +145,7 @@ Set the max number of times a connection should retry in VM providers. This opti ### RETRY\_FACTOR -When retrying, each attempt sleeps for the try count * retry_factor. +When retrying, each attempt sleeps for the try count * retry\_factor. Increase this number to lengthen the delay between retry attempts. This is particularly useful for instances with a large number of pools to prevent a thundering herd when retrying connections. @@ -183,6 +198,21 @@ The argument can accept a full path to a file, or multiple files comma separated Expects a string value (optional) +### ONDEMAND\_CLONE\_LIMIT + +Maximum number of simultaneous clones to perform for ondemand provisioning requests. +(default: 10) + +### REDIS\_CONNECTION\_POOL\_SIZE + +Maximum number of connections to utilize for the redis connection pool. +(default: 10) + +### REDIS\_CONNECTION\_POOL\_TIMEOUT + +How long a task should wait (in seconds) for a redis connection when all connections are in use. +(default: 5) + ## API options ### AUTH\_PROVIDER @@ -221,3 +251,8 @@ The name of your deployment. Enable experimental API capabilities such as changing pool template and size without application restart Expects a boolean value (optional; default: false) + +### MAX\_LIFETIME\_UPPER\_LIMIT + +Specify a maximum lifetime that a VM may be extended to in hours. +(optional) diff --git a/docs/dev-setup.md b/docs/dev-setup.md index 3a128b0..5999f09 100644 --- a/docs/dev-setup.md +++ b/docs/dev-setup.md @@ -1,13 +1,15 @@ # Setting up a vmpooler development environment -## Requirements +## Docker is the preferred development environment + +The docker compose file is the easiest way to get vmpooler running with any local code changes. The docker compose file expects to find a vmpooler.yaml configuration file in the root vmpooler directory. The file is mapped into the running container for the vmpooler application. This file primarily contains the pools configuration. Nearly all other configuration can be supplied with environment variables. + +## Requirements for local installation directly on your system (not recommended) * Supported on OSX, Windows and Linux * Ruby or JRuby - Note - Ruby 1.x support will be removed so it is best to use more modern ruby versions - Note - It is recommended to user Bundler instead of installing gems into the system repository * A local Redis server diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 5ae00f1..c1fe206 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module Vmpooler + require 'concurrent' require 'date' require 'json' require 'net/ldap' @@ -58,9 +59,14 @@ module Vmpooler # Set some configuration defaults parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10 + parsed_config[:config]['ondemand_clone_limit'] = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10 + parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10 parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT'] parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1 parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24 + parsed_config[:config]['max_lifetime_upper_limit'] = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit'] + parsed_config[:config]['ready_ttl'] = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60 + parsed_config[:config]['ondemand_request_ttl'] = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5 parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || '' parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE'] @@ -84,6 +90,8 @@ module Vmpooler parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT'] parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD'] parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168 + parsed_config[:redis]['connection_pool_size'] = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10 + parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5 parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER'] parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER'] @@ -117,6 +125,7 @@ module Vmpooler parsed_config[:pools].each do |pool| parsed_config[:pool_names] << pool['name'] + pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl'] if pool['alias'] if pool['alias'].is_a?(Array) pool['alias'].each do |pool_alias| @@ -154,6 +163,19 @@ module Vmpooler pools end + def self.redis_connection_pool(host, port, password, size, timeout, metrics) + Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + metric_prefix: 'redis_connection_pool', + size: size, + timeout: timeout + ) do + connection = Concurrent::Hash.new + redis = new_redis(host, port, password) + connection['connection'] = redis + end + end + def self.new_redis(host = 'localhost', port = nil, password = nil) Redis.new(host: host, port: port, password: password) end diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 0b98143..f696e52 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -238,7 +238,7 @@ module Vmpooler queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend) - queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i queue[:booting] = 0 if queue[:booting] < 0 queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index c588a4a..966e6a7 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -42,6 +42,68 @@ module Vmpooler Vmpooler::API.settings.checkoutlock end + def get_template_aliases(template) + result = [] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + result += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + end + result + end + + def get_pool_weights(template_backends) + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + weighted_pools + end + + def count_selection(selection) + result = {} + selection.uniq.each do |poolname| + result[poolname] = selection.count(poolname) + end + result + end + + def evaluate_template_aliases(template, count) + template_backends = [] + template_backends << template if backend.sismember('vmpooler__pools', template) + selection = [] + aliases = get_template_aliases(template) + if aliases + template_backends += aliases + weighted_pools = get_pool_weights(template_backends) + + pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count + count.to_i.times do + if pickup + selection << pickup.pick + else + selection << template_backends.sample + end + end + else + count.to_i.times do + selection << template + end + end + + count_selection(selection) + end + def fetch_single_vm(template) template_backends = [template] aliases = Vmpooler::API.settings.config[:alias] @@ -245,11 +307,9 @@ module Vmpooler pool_index = pool_index(pools) template_configs = backend.hgetall('vmpooler__config__template') template_configs&.each do |poolname, template| - if pool_index.include? poolname - unless pools[pool_index[poolname]]['template'] == template - pools[pool_index[poolname]]['template'] = template - end - end + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['template'] = template end end @@ -257,11 +317,9 @@ module Vmpooler pool_index = pool_index(pools) poolsize_configs = backend.hgetall('vmpooler__config__poolsize') poolsize_configs&.each do |poolname, size| - if pool_index.include? poolname - unless pools[pool_index[poolname]]['size'] == size.to_i - pools[pool_index[poolname]]['size'] == size.to_i - end - end + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['size'] = size.to_i end end @@ -269,14 +327,69 @@ module Vmpooler pool_index = pool_index(pools) clone_target_configs = backend.hgetall('vmpooler__config__clone_target') clone_target_configs&.each do |poolname, clone_target| - if pool_index.include? poolname - unless pools[pool_index[poolname]]['clone_target'] == clone_target - pools[pool_index[poolname]]['clone_target'] == clone_target - end - end + next unless pool_index.include? poolname + + pools[pool_index[poolname]]['clone_target'] = clone_target end end + def too_many_requested?(payload) + payload&.each do |_poolname, count| + next unless count.to_i > config['max_ondemand_instances_per_request'] + + return true + end + false + end + + def generate_ondemand_request(payload) + result = { 'ok': false } + + requested_instances = payload.reject { |k, _v| k == 'request_id' } + if too_many_requested?(requested_instances) + result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" + status 403 + return result + end + + score = Time.now.to_i + request_id = payload['request_id'] + request_id ||= generate_request_id + result['request_id'] = request_id + + if backend.exists("vmpooler__odrequest__#{request_id}") + result['message'] = "request_id '#{request_id}' has already been created" + status 409 + return result + end + + status 201 + + platforms_with_aliases = [] + requested_instances.each do |poolname, count| + selection = evaluate_template_aliases(poolname, count) + selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } + end + platforms_string = platforms_with_aliases.join(',') + + return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) + + backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) + if Vmpooler::API.settings.config[:auth] and has_token? + backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) + backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', + backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')) + end + + result['domain'] = config['domain'] if config['domain'] + result[:ok] = true + result + end + + def generate_request_id + SecureRandom.uuid + end + get '/' do sync_pool_sizes redirect to('/dashboard/') @@ -395,7 +508,7 @@ module Vmpooler end # for backwards compatibility, include separate "empty" stats in "status" block - if ready == 0 + if ready == 0 && max != 0 result[:status][:empty] ||= [] result[:status][:empty].push(pool['name']) @@ -689,6 +802,61 @@ module Vmpooler JSON.pretty_generate(result) end + post "#{api_prefix}/ondemandvm/?" do + content_type :json + + need_token! if Vmpooler::API.settings.config[:auth] + + result = { 'ok' => false } + + begin + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment('ondemandrequest.invalid.' + bad_template) + end + status 404 + end + else + metrics.increment('ondemandrequest.invalid.unknown') + status 404 + end + rescue JSON::ParserError + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } + end + + JSON.pretty_generate(result) + end + + get "#{api_prefix}/ondemandvm/:requestid/?" do + content_type :json + + status 404 + result = check_ondemand_request(params[:requestid]) + + JSON.pretty_generate(result) + end + + delete "#{api_prefix}/ondemandvm/:requestid/?" do + content_type :json + need_token! if Vmpooler::API.settings.config[:auth] + + status 404 + result = delete_ondemand_request(params[:requestid]) + + JSON.pretty_generate(result) + end + post "#{api_prefix}/vm/?" do content_type :json result = { 'ok' => false } @@ -764,6 +932,78 @@ module Vmpooler invalid end + def check_ondemand_request(request_id) + result = { 'ok' => false } + request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") + if request_hash.empty? + result['message'] = "no request found for request_id '#{request_id}'" + return result + end + + result['request_id'] = request_id + result['ready'] = false + result['ok'] = true + status 202 + + if request_hash['status'] == 'ready' + result['ready'] = true + platform_parts = request_hash['requested'].split(',') + platform_parts.each do |platform| + pool_alias, pool, _count = platform.split(':') + instances = backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}") + result[pool_alias] = { 'hostname': instances } + end + result['domain'] = config['domain'] if config['domain'] + status 200 + elsif request_hash['status'] == 'failed' + result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" + status 200 + elsif request_hash['status'] == 'deleted' + result['message'] = 'The request has been deleted' + status 200 + else + platform_parts = request_hash['requested'].split(',') + platform_parts.each do |platform| + pool_alias, pool, count = platform.split(':') + instance_count = backend.scard("vmpooler__#{request_id}__#{pool_alias}__#{pool}") + result[pool_alias] = { + 'ready': instance_count.to_s, + 'pending': (count.to_i - instance_count.to_i).to_s + } + end + end + + result + end + + def delete_ondemand_request(request_id) + result = { 'ok' => false } + + platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') + unless platforms + result['message'] = "no request found for request_id '#{request_id}'" + return result + end + + if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' + result['message'] = 'the request has already been deleted' + else + backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') + + platforms.split(',').each do |platform| + pool_alias, pool, _count = platform.split(':') + backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")&.each do |vm| + backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) + end + backend.del("vmpooler__#{request_id}__#{pool_alias}__#{pool}") + end + backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) + end + status 200 + result['ok'] = true + result + end + post "#{api_prefix}/vm/:template/?" do content_type :json result = { 'ok' => false } @@ -923,6 +1163,7 @@ module Vmpooler unless arg.to_i > 0 failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.") end + when 'tags' unless arg.is_a?(Hash) failure.push("You provided tags (#{arg}) as something other than a hash.") @@ -1047,7 +1288,7 @@ module Vmpooler invalid.each do |bad_template| metrics.increment("config.invalid.#{bad_template}") end - result[:bad_templates] = invalid + result[:not_configured] = invalid status 400 end else diff --git a/lib/vmpooler/generic_connection_pool.rb b/lib/vmpooler/generic_connection_pool.rb index 6ae916b..9e9fc0c 100644 --- a/lib/vmpooler/generic_connection_pool.rb +++ b/lib/vmpooler/generic_connection_pool.rb @@ -15,35 +15,17 @@ module Vmpooler @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == '' end - if Thread.respond_to?(:handle_interrupt) - # MRI - def with_metrics(options = {}) - Thread.handle_interrupt(Exception => :never) do - start = Time.now - conn = checkout(options) - timespan_ms = ((Time.now - start) * 1000).to_i - @metrics&.gauge(@metric_prefix + '.available', @available.length) - @metrics&.timing(@metric_prefix + '.waited', timespan_ms) - begin - Thread.handle_interrupt(Exception => :immediate) do - yield conn - end - ensure - checkin - @metrics&.gauge(@metric_prefix + '.available', @available.length) - end - end - end - else - # jruby 1.7.x - def with_metrics(options = {}) + def with_metrics(options = {}) + Thread.handle_interrupt(Exception => :never) do start = Time.now conn = checkout(options) timespan_ms = ((Time.now - start) * 1000).to_i @metrics&.gauge(@metric_prefix + '.available', @available.length) @metrics&.timing(@metric_prefix + '.waited', timespan_ms) begin - yield conn + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end ensure checkin @metrics&.gauge(@metric_prefix + '.available', @available.length) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 65847e8..f386b1f 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -9,7 +9,7 @@ module Vmpooler CHECK_LOOP_DELAY_MAX_DEFAULT = 60 CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0 - def initialize(config, logger, redis, metrics) + def initialize(config, logger, redis_connection_pool, metrics) $config = config # Load logger library @@ -18,19 +18,19 @@ module Vmpooler # metrics logging handle $metrics = metrics - # Connect to Redis - $redis = redis + # Redis connection pool + @redis = redis_connection_pool # VM Provider objects - $providers = {} + $providers = Concurrent::Hash.new # Our thread-tracker object - $threads = {} + $threads = Concurrent::Hash.new # Pool mutex - @reconfigure_pool = {} + @reconfigure_pool = Concurrent::Hash.new - @vm_mutex = {} + @vm_mutex = Concurrent::Hash.new # Name generator for generating host names @name_generator = Spicy::Proton.new @@ -45,24 +45,26 @@ module Vmpooler # Place pool configuration in redis so an API instance can discover running pool configuration def load_pools_to_redis - previously_configured_pools = $redis.smembers('vmpooler__pools') - currently_configured_pools = [] - config[:pools].each do |pool| - currently_configured_pools << pool['name'] - $redis.sadd('vmpooler__pools', pool['name']) - pool_keys = pool.keys - pool_keys.delete('alias') - to_set = {} - pool_keys.each do |k| - to_set[k] = pool[k] + @redis.with_metrics do |redis| + previously_configured_pools = redis.smembers('vmpooler__pools') + currently_configured_pools = [] + config[:pools].each do |pool| + currently_configured_pools << pool['name'] + redis.sadd('vmpooler__pools', pool['name']) + pool_keys = pool.keys + pool_keys.delete('alias') + to_set = {} + pool_keys.each do |k| + to_set[k] = pool[k] + end + to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias') + redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty? end - to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias') - $redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty? - end - previously_configured_pools.each do |pool| - unless currently_configured_pools.include? pool - $redis.srem('vmpooler__pools', pool) - $redis.del("vmpooler__pool__#{pool}") + previously_configured_pools.each do |pool| + unless currently_configured_pools.include? pool + redis.srem('vmpooler__pools', pool) + redis.del("vmpooler__pool__#{pool}") + end end end nil @@ -75,7 +77,9 @@ module Vmpooler _check_pending_vm(vm, pool, timeout, provider) rescue StandardError => e $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}") - fail_pending_vm(vm, pool, timeout) + @redis.with_metrics do |redis| + fail_pending_vm(vm, pool, timeout, redis) + end raise end end @@ -86,31 +90,38 @@ module Vmpooler return if mutex.locked? mutex.synchronize do - if provider.vm_ready?(pool, vm) - move_pending_vm_to_ready(vm, pool) - else - fail_pending_vm(vm, pool, timeout) + @redis.with_metrics do |redis| + request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id') + if provider.vm_ready?(pool, vm) + move_pending_vm_to_ready(vm, pool, redis, request_id) + else + fail_pending_vm(vm, pool, timeout, redis) + end end end end - def remove_nonexistent_vm(vm, pool) - $redis.srem("vmpooler__pending__#{pool}", vm) + def remove_nonexistent_vm(vm, pool, redis) + redis.srem("vmpooler__pending__#{pool}", vm) $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") end - def fail_pending_vm(vm, pool, timeout, exists = true) - clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone') - return true unless clone_stamp + def fail_pending_vm(vm, pool, timeout, redis, exists = true) + clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone') time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60 if time_since_clone > timeout if exists - $redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm) + request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id') + pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id + redis.multi + redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm) + redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id + redis.exec $metrics.increment("errors.markedasfailed.#{pool}") $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") else - remove_nonexistent_vm(vm, pool) + remove_nonexistent_vm(vm, pool, redis) end end true @@ -119,28 +130,53 @@ module Vmpooler false end - def move_pending_vm_to_ready(vm, pool) - clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone') - finish = format('%