mirror of
https://github.com/puppetlabs/vmpooler.git
synced 2026-01-26 18:08:42 -05:00
(POOLER-158) Add capability to provision VMs on demand
This change adds a capability to vmpooler to provision instances on demand. Without this change vmpooler only supports retrieving machines from pre-provisioned pools. Additionally, this change refactors redis interactions to reduce round trips to redis. Specifically, multi and pipelined redis commands are added where possible to reduce the number of times we are calling redis. To support the redis refactor the redis interaction has changed to leveraging a connection pool. In addition to offering multiple connections for pool manager to use, the redis interactions in pool manager are now thread safe. Ready TTL is now a global parameter that can be set as a default for all pools. A default of 0 has been removed, because this is an unreasonable default behavior, which would leave a provisioned instance in the pool indefinitely. Pool empty messages have been removed when the pool size is set to 0. Without this change, when a pool was set to a size of 0 the API and pool manager would both show that a pool is empty.
This commit is contained in:
kirby@puppetlabs.com
parent
1f6f08d172
commit
811fd8b60f
34 changed files with 3326 additions and 1098 deletions
|
|
@ -238,7 +238,7 @@ module Vmpooler
|
|||
queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
|
||||
queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
|
||||
|
||||
queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
|
||||
queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
|
||||
queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
|
||||
queue[:booting] = 0 if queue[:booting] < 0
|
||||
queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
|
||||
|
|
|
|||
|
|
@ -42,6 +42,68 @@ module Vmpooler
|
|||
Vmpooler::API.settings.checkoutlock
|
||||
end
|
||||
|
||||
def get_template_aliases(template)
|
||||
result = []
|
||||
aliases = Vmpooler::API.settings.config[:alias]
|
||||
if aliases
|
||||
result += aliases[template] if aliases[template].is_a?(Array)
|
||||
template_backends << aliases[template] if aliases[template].is_a?(String)
|
||||
end
|
||||
result
|
||||
end
|
||||
|
||||
def get_pool_weights(template_backends)
|
||||
pool_index = pool_index(pools)
|
||||
weighted_pools = {}
|
||||
template_backends.each do |t|
|
||||
next unless pool_index.key? t
|
||||
|
||||
index = pool_index[t]
|
||||
clone_target = pools[index]['clone_target'] || config['clone_target']
|
||||
next unless config.key?('backend_weight')
|
||||
|
||||
weight = config['backend_weight'][clone_target]
|
||||
if weight
|
||||
weighted_pools[t] = weight
|
||||
end
|
||||
end
|
||||
weighted_pools
|
||||
end
|
||||
|
||||
def count_selection(selection)
|
||||
result = {}
|
||||
selection.uniq.each do |poolname|
|
||||
result[poolname] = selection.count(poolname)
|
||||
end
|
||||
result
|
||||
end
|
||||
|
||||
def evaluate_template_aliases(template, count)
|
||||
template_backends = []
|
||||
template_backends << template if backend.sismember('vmpooler__pools', template)
|
||||
selection = []
|
||||
aliases = get_template_aliases(template)
|
||||
if aliases
|
||||
template_backends += aliases
|
||||
weighted_pools = get_pool_weights(template_backends)
|
||||
|
||||
pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count
|
||||
count.to_i.times do
|
||||
if pickup
|
||||
selection << pickup.pick
|
||||
else
|
||||
selection << template_backends.sample
|
||||
end
|
||||
end
|
||||
else
|
||||
count.to_i.times do
|
||||
selection << template
|
||||
end
|
||||
end
|
||||
|
||||
count_selection(selection)
|
||||
end
|
||||
|
||||
def fetch_single_vm(template)
|
||||
template_backends = [template]
|
||||
aliases = Vmpooler::API.settings.config[:alias]
|
||||
|
|
@ -245,11 +307,9 @@ module Vmpooler
|
|||
pool_index = pool_index(pools)
|
||||
template_configs = backend.hgetall('vmpooler__config__template')
|
||||
template_configs&.each do |poolname, template|
|
||||
if pool_index.include? poolname
|
||||
unless pools[pool_index[poolname]]['template'] == template
|
||||
pools[pool_index[poolname]]['template'] = template
|
||||
end
|
||||
end
|
||||
next unless pool_index.include? poolname
|
||||
|
||||
pools[pool_index[poolname]]['template'] = template
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -257,11 +317,9 @@ module Vmpooler
|
|||
pool_index = pool_index(pools)
|
||||
poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
|
||||
poolsize_configs&.each do |poolname, size|
|
||||
if pool_index.include? poolname
|
||||
unless pools[pool_index[poolname]]['size'] == size.to_i
|
||||
pools[pool_index[poolname]]['size'] == size.to_i
|
||||
end
|
||||
end
|
||||
next unless pool_index.include? poolname
|
||||
|
||||
pools[pool_index[poolname]]['size'] = size.to_i
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -269,14 +327,69 @@ module Vmpooler
|
|||
pool_index = pool_index(pools)
|
||||
clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
|
||||
clone_target_configs&.each do |poolname, clone_target|
|
||||
if pool_index.include? poolname
|
||||
unless pools[pool_index[poolname]]['clone_target'] == clone_target
|
||||
pools[pool_index[poolname]]['clone_target'] == clone_target
|
||||
end
|
||||
end
|
||||
next unless pool_index.include? poolname
|
||||
|
||||
pools[pool_index[poolname]]['clone_target'] = clone_target
|
||||
end
|
||||
end
|
||||
|
||||
def too_many_requested?(payload)
|
||||
payload&.each do |_poolname, count|
|
||||
next unless count.to_i > config['max_ondemand_instances_per_request']
|
||||
|
||||
return true
|
||||
end
|
||||
false
|
||||
end
|
||||
|
||||
def generate_ondemand_request(payload)
|
||||
result = { 'ok': false }
|
||||
|
||||
requested_instances = payload.reject { |k, _v| k == 'request_id' }
|
||||
if too_many_requested?(requested_instances)
|
||||
result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
|
||||
status 403
|
||||
return result
|
||||
end
|
||||
|
||||
score = Time.now.to_i
|
||||
request_id = payload['request_id']
|
||||
request_id ||= generate_request_id
|
||||
result['request_id'] = request_id
|
||||
|
||||
if backend.exists("vmpooler__odrequest__#{request_id}")
|
||||
result['message'] = "request_id '#{request_id}' has already been created"
|
||||
status 409
|
||||
return result
|
||||
end
|
||||
|
||||
status 201
|
||||
|
||||
platforms_with_aliases = []
|
||||
requested_instances.each do |poolname, count|
|
||||
selection = evaluate_template_aliases(poolname, count)
|
||||
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
|
||||
end
|
||||
platforms_string = platforms_with_aliases.join(',')
|
||||
|
||||
return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
|
||||
|
||||
backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
|
||||
if Vmpooler::API.settings.config[:auth] and has_token?
|
||||
backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
|
||||
backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
|
||||
backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user'))
|
||||
end
|
||||
|
||||
result['domain'] = config['domain'] if config['domain']
|
||||
result[:ok] = true
|
||||
result
|
||||
end
|
||||
|
||||
def generate_request_id
|
||||
SecureRandom.uuid
|
||||
end
|
||||
|
||||
get '/' do
|
||||
sync_pool_sizes
|
||||
redirect to('/dashboard/')
|
||||
|
|
@ -395,7 +508,7 @@ module Vmpooler
|
|||
end
|
||||
|
||||
# for backwards compatibility, include separate "empty" stats in "status" block
|
||||
if ready == 0
|
||||
if ready == 0 && max != 0
|
||||
result[:status][:empty] ||= []
|
||||
result[:status][:empty].push(pool['name'])
|
||||
|
||||
|
|
@ -689,6 +802,88 @@ module Vmpooler
|
|||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
post "#{api_prefix}/ondemandvm/?" do
|
||||
content_type :json
|
||||
|
||||
need_token! if Vmpooler::API.settings.config[:auth]
|
||||
|
||||
result = { 'ok' => false }
|
||||
|
||||
begin
|
||||
payload = JSON.parse(request.body.read)
|
||||
|
||||
if payload
|
||||
invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
|
||||
if invalid.empty?
|
||||
result = generate_ondemand_request(payload)
|
||||
else
|
||||
result[:bad_templates] = invalid
|
||||
invalid.each do |bad_template|
|
||||
metrics.increment('ondemandrequest.invalid.' + bad_template)
|
||||
end
|
||||
status 404
|
||||
end
|
||||
else
|
||||
metrics.increment('ondemandrequest.invalid.unknown')
|
||||
status 404
|
||||
end
|
||||
rescue JSON::ParserError
|
||||
status 400
|
||||
result = {
|
||||
'ok' => false,
|
||||
'message' => 'JSON payload could not be parsed'
|
||||
}
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
post "#{api_prefix}/ondemandvm/:template/?" do
|
||||
content_type :json
|
||||
result = { 'ok' => false }
|
||||
|
||||
need_token! if Vmpooler::API.settings.config[:auth]
|
||||
|
||||
payload = extract_templates_from_query_params(params[:template])
|
||||
|
||||
if payload
|
||||
invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
|
||||
if invalid.empty?
|
||||
result = generate_ondemand_request(payload)
|
||||
else
|
||||
result[:bad_templates] = invalid
|
||||
invalid.each do |bad_template|
|
||||
metrics.increment('ondemandrequest.invalid.' + bad_template)
|
||||
end
|
||||
status 404
|
||||
end
|
||||
else
|
||||
metrics.increment('ondemandrequest.invalid.unknown')
|
||||
status 404
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get "#{api_prefix}/ondemandvm/:requestid/?" do
|
||||
content_type :json
|
||||
|
||||
status 404
|
||||
result = check_ondemand_request(params[:requestid])
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
delete "#{api_prefix}/ondemandvm/:requestid/?" do
|
||||
content_type :json
|
||||
need_token! if Vmpooler::API.settings.config[:auth]
|
||||
|
||||
status 404
|
||||
result = delete_ondemand_request(params[:requestid])
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
post "#{api_prefix}/vm/?" do
|
||||
content_type :json
|
||||
result = { 'ok' => false }
|
||||
|
|
@ -764,6 +959,78 @@ module Vmpooler
|
|||
invalid
|
||||
end
|
||||
|
||||
def check_ondemand_request(request_id)
|
||||
result = { 'ok' => false }
|
||||
request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
|
||||
if request_hash.empty?
|
||||
result['message'] = "no request found for request_id '#{request_id}'"
|
||||
return result
|
||||
end
|
||||
|
||||
result['request_id'] = request_id
|
||||
result['ready'] = false
|
||||
result['ok'] = true
|
||||
status 202
|
||||
|
||||
if request_hash['status'] == 'ready'
|
||||
result['ready'] = true
|
||||
platform_parts = request_hash['requested'].split(',')
|
||||
platform_parts.each do |platform|
|
||||
pool_alias, pool, _count = platform.split(':')
|
||||
instances = backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
|
||||
result[pool_alias] = { 'hostname': instances }
|
||||
end
|
||||
result['domain'] = config['domain'] if config['domain']
|
||||
status 200
|
||||
elsif request_hash['status'] == 'failed'
|
||||
result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
|
||||
status 200
|
||||
elsif request_hash['status'] == 'deleted'
|
||||
result['message'] = 'The request has been deleted'
|
||||
status 200
|
||||
else
|
||||
platform_parts = request_hash['requested'].split(',')
|
||||
platform_parts.each do |platform|
|
||||
pool_alias, pool, count = platform.split(':')
|
||||
instance_count = backend.scard("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
|
||||
result[pool_alias] = {
|
||||
'ready': instance_count.to_s,
|
||||
'pending': (count.to_i - instance_count.to_i).to_s
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
result
|
||||
end
|
||||
|
||||
def delete_ondemand_request(request_id)
|
||||
result = { 'ok' => false }
|
||||
|
||||
platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
|
||||
unless platforms
|
||||
result['message'] = "no request found for request_id '#{request_id}'"
|
||||
return result
|
||||
end
|
||||
|
||||
if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
|
||||
result['message'] = 'the request has already been deleted'
|
||||
else
|
||||
backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
|
||||
|
||||
platforms.split(',').each do |platform|
|
||||
pool_alias, pool, _count = platform.split(':')
|
||||
backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")&.each do |vm|
|
||||
backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
|
||||
end
|
||||
backend.del("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
|
||||
end
|
||||
backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
|
||||
end
|
||||
status 200
|
||||
result['ok'] = true
|
||||
result
|
||||
end
|
||||
|
||||
post "#{api_prefix}/vm/:template/?" do
|
||||
content_type :json
|
||||
result = { 'ok' => false }
|
||||
|
|
@ -923,6 +1190,7 @@ module Vmpooler
|
|||
unless arg.to_i > 0
|
||||
failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
|
||||
end
|
||||
|
||||
when 'tags'
|
||||
unless arg.is_a?(Hash)
|
||||
failure.push("You provided tags (#{arg}) as something other than a hash.")
|
||||
|
|
@ -1047,7 +1315,7 @@ module Vmpooler
|
|||
invalid.each do |bad_template|
|
||||
metrics.increment("config.invalid.#{bad_template}")
|
||||
end
|
||||
result[:bad_templates] = invalid
|
||||
result[:not_configured] = invalid
|
||||
status 400
|
||||
end
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue