puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 18:08:42 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Address rubocop offenses

Browse source
This commit is contained in:
kirby@puppetlabs.com 2020-05-07 15:19:44 -07:00
parent 86f6848ef0
commit 9538c0b3e3
3 changed files with 66 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/vmpooler.rb
View file

@ -166,11 +166,11 @@ module Vmpooler
metric_prefix: 'redis_connection_pool', metric_prefix: 'redis_connection_pool',
size: size, size: size,
timeout: timeout timeout: timeout
) { ) do
connection = Concurrent::Hash.new connection = Concurrent::Hash.new
redis = new_redis(host, port, password) redis = new_redis(host, port, password)
connection['connection'] = redis connection['connection'] = redis
} end
end end
def self.new_redis(host = 'localhost', port = nil, password = nil) def self.new_redis(host = 'localhost', port = nil, password = nil)

16
lib/vmpooler/api/v1.rb
View file

@ -343,7 +343,7 @@ module Vmpooler
result = { 'ok': false } result = { 'ok': false }
request_id = payload['request_id'] request_id = payload['request_id']
request_id = generate_request_id unless request_id request_id ||= generate_request_id
score = Time.now.to_i score = Time.now.to_i
result['request_id'] = request_id result['request_id'] = request_id
@ -357,13 +357,14 @@ module Vmpooler
status 201 status 201
platforms_with_aliases = [] platforms_with_aliases = []
payload.reject { |k,v| k == 'request_id' }.each do |poolname, count| payload.reject { |k, _v| k == 'request_id' }.each do |poolname, count|
selection = evaluate_template_aliases(poolname, count) selection = evaluate_template_aliases(poolname, count)
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
end end
platforms_string = platforms_with_aliases.join(',') platforms_string = platforms_with_aliases.join(',')
return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
if Vmpooler::API.settings.config[:auth] and has_token? if Vmpooler::API.settings.config[:auth] and has_token?
backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
@ -802,7 +803,7 @@ module Vmpooler
payload = JSON.parse(request.body.read) payload = JSON.parse(request.body.read)
if payload if payload
invalid = invalid_templates(payload.reject { |k,v| k == 'request_id' }) invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
if invalid.empty? if invalid.empty?
result = generate_ondemand_request(payload) result = generate_ondemand_request(payload)
else else
@ -829,7 +830,6 @@ module Vmpooler
get "#{api_prefix}/ondemandvm/:requestid/?" do get "#{api_prefix}/ondemandvm/:requestid/?" do
content_type :json content_type :json
result = { 'ok' => false }
status 404 status 404
result = check_ondemand_request(params[:requestid]) result = check_ondemand_request(params[:requestid])
@ -932,7 +932,6 @@ module Vmpooler
pool_alias, pool, _count = platform.split(':') pool_alias, pool, _count = platform.split(':')
instances = backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}") instances = backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
result[pool_alias] = { 'hostname': instances } result[pool_alias] = { 'hostname': instances }
#backend.del("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
end end
status 200 status 200
else else
@ -1110,13 +1109,6 @@ module Vmpooler
failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.") failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
end end
#checkout = backend.hget("vmpooler__vm__#{params[:hostname]}", 'checkout')
#if checkout
# existing_lifetime = (Time.now - Time.parse(checkout)) / 60 / 60
# if (arg.to_i + existing_lifetime) >= max_lifetime_upper_limit
# failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit} when added to the time since checkout.")
# end
#end
when 'tags' when 'tags'
unless arg.is_a?(Hash) unless arg.is_a?(Hash)
failure.push("You provided tags (#{arg}) as something other than a hash.") failure.push("You provided tags (#{arg}) as something other than a hash.")

117
lib/vmpooler/pool_manager.rb
View file

@ -81,7 +81,6 @@ module Vmpooler
rescue StandardError => e rescue StandardError => e
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}") $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
@redis.with_metrics do |redis| @redis.with_metrics do |redis|
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
fail_pending_vm(vm, pool, timeout, redis) fail_pending_vm(vm, pool, timeout, redis)
end end
raise raise
@ -192,7 +191,7 @@ module Vmpooler
mutex.synchronize do mutex.synchronize do
@redis.with_metrics do |redis| @redis.with_metrics do |redis|
check_stamp = redis.hget('vmpooler__vm__' + vm, 'check') check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) break if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
redis.hset('vmpooler__vm__' + vm, 'check', Time.now) redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
# Check if the hosts TTL has expired # Check if the hosts TTL has expired
@ -208,11 +207,11 @@ module Vmpooler
redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm) redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
$logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
return return nil
end end
end end
return if mismatched_hostname?(vm, pool_name, provider, redis) break if mismatched_hostname?(vm, pool_name, provider, redis)
vm_still_ready?(pool_name, vm, provider, redis) vm_still_ready?(pool_name, vm, provider, redis)
end end
@ -266,28 +265,30 @@ module Vmpooler
return if mutex.locked? return if mutex.locked?
mutex.synchronize do mutex.synchronize do
@redis.with_metrics do |redis| catch :stop_checking do
# Check that VM is within defined lifetime @redis.with_metrics do |redis|
checkouttime = redis.hget('vmpooler__active__' + pool, vm) # Check that VM is within defined lifetime
if checkouttime checkouttime = redis.hget('vmpooler__active__' + pool, vm)
time_since_checkout = Time.now - Time.parse(checkouttime) if checkouttime
running = time_since_checkout / 60 / 60 time_since_checkout = Time.now - Time.parse(checkouttime)
running = time_since_checkout / 60 / 60
if (ttl.to_i > 0) && (running.to_i >= ttl.to_i) if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours") move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
return throw :stop_checking
end
end end
end
if provider.vm_ready?(pool, vm) if provider.vm_ready?(pool, vm)
return throw :stop_checking
else
host = provider.get_vm(pool, vm)
if host
return
else else
move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running') host = provider.get_vm(pool, vm)
if host
throw :stop_checking
else
move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
end
end end
end end
end end
@ -358,7 +359,6 @@ module Vmpooler
new_vmname = find_unique_hostname(pool_name, redis) new_vmname = find_unique_hostname(pool_name, redis)
mutex = vm_mutex(new_vmname) mutex = vm_mutex(new_vmname)
mutex.synchronize do mutex.synchronize do
# Add VM to Redis inventory ('pending' pool) # Add VM to Redis inventory ('pending' pool)
redis.multi redis.multi
redis.sadd('vmpooler__pending__' + pool_name, new_vmname) redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
@ -369,8 +369,6 @@ module Vmpooler
redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
redis.exec redis.exec
vm_hash = redis.hgetall("vmpooler__vm__#{new_vmname}")
begin begin
$logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'") $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
start = Time.now start = Time.now
@ -453,8 +451,8 @@ module Vmpooler
redis.hget("vmpooler__vm__#{vm}", 'template') redis.hget("vmpooler__vm__#{vm}", 'template')
checkout, jenkins_build_url, user, poolname = redis.exec checkout, jenkins_build_url, user, poolname = redis.exec
return if checkout.nil? return if checkout.nil?
user ||= 'unauthenticated'
user ||= 'unauthenticated'
unless jenkins_build_url unless jenkins_build_url
user = user.gsub('.', '_') user = user.gsub('.', '_')
$metrics.increment("usage.#{user}.#{poolname}") $metrics.increment("usage.#{user}.#{poolname}")
@ -965,29 +963,32 @@ module Vmpooler
def evaluate_template(pool, provider) def evaluate_template(pool, provider)
mutex = pool_mutex(pool['name']) mutex = pool_mutex(pool['name'])
@redis.with_metrics do |redis| return if mutex.locked?
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
configured_template = redis.hget('vmpooler__config__template', pool['name'])
return if mutex.locked?
if prepared_template.nil? catch :update_not_needed do
mutex.synchronize do @redis.with_metrics do |redis|
prepare_template(pool, provider, redis) prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
prepared_template = redis.hget('vmpooler__template__prepared', pool['name']) configured_template = redis.hget('vmpooler__config__template', pool['name'])
end
elsif prepared_template != pool['template'] if prepared_template.nil?
if configured_template.nil?
mutex.synchronize do mutex.synchronize do
prepare_template(pool, provider, redis) prepare_template(pool, provider, redis)
prepared_template = redis.hget('vmpooler__template__prepared', pool['name']) prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
end end
elsif prepared_template != pool['template']
if configured_template.nil?
mutex.synchronize do
prepare_template(pool, provider, redis)
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
end
end
end end
end throw :update_not_needed if configured_template.nil?
return if configured_template.nil? throw :update_not_needed if configured_template == prepared_template
return if configured_template == prepared_template
mutex.synchronize do mutex.synchronize do
update_pool_template(pool, provider, configured_template, prepared_template, redis) update_pool_template(pool, provider, configured_template, prepared_template, redis)
end
end end
end end
end end
@ -1025,8 +1026,8 @@ module Vmpooler
@redis.with_metrics do |redis| @redis.with_metrics do |redis|
clone_target = redis.hget('vmpooler__config__clone_target', pool['name']) clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
return if clone_target.nil? break if clone_target.nil?
return if clone_target == pool['clone_target'] break if clone_target == pool['clone_target']
$logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}") $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
mutex.synchronize do mutex.synchronize do
@ -1045,12 +1046,12 @@ module Vmpooler
redis.scard("vmpooler__pending__#{pool['name']}") redis.scard("vmpooler__pending__#{pool['name']}")
ready, pending = redis.exec ready, pending = redis.exec
total = pending.to_i + ready.to_i total = pending.to_i + ready.to_i
return if total.nil? break if total.nil?
return if total == 0 break if total == 0
mutex = pool_mutex(pool['name']) mutex = pool_mutex(pool['name'])
return if mutex.locked? break if mutex.locked?
return unless ready.to_i > pool['size'] break unless ready.to_i > pool['size']
mutex.synchronize do mutex.synchronize do
difference = ready.to_i - pool['size'] difference = ready.to_i - pool['size']
@ -1073,10 +1074,10 @@ module Vmpooler
@redis.with_metrics do |redis| @redis.with_metrics do |redis|
poolsize = redis.hget('vmpooler__config__poolsize', pool['name']) poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
return if poolsize.nil? break if poolsize.nil?
poolsize = Integer(poolsize) poolsize = Integer(poolsize)
return if poolsize == pool['size'] break if poolsize == pool['size']
mutex.synchronize do mutex.synchronize do
pool['size'] = poolsize pool['size'] = poolsize
@ -1087,7 +1088,7 @@ module Vmpooler
def reset_pool(pool) def reset_pool(pool)
poolname = pool['name'] poolname = pool['name']
@redis.with_metrics do |redis| @redis.with_metrics do |redis|
return unless redis.sismember('vmpooler__poolreset', poolname) break unless redis.sismember('vmpooler__poolreset', poolname)
redis.srem('vmpooler__poolreset', poolname) redis.srem('vmpooler__poolreset', poolname)
mutex = pool_mutex(poolname) mutex = pool_mutex(poolname)
@ -1363,9 +1364,9 @@ module Vmpooler
end end
def _check_ondemand_requests(maxloop = 0, def _check_ondemand_requests(maxloop = 0,
loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT, loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT, loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT) loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil? loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil? loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
@ -1405,7 +1406,7 @@ module Vmpooler
def create_ondemand_vms(request_id, redis) def create_ondemand_vms(request_id, redis)
requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested') requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
if ! requested unless requested
$logger.log('s', "Failed to find odrequest for request_id '#{request_id}'") $logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
redis.zrem('vmpooler__provisioning__request', request_id) redis.zrem('vmpooler__provisioning__request', request_id)
return return
@ -1424,16 +1425,18 @@ module Vmpooler
def process_ondemand_vms(redis) def process_ondemand_vms(redis)
queue_key = 'vmpooler__odcreate__task' queue_key = 'vmpooler__odcreate__task'
queue = redis.zrange(queue_key, 0, -1, :with_scores => true) queue = redis.zrange(queue_key, 0, -1, with_scores: true)
ondemand_clone_limit = $config[:config]['ondemand_clone_limit'] ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
@tasks['ondemand_clone_count'] = 0 unless @tasks['ondemand_clone_count'] @tasks['ondemand_clone_count'] = 0 unless @tasks['ondemand_clone_count']
queue.each do |request, score| queue.each do |request, score|
break unless @tasks['ondemand_clone_count'] < ondemand_clone_limit break unless @tasks['ondemand_clone_count'] < ondemand_clone_limit
pool_alias, pool, count, request_id = request.split(':') pool_alias, pool, count, request_id = request.split(':')
count = count.to_i count = count.to_i
provider = get_provider_for_pool(pool) provider = get_provider_for_pool(pool)
slots = ondemand_clone_limit - @tasks['ondemand_clone_count'] slots = ondemand_clone_limit - @tasks['ondemand_clone_count']
return if slots == 0 break if slots == 0
if slots >= count if slots >= count
count.times do count.times do
@tasks['ondemand_clone_count'] += 1 @tasks['ondemand_clone_count'] += 1