From 48107bb6c762d5a4d6c0f1bd74410b75d55f2712 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Mon, 20 Dec 2021 12:36:34 -0500 Subject: [PATCH 001/184] Update to latest OTel gems --- vmpooler.gemspec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 75eb67c..1f3299e 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -21,12 +21,12 @@ Gem::Specification.new do |s| s.add_dependency 'connection_pool', '~> 2.2' s.add_dependency 'net-ldap', '~> 0.16' s.add_dependency 'nokogiri', '~> 1.10' - s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.17.0' - s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.17.0' - s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.17.0' - s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.17.0' - s.add_dependency 'opentelemetry-resource_detectors', '= 0.17.0' - s.add_dependency 'opentelemetry-sdk', '= 0.17.0' + s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.20.1' + s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.19.2' + s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.21.2' + s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.19.3' + s.add_dependency 'opentelemetry-resource_detectors', '= 0.19.1' + s.add_dependency 'opentelemetry-sdk', '~> 1.0', '>= 1.0.2' s.add_dependency 'pickup', '~> 0.0.11' s.add_dependency 'prometheus-client', '~> 2.0' s.add_dependency 'puma', '~> 5.0', '>= 5.0.4' From 2d1d28ed49ed54db79bd61d75fec08c132cef84b Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Mon, 20 Dec 2021 15:33:25 -0500 Subject: [PATCH 002/184] Add additional data to spans in api/v1.rb --- lib/vmpooler/api/helpers.rb | 699 ++++++++++++++++------------- lib/vmpooler/api/v1.rb | 869 ++++++++++++++++++++---------------- 2 files changed, 871 insertions(+), 697 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index c7be9ba..0f216ab 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -6,108 +6,61 @@ module Vmpooler module Helpers + def tracer + @tracer ||= OpenTelemetry.tracer_provider.tracer('api', Vmpooler::VERSION) + end + def has_token? request.env['HTTP_X_AUTH_TOKEN'].nil? ? false : true end def valid_token?(backend) - return false unless has_token? + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + return false unless has_token? - backend.exists?("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}") ? true : false + backend.exists?("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}") ? true : false + end end def validate_token(backend) - if valid_token?(backend) - backend.hset("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'last', Time.now) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + if valid_token?(backend) + backend.hset("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'last', Time.now) - return true + return true + end + + content_type :json + + result = { 'ok' => false } + + headers['WWW-Authenticate'] = 'Basic realm="Authentication required"' + halt 401, JSON.pretty_generate(result) end - - content_type :json - - result = { 'ok' => false } - - headers['WWW-Authenticate'] = 'Basic realm="Authentication required"' - halt 401, JSON.pretty_generate(result) end def validate_auth(backend) - return if authorized? + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + return if authorized? - content_type :json + content_type :json - result = { 'ok' => false } + result = { 'ok' => false } - headers['WWW-Authenticate'] = 'Basic realm="Authentication required"' - halt 401, JSON.pretty_generate(result) + headers['WWW-Authenticate'] = 'Basic realm="Authentication required"' + halt 401, JSON.pretty_generate(result) + end end def authorized? - @auth ||= Rack::Auth::Basic::Request.new(request.env) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + @auth ||= Rack::Auth::Basic::Request.new(request.env) - if @auth.provided? and @auth.basic? and @auth.credentials - username, password = @auth.credentials + if @auth.provided? and @auth.basic? and @auth.credentials + username, password = @auth.credentials - if authenticate(Vmpooler::API.settings.config[:auth], username, password) - return true - end - end - - return false - end - - def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str) - ldap = Net::LDAP.new( - :host => host, - :port => port, - :encryption => encryption_hash, - :base => base, - :auth => { - :method => :simple, - :username => "#{user_object}=#{username_str},#{base}", - :password => password_str - } - ) - - return true if ldap.bind - - return false - end - - def authenticate(auth, username_str, password_str) - case auth['provider'] - when 'dummy' - return (username_str != password_str) - when 'ldap' - ldap_base = auth[:ldap]['base'] - ldap_port = auth[:ldap]['port'] || 389 - ldap_user_obj = auth[:ldap]['user_object'] - ldap_host = auth[:ldap]['host'] - ldap_encryption_hash = auth[:ldap]['encryption'] || { - :method => :start_tls, - :tls_options => { :ssl_version => 'TLSv1' } - } - - unless ldap_base.is_a? Array - ldap_base = ldap_base.split - end - - unless ldap_user_obj.is_a? Array - ldap_user_obj = ldap_user_obj.split - end - - ldap_base.each do |search_base| - ldap_user_obj.each do |search_user_obj| - result = authenticate_ldap( - ldap_port, - ldap_host, - ldap_encryption_hash, - search_user_obj, - search_base, - username_str, - password_str - ) - return true if result + if authenticate(Vmpooler::API.settings.config[:auth], username, password) + return true end end @@ -115,27 +68,108 @@ module Vmpooler end end - def export_tags(backend, hostname, tags) - backend.pipelined do - tags.each_pair do |tag, value| - next if value.nil? or value.empty? + def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str) + tracer.in_span( + "Vmpooler::API::Helpers.#{__method__}", + attributes: { + 'net.peer.name' => host, + 'net.peer.port' => port, + 'net.transport' => 'ip_tcp', + 'enduser.id' => username_str + }, + kind: :client + ) do + ldap = Net::LDAP.new( + :host => host, + :port => port, + :encryption => encryption_hash, + :base => base, + :auth => { + :method => :simple, + :username => "#{user_object}=#{username_str},#{base}", + :password => password_str + } + ) - backend.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value) - backend.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value) + return true if ldap.bind + + return false + end + end + + def authenticate(auth, username_str, password_str) + tracer.in_span( + "Vmpooler::API::Helpers.#{__method__}", + attributes: { + 'enduser.id' => username_str + } + ) do + case auth['provider'] + when 'dummy' + return (username_str != password_str) + when 'ldap' + ldap_base = auth[:ldap]['base'] + ldap_port = auth[:ldap]['port'] || 389 + ldap_user_obj = auth[:ldap]['user_object'] + ldap_host = auth[:ldap]['host'] + ldap_encryption_hash = auth[:ldap]['encryption'] || { + :method => :start_tls, + :tls_options => { :ssl_version => 'TLSv1' } + } + + unless ldap_base.is_a? Array + ldap_base = ldap_base.split + end + + unless ldap_user_obj.is_a? Array + ldap_user_obj = ldap_user_obj.split + end + + ldap_base.each do |search_base| + ldap_user_obj.each do |search_user_obj| + result = authenticate_ldap( + ldap_port, + ldap_host, + ldap_encryption_hash, + search_user_obj, + search_base, + username_str, + password_str + ) + return true if result + end + end + + return false + end + end + end + + def export_tags(backend, hostname, tags) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + backend.pipelined do + tags.each_pair do |tag, value| + next if value.nil? or value.empty? + + backend.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value) + backend.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value) + end end end end def filter_tags(tags) - return unless Vmpooler::API.settings.config[:tagfilter] + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + return unless Vmpooler::API.settings.config[:tagfilter] - tags.each_pair do |tag, value| - next unless filter = Vmpooler::API.settings.config[:tagfilter][tag] + tags.each_pair do |tag, value| + next unless filter = Vmpooler::API.settings.config[:tagfilter][tag] - tags[tag] = value.match(filter).captures.join if value.match(filter) + tags[tag] = value.match(filter).captures.join if value.match(filter) + end + + tags end - - tags end def mean(list) @@ -156,301 +190,321 @@ module Vmpooler end def get_task_times(backend, task, date_str) - backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) + end end # Takes the pools and a key to run scard on # returns an integer for the total count def get_total_across_pools_redis_scard(pools, key, backend) - # using pipelined is much faster than querying each of the pools and adding them - # as we get the result. - res = backend.pipelined do - pools.each do |pool| - backend.scard(key + pool['name']) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + # using pipelined is much faster than querying each of the pools and adding them + # as we get the result. + res = backend.pipelined do + pools.each do |pool| + backend.scard(key + pool['name']) + end end + res.inject(0) { |m, x| m + x }.to_i end - res.inject(0) { |m, x| m + x }.to_i end # Takes the pools and a key to run scard on # returns a hash with each pool name as key and the value being the count as integer def get_list_across_pools_redis_scard(pools, key, backend) - # using pipelined is much faster than querying each of the pools and adding them - # as we get the result. - temp_hash = {} - res = backend.pipelined do - pools.each do |pool| - backend.scard(key + pool['name']) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + # using pipelined is much faster than querying each of the pools and adding them + # as we get the result. + temp_hash = {} + res = backend.pipelined do + pools.each do |pool| + backend.scard(key + pool['name']) + end end + pools.each_with_index do |pool, i| + temp_hash[pool['name']] = res[i].to_i + end + temp_hash end - pools.each_with_index do |pool, i| - temp_hash[pool['name']] = res[i].to_i - end - temp_hash end # Takes the pools and a key to run hget on # returns a hash with each pool name as key and the value as string def get_list_across_pools_redis_hget(pools, key, backend) - # using pipelined is much faster than querying each of the pools and adding them - # as we get the result. - temp_hash = {} - res = backend.pipelined do - pools.each do |pool| - backend.hget(key, pool['name']) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + # using pipelined is much faster than querying each of the pools and adding them + # as we get the result. + temp_hash = {} + res = backend.pipelined do + pools.each do |pool| + backend.hget(key, pool['name']) + end end + pools.each_with_index do |pool, i| + temp_hash[pool['name']] = res[i].to_s + end + temp_hash end - pools.each_with_index do |pool, i| - temp_hash[pool['name']] = res[i].to_s - end - temp_hash end def get_capacity_metrics(pools, backend) - capacity = { - current: 0, - total: 0, - percent: 0 - } + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + capacity = { + current: 0, + total: 0, + percent: 0 + } - pools.each do |pool| - capacity[:total] += pool['size'].to_i + pools.each do |pool| + capacity[:total] += pool['size'].to_i + end + + capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) + + if capacity[:total] > 0 + capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1) + end + + capacity end - - capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) - - if capacity[:total] > 0 - capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1) - end - - capacity end def get_queue_metrics(pools, backend) - queue = { - pending: 0, - cloning: 0, - booting: 0, - ready: 0, - running: 0, - completed: 0, - total: 0 - } + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + queue = { + pending: 0, + cloning: 0, + booting: 0, + ready: 0, + running: 0, + completed: 0, + total: 0 + } - queue[:pending] = get_total_across_pools_redis_scard(pools, 'vmpooler__pending__', backend) - queue[:ready] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) - queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) - queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend) + queue[:pending] = get_total_across_pools_redis_scard(pools, 'vmpooler__pending__', backend) + queue[:ready] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend) + queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) + queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend) - queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i - queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i - queue[:booting] = 0 if queue[:booting] < 0 - queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i + queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i + queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i + queue[:booting] = 0 if queue[:booting] < 0 + queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i - queue + queue + end end def get_tag_metrics(backend, date_str, opts = {}) - opts = {:only => false}.merge(opts) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + opts = {:only => false}.merge(opts) - tags = {} + tags = {} - backend.hgetall("vmpooler__tag__#{date_str}").each do |key, value| - hostname = 'unknown' - tag = 'unknown' + backend.hgetall("vmpooler__tag__#{date_str}").each do |key, value| + hostname = 'unknown' + tag = 'unknown' - if key =~ /:/ - hostname, tag = key.split(':', 2) + if key =~ /:/ + hostname, tag = key.split(':', 2) + end + + next if opts[:only] && tag != opts[:only] + + tags[tag] ||= {} + tags[tag][value] ||= 0 + tags[tag][value] += 1 + + tags[tag]['total'] ||= 0 + tags[tag]['total'] += 1 end - next if opts[:only] && tag != opts[:only] - - tags[tag] ||= {} - tags[tag][value] ||= 0 - tags[tag][value] += 1 - - tags[tag]['total'] ||= 0 - tags[tag]['total'] += 1 + tags end - - tags end def get_tag_summary(backend, from_date, to_date, opts = {}) - opts = {:only => false}.merge(opts) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + opts = {:only => false}.merge(opts) - result = { - tag: {}, - daily: [] - } - - (from_date..to_date).each do |date| - daily = { - date: date.to_s, - tag: get_tag_metrics(backend, date.to_s, opts) + result = { + tag: {}, + daily: [] } - result[:daily].push(daily) - end - result[:daily].each do |daily| - daily[:tag].each_key do |tag| - result[:tag][tag] ||= {} + (from_date..to_date).each do |date| + daily = { + date: date.to_s, + tag: get_tag_metrics(backend, date.to_s, opts) + } + result[:daily].push(daily) + end - daily[:tag][tag].each do |key, value| - result[:tag][tag][key] ||= 0 - result[:tag][tag][key] += value + result[:daily].each do |daily| + daily[:tag].each_key do |tag| + result[:tag][tag] ||= {} + + daily[:tag][tag].each do |key, value| + result[:tag][tag][key] ||= 0 + result[:tag][tag][key] += value + end end end - end - result + result + end end def get_task_metrics(backend, task_str, date_str, opts = {}) - opts = {:bypool => false, :only => false}.merge(opts) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + opts = {:bypool => false, :only => false}.merge(opts) - task = { - duration: { - average: 0, - min: 0, - max: 0, - total: 0 - }, - count: { - total: 0 - } - } + task = { + duration: { + average: 0, + min: 0, + max: 0, + total: 0 + }, + count: { + total: 0 + } + } - task[:count][:total] = backend.hlen("vmpooler__#{task_str}__#{date_str}").to_i + task[:count][:total] = backend.hlen("vmpooler__#{task_str}__#{date_str}").to_i - if task[:count][:total] > 0 - if opts[:bypool] == true - task_times_bypool = {} + if task[:count][:total] > 0 + if opts[:bypool] == true + task_times_bypool = {} - task[:count][:pool] = {} - task[:duration][:pool] = {} + task[:count][:pool] = {} + task[:duration][:pool] = {} - backend.hgetall("vmpooler__#{task_str}__#{date_str}").each do |key, value| - pool = 'unknown' - hostname = 'unknown' + backend.hgetall("vmpooler__#{task_str}__#{date_str}").each do |key, value| + pool = 'unknown' + hostname = 'unknown' - if key =~ /:/ - pool, hostname = key.split(':') - else - hostname = key + if key =~ /:/ + pool, hostname = key.split(':') + else + hostname = key + end + + task[:count][:pool][pool] ||= {} + task[:duration][:pool][pool] ||= {} + + task_times_bypool[pool] ||= [] + task_times_bypool[pool].push(value.to_f) end - task[:count][:pool][pool] ||= {} - task[:duration][:pool][pool] ||= {} + task_times_bypool.each_key do |pool| + task[:count][:pool][pool][:total] = task_times_bypool[pool].length - task_times_bypool[pool] ||= [] - task_times_bypool[pool].push(value.to_f) + task[:duration][:pool][pool][:total] = task_times_bypool[pool].reduce(:+).to_f + task[:duration][:pool][pool][:average] = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1) + task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax + end end - task_times_bypool.each_key do |pool| - task[:count][:pool][pool][:total] = task_times_bypool[pool].length + task_times = get_task_times(backend, task_str, date_str) - task[:duration][:pool][pool][:total] = task_times_bypool[pool].reduce(:+).to_f - task[:duration][:pool][pool][:average] = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1) - task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax + task[:duration][:total] = task_times.reduce(:+).to_f + task[:duration][:average] = (task[:duration][:total] / task[:count][:total]).round(1) + task[:duration][:min], task[:duration][:max] = task_times.minmax + end + + if opts[:only] + task.each_key do |key| + task.delete(key) unless key.to_s == opts[:only] end end - task_times = get_task_times(backend, task_str, date_str) - - task[:duration][:total] = task_times.reduce(:+).to_f - task[:duration][:average] = (task[:duration][:total] / task[:count][:total]).round(1) - task[:duration][:min], task[:duration][:max] = task_times.minmax + task end - - if opts[:only] - task.each_key do |key| - task.delete(key) unless key.to_s == opts[:only] - end - end - - task end def get_task_summary(backend, task_str, from_date, to_date, opts = {}) - opts = {:bypool => false, :only => false}.merge(opts) + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + opts = {:bypool => false, :only => false}.merge(opts) - task_sym = task_str.to_sym + task_sym = task_str.to_sym - result = { - task_sym => {}, - daily: [] - } - - (from_date..to_date).each do |date| - daily = { - date: date.to_s, - task_sym => get_task_metrics(backend, task_str, date.to_s, opts) + result = { + task_sym => {}, + daily: [] } - result[:daily].push(daily) - end - daily_task = {} - daily_task_bypool = {} if opts[:bypool] == true + (from_date..to_date).each do |date| + daily = { + date: date.to_s, + task_sym => get_task_metrics(backend, task_str, date.to_s, opts) + } + result[:daily].push(daily) + end - result[:daily].each do |daily| - daily[task_sym].each_key do |type| - result[task_sym][type] ||= {} - daily_task[type] ||= {} + daily_task = {} + daily_task_bypool = {} if opts[:bypool] == true - ['min', 'max'].each do |key| - if daily[task_sym][type][key] - daily_task[type][:data] ||= [] - daily_task[type][:data].push(daily[task_sym][type][key]) + result[:daily].each do |daily| + daily[task_sym].each_key do |type| + result[task_sym][type] ||= {} + daily_task[type] ||= {} + + ['min', 'max'].each do |key| + if daily[task_sym][type][key] + daily_task[type][:data] ||= [] + daily_task[type][:data].push(daily[task_sym][type][key]) + end + end + + result[task_sym][type][:total] ||= 0 + result[task_sym][type][:total] += daily[task_sym][type][:total] + + if opts[:bypool] == true + result[task_sym][type][:pool] ||= {} + daily_task_bypool[type] ||= {} + + next unless daily[task_sym][type][:pool] + + daily[task_sym][type][:pool].each_key do |pool| + result[task_sym][type][:pool][pool] ||= {} + daily_task_bypool[type][pool] ||= {} + + ['min', 'max'].each do |key| + if daily[task_sym][type][:pool][pool][key.to_sym] + daily_task_bypool[type][pool][:data] ||= [] + daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym]) + end + end + + result[task_sym][type][:pool][pool][:total] ||= 0 + result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total] + end end end + end - result[task_sym][type][:total] ||= 0 - result[task_sym][type][:total] += daily[task_sym][type][:total] + result[task_sym].each_key do |type| + if daily_task[type][:data] + result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax + result[task_sym][type][:average] = mean(daily_task[type][:data]) + end if opts[:bypool] == true - result[task_sym][type][:pool] ||= {} - daily_task_bypool[type] ||= {} - - next unless daily[task_sym][type][:pool] - - daily[task_sym][type][:pool].each_key do |pool| - result[task_sym][type][:pool][pool] ||= {} - daily_task_bypool[type][pool] ||= {} - - ['min', 'max'].each do |key| - if daily[task_sym][type][:pool][pool][key.to_sym] - daily_task_bypool[type][pool][:data] ||= [] - daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym]) + result[task_sym].each_key do |type| + result[task_sym][type][:pool].each_key do |pool| + if daily_task_bypool[type][pool][:data] + result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax + result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data]) end end - - result[task_sym][type][:pool][pool][:total] ||= 0 - result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total] end end end + + result end - - result[task_sym].each_key do |type| - if daily_task[type][:data] - result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax - result[task_sym][type][:average] = mean(daily_task[type][:data]) - end - - if opts[:bypool] == true - result[task_sym].each_key do |type| - result[task_sym][type][:pool].each_key do |pool| - if daily_task_bypool[type][pool][:data] - result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax - result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data]) - end - end - end - end - end - - result end def pool_index(pools) @@ -464,11 +518,13 @@ module Vmpooler end def template_ready?(pool, backend) - prepared_template = backend.hget('vmpooler__template__prepared', pool['name']) - return false if prepared_template.nil? - return true if pool['template'] == prepared_template + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + prepared_template = backend.hget('vmpooler__template__prepared', pool['name']) + return false if prepared_template.nil? + return true if pool['template'] == prepared_template - return false + return false + end end def is_integer?(x) @@ -479,26 +535,39 @@ module Vmpooler end def open_socket(host, domain = nil, timeout = 1, port = 22, &_block) - Timeout.timeout(timeout) do - target_host = host - target_host = "#{host}.#{domain}" if domain - sock = TCPSocket.new target_host, port - begin - yield sock if block_given? - ensure - sock.close + tracer.in_span( + "Vmpooler::API::Helpers.#{__method__}", + attributes: { + 'net.peer.port' => port, + 'net.transport' => 'ip_tcp' + }, + kind: :client + ) do + Timeout.timeout(timeout) do + target_host = host + target_host = "#{host}.#{domain}" if domain + span = OpenTelemetry::Trace.current_span + span.set_attribute('net.peer.name', target_host) + sock = TCPSocket.new target_host, port + begin + yield sock if block_given? + ensure + sock.close + end end end end def vm_ready?(vm_name, domain = nil) - begin - open_socket(vm_name, domain) - rescue StandardError => _e - return false - end + tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do + begin + open_socket(vm_name, domain) + rescue StandardError => _e + return false + end - true + true + end end end end diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index f209de2..c11b2cc 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -49,13 +49,15 @@ module Vmpooler end def get_template_aliases(template) - result = [] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - result += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = [] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + result += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + end + result end - result end def get_pool_weights(template_backends) @@ -109,398 +111,463 @@ module Vmpooler end def fetch_single_vm(template) - template_backends = [template] - aliases = Vmpooler::API.settings.config[:alias] - if aliases - template_backends += aliases[template] if aliases[template].is_a?(Array) - template_backends << aliases[template] if aliases[template].is_a?(String) - pool_index = pool_index(pools) - weighted_pools = {} - template_backends.each do |t| - next unless pool_index.key? t + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + template_backends = [template] + aliases = Vmpooler::API.settings.config[:alias] + if aliases + template_backends += aliases[template] if aliases[template].is_a?(Array) + template_backends << aliases[template] if aliases[template].is_a?(String) + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t - index = pool_index[t] - clone_target = pools[index]['clone_target'] || config['clone_target'] - next unless config.key?('backend_weight') + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') - weight = config['backend_weight'][clone_target] - if weight - weighted_pools[t] = weight - end - end - - if weighted_pools.count == template_backends.count - pickup = Pickup.new(weighted_pools) - selection = pickup.pick - template_backends.delete(selection) - template_backends.unshift(selection) - else - first = template_backends.sample - template_backends.delete(first) - template_backends.unshift(first) - end - end - - checkoutlock.synchronize do - template_backends.each do |template_backend| - vms = backend.smembers("vmpooler__ready__#{template_backend}") - next if vms.empty? - - vms.reverse.each do |vm| - ready = vm_ready?(vm, config['domain']) - if ready - smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) - if smoved - return [vm, template_backend, template] - else - metrics.increment("checkout.smove.failed.#{template_backend}") - return [nil, nil, nil] - end - else - backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) - metrics.increment("checkout.nonresponsive.#{template_backend}") + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight end end + + if weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + selection = pickup.pick + template_backends.delete(selection) + template_backends.unshift(selection) + else + first = template_backends.sample + template_backends.delete(first) + template_backends.unshift(first) + end + end + + checkoutlock.synchronize do + template_backends.each do |template_backend| + vms = backend.smembers("vmpooler__ready__#{template_backend}") + next if vms.empty? + + vms.reverse.each do |vm| + ready = vm_ready?(vm, config['domain']) + if ready + smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) + if smoved + return [vm, template_backend, template] + else + metrics.increment("checkout.smove.failed.#{template_backend}") + return [nil, nil, nil] + end + else + backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) + metrics.increment("checkout.nonresponsive.#{template_backend}") + end + end + end + [nil, nil, nil] end - [nil, nil, nil] end end def return_vm_to_ready_state(template, vm) - backend.srem("vmpooler__migrating__#{template}", vm) - backend.hdel("vmpooler__active__#{template}", vm) - backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user') - backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm) + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + backend.srem("vmpooler__migrating__#{template}", vm) + backend.hdel("vmpooler__active__#{template}", vm) + backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user') + backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm) + end end def account_for_starting_vm(template, vm) - user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user') - has_token_result = has_token? - backend.sadd("vmpooler__migrating__#{template}", vm) - backend.hset("vmpooler__active__#{template}", vm, Time.now) - backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user') + span.set_attribute('enduser.id', user) + has_token_result = has_token? + backend.sadd("vmpooler__migrating__#{template}", vm) + backend.hset("vmpooler__active__#{template}", vm, Time.now) + backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) - if Vmpooler::API.settings.config[:auth] and has_token_result - backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) - backend.hset("vmpooler__vm__#{vm}", 'token:user', user) + if Vmpooler::API.settings.config[:auth] and has_token_result + backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) + backend.hset("vmpooler__vm__#{vm}", 'token:user', user) - if config['vm_lifetime_auth'].to_i > 0 - backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i) + if config['vm_lifetime_auth'].to_i > 0 + backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i) + end end end end def update_result_hosts(result, template, vm) - result[template] ||= {} - if result[template]['hostname'] - result[template]['hostname'] = Array(result[template]['hostname']) - result[template]['hostname'].push(vm) - else - result[template]['hostname'] = vm + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result[template] ||= {} + if result[template]['hostname'] + result[template]['hostname'] = Array(result[template]['hostname']) + result[template]['hostname'].push(vm) + else + result[template]['hostname'] = vm + end end end def atomically_allocate_vms(payload) - result = { 'ok' => false } - failed = false - vms = [] + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + result = { 'ok' => false } + failed = false + vms = [] - validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? + validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? - payload.each do |requested, count| - count.to_i.times do |_i| - vmname, vmpool, vmtemplate = fetch_single_vm(requested) - if vmname - account_for_starting_vm(vmpool, vmname) - vms << [vmpool, vmname, vmtemplate] - metrics.increment("checkout.success.#{vmpool}") - update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] - else - failed = true - metrics.increment("checkout.empty.#{requested}") - break + payload.each do |requested, count| + count.to_i.times do |_i| + vmname, vmpool, vmtemplate = fetch_single_vm(requested) + if vmname + account_for_starting_vm(vmpool, vmname) + vms << [vmpool, vmname, vmtemplate] + metrics.increment("checkout.success.#{vmpool}") + update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] + else + failed = true + metrics.increment("checkout.empty.#{requested}") + break + end end end - end - if failed - vms.each do |(vmpool, vmname, _vmtemplate)| - return_vm_to_ready_state(vmpool, vmname) - end - status 503 - else - vms.each do |(_vmpool, vmname, vmtemplate)| - update_result_hosts(result, vmtemplate, vmname) + if failed + vms.each do |(vmpool, vmname, _vmtemplate)| + return_vm_to_ready_state(vmpool, vmname) + end + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V1.atomically_allocate_vms', + 'error.message' => '503 due to failing to allocate one or more vms' + }) + status 503 + else + vm_names = [] + vms.each do |(_vmpool, vmname, vmtemplate)| + update_result_hosts(result, vmtemplate, vmname) + vm_names.append(vmname) + end + + span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty? + + result['ok'] = true + result['domain'] = config['domain'] if config['domain'] end - result['ok'] = true - result['domain'] = config['domain'] if config['domain'] + result end - - result end def component_to_test(match, labels_string) - return if labels_string.nil? + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + return if labels_string.nil? - labels_string_parts = labels_string.split(',') - labels_string_parts.each do |part| - key, value = part.split('=') - next if value.nil? - return value if key == match + labels_string_parts = labels_string.split(',') + labels_string_parts.each do |part| + key, value = part.split('=') + next if value.nil? + return value if key == match + end + 'none' end - 'none' end def update_user_metrics(operation, vmname) - backend.multi - backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url') - backend.hget("vmpooler__vm__#{vmname}", 'token:user') - backend.hget("vmpooler__vm__#{vmname}", 'template') - jenkins_build_url, user, poolname = backend.exec - poolname = poolname.gsub('.', '_') + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + begin + backend.multi + backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url') + backend.hget("vmpooler__vm__#{vmname}", 'token:user') + backend.hget("vmpooler__vm__#{vmname}", 'template') + jenkins_build_url, user, poolname = backend.exec + poolname = poolname.gsub('.', '_') - if user - user = user.gsub('.', '_') - else - user = 'unauthenticated' - end - metrics.increment("user.#{user}.#{operation}.#{poolname}") + if user + user = user.gsub('.', '_') + else + user = 'unauthenticated' + end + metrics.increment("user.#{user}.#{operation}.#{poolname}") - if jenkins_build_url - if jenkins_build_url.include? 'litmus' - # Very simple filter for Litmus jobs - just count them coming through for the moment. - metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}") - return + if jenkins_build_url + if jenkins_build_url.include? 'litmus' + # Very simple filter for Litmus jobs - just count them coming through for the moment. + metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}") + else + url_parts = jenkins_build_url.split('/')[2..-1] + jenkins_instance = url_parts[0].gsub('.', '_') + value_stream_parts = url_parts[2].split('_') + value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') } + value_stream = value_stream_parts.shift + branch = value_stream_parts.pop + project = value_stream_parts.shift + job_name = value_stream_parts.join('_') + build_metadata_parts = url_parts[3] + component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) + + metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") + metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") + metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") + end + end + rescue StandardError => e + puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error(e.to_s) + span.add_event('log', attributes: { + 'log.severity' => 'debug', + 'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" + }) end - - url_parts = jenkins_build_url.split('/')[2..-1] - jenkins_instance = url_parts[0].gsub('.', '_') - value_stream_parts = url_parts[2].split('_') - value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') } - value_stream = value_stream_parts.shift - branch = value_stream_parts.pop - project = value_stream_parts.shift - job_name = value_stream_parts.join('_') - build_metadata_parts = url_parts[3] - component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) - - metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") - metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") - metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") end - rescue StandardError => e - puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" end def reset_pool_size(poolname) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - pool_index = pool_index(pools) + pool_index = pool_index(pools) - pools_updated = 0 - sync_pool_sizes + pools_updated = 0 + sync_pool_sizes - pool_size_now = pools[pool_index[poolname]]['size'].to_i - pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i - result['pool_size_before_reset'] = pool_size_now - result['pool_size_before_overrides'] = pool_size_original + pool_size_now = pools[pool_index[poolname]]['size'].to_i + pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i + result['pool_size_before_reset'] = pool_size_now + result['pool_size_before_overrides'] = pool_size_original - unless pool_size_now == pool_size_original - pools[pool_index[poolname]]['size'] = pool_size_original - backend.hdel('vmpooler__config__poolsize', poolname) - backend.sadd('vmpooler__pool__undo_size_override', poolname) - pools_updated += 1 - status 201 + unless pool_size_now == pool_size_original + pools[pool_index[poolname]]['size'] = pool_size_original + backend.hdel('vmpooler__config__poolsize', poolname) + backend.sadd('vmpooler__pool__undo_size_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result end - - status 200 unless pools_updated > 0 - result['ok'] = true - result end def update_pool_size(payload) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - pool_index = pool_index(pools) - pools_updated = 0 - sync_pool_sizes + pool_index = pool_index(pools) + pools_updated = 0 + sync_pool_sizes - payload.each do |poolname, size| - unless pools[pool_index[poolname]]['size'] == size.to_i - pools[pool_index[poolname]]['size'] = size.to_i - backend.hset('vmpooler__config__poolsize', poolname, size) - pools_updated += 1 - status 201 + payload.each do |poolname, size| + unless pools[pool_index[poolname]]['size'] == size.to_i + pools[pool_index[poolname]]['size'] = size.to_i + backend.hset('vmpooler__config__poolsize', poolname, size) + pools_updated += 1 + status 201 + end end + status 200 unless pools_updated > 0 + result['ok'] = true + result end - status 200 unless pools_updated > 0 - result['ok'] = true - result end def reset_pool_template(poolname) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - pool_index_live = pool_index(pools) - pool_index_original = pool_index(pools_at_startup) + pool_index_live = pool_index(pools) + pool_index_original = pool_index(pools_at_startup) - pools_updated = 0 - sync_pool_templates + pools_updated = 0 + sync_pool_templates - template_now = pools[pool_index_live[poolname]]['template'] - template_original = pools_at_startup[pool_index_original[poolname]]['template'] - result['template_before_reset'] = template_now - result['template_before_overrides'] = template_original + template_now = pools[pool_index_live[poolname]]['template'] + template_original = pools_at_startup[pool_index_original[poolname]]['template'] + result['template_before_reset'] = template_now + result['template_before_overrides'] = template_original - unless template_now == template_original - pools[pool_index_live[poolname]]['template'] = template_original - backend.hdel('vmpooler__config__template', poolname) - backend.sadd('vmpooler__pool__undo_template_override', poolname) - pools_updated += 1 - status 201 + unless template_now == template_original + pools[pool_index_live[poolname]]['template'] = template_original + backend.hdel('vmpooler__config__template', poolname) + backend.sadd('vmpooler__pool__undo_template_override', poolname) + pools_updated += 1 + status 201 + end + + status 200 unless pools_updated > 0 + result['ok'] = true + result end - - status 200 unless pools_updated > 0 - result['ok'] = true - result end def update_pool_template(payload) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - pool_index = pool_index(pools) - pools_updated = 0 - sync_pool_templates + pool_index = pool_index(pools) + pools_updated = 0 + sync_pool_templates - payload.each do |poolname, template| - unless pools[pool_index[poolname]]['template'] == template - pools[pool_index[poolname]]['template'] = template - backend.hset('vmpooler__config__template', poolname, template) - pools_updated += 1 - status 201 + payload.each do |poolname, template| + unless pools[pool_index[poolname]]['template'] == template + pools[pool_index[poolname]]['template'] = template + backend.hset('vmpooler__config__template', poolname, template) + pools_updated += 1 + status 201 + end end + status 200 unless pools_updated > 0 + result['ok'] = true + result end - status 200 unless pools_updated > 0 - result['ok'] = true - result end def reset_pool(payload) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - payload.each do |poolname, _count| - backend.sadd('vmpooler__poolreset', poolname) + payload.each do |poolname, _count| + backend.sadd('vmpooler__poolreset', poolname) + end + status 201 + result['ok'] = true + result end - status 201 - result['ok'] = true - result end def update_clone_target(payload) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + result = { 'ok' => false } - pool_index = pool_index(pools) - pools_updated = 0 - sync_clone_targets + pool_index = pool_index(pools) + pools_updated = 0 + sync_clone_targets - payload.each do |poolname, clone_target| - unless pools[pool_index[poolname]]['clone_target'] == clone_target - pools[pool_index[poolname]]['clone_target'] = clone_target - backend.hset('vmpooler__config__clone_target', poolname, clone_target) - pools_updated += 1 - status 201 + payload.each do |poolname, clone_target| + unless pools[pool_index[poolname]]['clone_target'] == clone_target + pools[pool_index[poolname]]['clone_target'] = clone_target + backend.hset('vmpooler__config__clone_target', poolname, clone_target) + pools_updated += 1 + status 201 + end end + status 200 unless pools_updated > 0 + result['ok'] = true + result end - status 200 unless pools_updated > 0 - result['ok'] = true - result end def sync_pool_templates - pool_index = pool_index(pools) - template_configs = backend.hgetall('vmpooler__config__template') - template_configs&.each do |poolname, template| - next unless pool_index.include? poolname + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + pool_index = pool_index(pools) + template_configs = backend.hgetall('vmpooler__config__template') + template_configs&.each do |poolname, template| + next unless pool_index.include? poolname - pools[pool_index[poolname]]['template'] = template + pools[pool_index[poolname]]['template'] = template + end end end def sync_pool_sizes - pool_index = pool_index(pools) - poolsize_configs = backend.hgetall('vmpooler__config__poolsize') - poolsize_configs&.each do |poolname, size| - next unless pool_index.include? poolname + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + pool_index = pool_index(pools) + poolsize_configs = backend.hgetall('vmpooler__config__poolsize') + poolsize_configs&.each do |poolname, size| + next unless pool_index.include? poolname - pools[pool_index[poolname]]['size'] = size.to_i + pools[pool_index[poolname]]['size'] = size.to_i + end end end def sync_clone_targets - pool_index = pool_index(pools) - clone_target_configs = backend.hgetall('vmpooler__config__clone_target') - clone_target_configs&.each do |poolname, clone_target| - next unless pool_index.include? poolname + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + pool_index = pool_index(pools) + clone_target_configs = backend.hgetall('vmpooler__config__clone_target') + clone_target_configs&.each do |poolname, clone_target| + next unless pool_index.include? poolname - pools[pool_index[poolname]]['clone_target'] = clone_target + pools[pool_index[poolname]]['clone_target'] = clone_target + end end end def too_many_requested?(payload) - payload&.each do |poolname, count| - next unless count.to_i > config['max_ondemand_instances_per_request'] + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + payload&.each do |poolname, count| + next unless count.to_i > config['max_ondemand_instances_per_request'] - metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}") - return true + metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}") + return true + end + false end - false end def generate_ondemand_request(payload) - result = { 'ok': false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + result = { 'ok': false } - requested_instances = payload.reject { |k, _v| k == 'request_id' } - if too_many_requested?(requested_instances) - result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" - status 403 - return result + requested_instances = payload.reject { |k, _v| k == 'request_id' } + if too_many_requested?(requested_instances) + e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" + result['message'] = e_message + status 403 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request', + 'error.message' => "403 due to #{e_message}" + }) + return result + end + + score = Time.now.to_i + request_id = payload['request_id'] + request_id ||= generate_request_id + result['request_id'] = request_id + span.set_attribute('vmpooler.request_id', request_id) + + if backend.exists?("vmpooler__odrequest__#{request_id}") + e_message = "request_id '#{request_id}' has already been created" + result['message'] = e_message + status 409 + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request', + 'error.message' => "409 due to #{e_message}" + }) + metrics.increment('ondemandrequest_generate.duplicaterequests') + return result + end + + status 201 + + platforms_with_aliases = [] + requested_instances.each do |poolname, count| + selection = evaluate_template_aliases(poolname, count) + selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } + end + platforms_string = platforms_with_aliases.join(',') + + return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) + + backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) + if Vmpooler::API.settings.config[:auth] and has_token? + token_token = request.env['HTTP_X_AUTH_TOKEN'] + token_user = backend.hget("vmpooler__token__#{token_token}", 'user') + backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token) + backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user) + span.set_attribute('enduser.id', token_user) + end + + result['domain'] = config['domain'] if config['domain'] + result[:ok] = true + metrics.increment('ondemandrequest_generate.success') + result end - - score = Time.now.to_i - request_id = payload['request_id'] - request_id ||= generate_request_id - result['request_id'] = request_id - - if backend.exists?("vmpooler__odrequest__#{request_id}") - result['message'] = "request_id '#{request_id}' has already been created" - status 409 - metrics.increment('ondemandrequest_generate.duplicaterequests') - return result - end - - status 201 - - platforms_with_aliases = [] - requested_instances.each do |poolname, count| - selection = evaluate_template_aliases(poolname, count) - selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } - end - platforms_string = platforms_with_aliases.join(',') - - return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) - - backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) - if Vmpooler::API.settings.config[:auth] and has_token? - backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) - backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', - backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')) - end - - result['domain'] = config['domain'] if config['domain'] - result[:ok] = true - metrics.increment('ondemandrequest_generate.success') - result end def generate_request_id @@ -813,6 +880,8 @@ module Vmpooler data = backend.hgetall(key) if data['user'] == Rack::Auth::Basic::Request.new(request.env).username + span = OpenTelemetry::Trace.current_span + span.set_attribute('enduser.id', data['user']) token = key.split('__').last result[token] ||= {} @@ -899,6 +968,8 @@ module Vmpooler backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username) backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now) + span = OpenTelemetry::Trace.current_span + span.set_attribute('enduser.id', @auth.username) status 200 result['ok'] = true @@ -946,6 +1017,8 @@ module Vmpooler status 404 end rescue JSON::ParserError + span = OpenTelemetry::Trace.current_span + span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') status 400 result = { 'ok' => false, @@ -1031,134 +1104,160 @@ module Vmpooler end def extract_templates_from_query_params(params) - payload = {} + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + payload = {} - params.split('+').each do |template| - payload[template] ||= 0 - payload[template] += 1 + params.split('+').each do |template| + payload[template] ||= 0 + payload[template] += 1 + end + + payload end - - payload end def invalid_templates(payload) - invalid = [] - payload.keys.each do |template| - invalid << template unless pool_exists?(template) + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + invalid = [] + payload.keys.each do |template| + invalid << template unless pool_exists?(template) + end + invalid end - invalid end def invalid_template_or_size(payload) - invalid = [] - payload.each do |pool, size| - invalid << pool unless pool_exists?(pool) - unless is_integer?(size) - invalid << pool - next + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + invalid = [] + payload.each do |pool, size| + invalid << pool unless pool_exists?(pool) + unless is_integer?(size) + invalid << pool + next + end + invalid << pool unless Integer(size) >= 0 end - invalid << pool unless Integer(size) >= 0 + invalid end - invalid end def invalid_template_or_path(payload) - invalid = [] - payload.each do |pool, template| - invalid << pool unless pool_exists?(pool) - invalid << pool unless template.include? '/' - invalid << pool if template[0] == '/' - invalid << pool if template[-1] == '/' + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + invalid = [] + payload.each do |pool, template| + invalid << pool unless pool_exists?(pool) + invalid << pool unless template.include? '/' + invalid << pool if template[0] == '/' + invalid << pool if template[-1] == '/' + end + invalid end - invalid end def invalid_pool(payload) - invalid = [] - payload.each do |pool, _clone_target| - invalid << pool unless pool_exists?(pool) + tracer.in_span("Vmpooler::API::V1.#{__method__}") do + invalid = [] + payload.each do |pool, _clone_target| + invalid << pool unless pool_exists?(pool) + end + invalid end - invalid end def check_ondemand_request(request_id) - result = { 'ok' => false } - request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") - if request_hash.empty? - result['message'] = "no request found for request_id '#{request_id}'" - return result - end + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + span.set_attribute('vmpooler.request_id', request_id) + result = { 'ok' => false } + request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") + if request_hash.empty? + e_message = "no request found for request_id '#{request_id}'" + result['message'] = e_message + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V1.check_ondemand_request', + 'error.message' => e_message + }) + return result + end - result['request_id'] = request_id - result['ready'] = false - result['ok'] = true - status 202 + result['request_id'] = request_id + result['ready'] = false + result['ok'] = true + status 202 - case request_hash['status'] - when 'ready' - result['ready'] = true - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| - instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + case request_hash['status'] + when 'ready' + result['ready'] = true + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| + instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - if result.key?(platform_alias) - result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances - else - result[platform_alias] = { 'hostname': instances } + if result.key?(platform_alias) + result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances + else + result[platform_alias] = { 'hostname': instances } + end + end + result['domain'] = config['domain'] if config['domain'] + status 200 + when 'failed' + result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" + status 200 + when 'deleted' + result['message'] = 'The request has been deleted' + status 200 + else + Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| + instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + instances_pending = count.to_i - instance_count.to_i + + if result.key?(platform_alias) && result[platform_alias].key?(:ready) + result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s + result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s + else + result[platform_alias] = { + 'ready': instance_count.to_s, + 'pending': instances_pending.to_s + } + end end end - result['domain'] = config['domain'] if config['domain'] - status 200 - when 'failed' - result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" - status 200 - when 'deleted' - result['message'] = 'The request has been deleted' - status 200 - else - Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| - instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") - instances_pending = count.to_i - instance_count.to_i - if result.key?(platform_alias) && result[platform_alias].key?(:ready) - result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s - result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s - else - result[platform_alias] = { - 'ready': instance_count.to_s, - 'pending': instances_pending.to_s - } - end - end + result end - - result end def delete_ondemand_request(request_id) - result = { 'ok' => false } + tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span| + span.set_attribute('vmpooler.request_id', request_id) + result = { 'ok' => false } - platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') - unless platforms - result['message'] = "no request found for request_id '#{request_id}'" - return result - end - - if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' - result['message'] = 'the request has already been deleted' - else - backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') - - Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count| - backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm| - backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) - end - backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') + unless platforms + e_message = "no request found for request_id '#{request_id}'" + result['message'] = e_message + span.add_event('error', attributes: { + 'error.type' => 'Vmpooler::API::V1.delete_ondemand_request', + 'error.message' => e_message + }) + return result end - backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) + + if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' + result['message'] = 'the request has already been deleted' + else + backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') + + Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count| + backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm| + backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) + end + backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}") + end + backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) + end + status 200 + result['ok'] = true + result end - status 200 - result['ok'] = true - result end post "#{api_prefix}/vm/:template/?" do @@ -1303,7 +1402,10 @@ module Vmpooler if backend.exists?("vmpooler__vm__#{params[:hostname]}") begin jdata = JSON.parse(request.body.read) - rescue StandardError + rescue StandardError => e + span = OpenTelemetry::Trace.current_span + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error(e.to_s) halt 400, JSON.pretty_generate(result) end @@ -1559,6 +1661,9 @@ module Vmpooler status 404 end rescue JSON::ParserError + span = OpenTelemetry::Trace.current_span + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed') status 400 result = { 'ok' => false, From 23efcc4cc0b12c791aa45a224273e40db48dc04a Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Thu, 23 Dec 2021 10:58:38 -0600 Subject: [PATCH 003/184] (maint) Fix EXTRA_CONFIG merge behavior Before this change if an extra config file had new keys they would get merged to the main config but if it contained an existing key, like 'providers' it would overwrite the original config. Adding a library 'deep_merge' to do a more natural merge, where existing keys get sub-elements added together and arrays are combined like for the pool configuration. Adding spec tests around EXTRA_CONFIG as they were missing, by adding and testing two new extra_config.yaml fixture files --- lib/vmpooler.rb | 3 ++- spec/fixtures/extra_config1.yaml | 4 ++++ spec/fixtures/extra_config2.yaml | 11 +++++++++++ spec/unit/vmpooler_spec.rb | 22 +++++++++++++++++++++- vmpooler.gemspec | 1 + 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/extra_config1.yaml create mode 100644 spec/fixtures/extra_config2.yaml diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 8b86fdb..c68e4ef 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -3,6 +3,7 @@ module Vmpooler require 'concurrent' require 'date' + require 'deep_merge' require 'json' require 'net/ldap' require 'open-uri' @@ -42,7 +43,7 @@ module Vmpooler extra_configs = parsed_config[:config]['extra_config'].split(',') extra_configs.each do |config| extra_config = YAML.load_file(config) - parsed_config.merge!(extra_config) + parsed_config.deep_merge(extra_config) end end end diff --git a/spec/fixtures/extra_config1.yaml b/spec/fixtures/extra_config1.yaml new file mode 100644 index 0000000..d19523c --- /dev/null +++ b/spec/fixtures/extra_config1.yaml @@ -0,0 +1,4 @@ +--- +:providers: + :alice: + foo: "foo" \ No newline at end of file diff --git a/spec/fixtures/extra_config2.yaml b/spec/fixtures/extra_config2.yaml new file mode 100644 index 0000000..e9f30d8 --- /dev/null +++ b/spec/fixtures/extra_config2.yaml @@ -0,0 +1,11 @@ +--- +:providers: + :bob: + foo: "foo_bob" + bar: "bar" + +:pools: + - name: 'pool05' + size: 5 + provider: dummy + ready_ttl: 5 \ No newline at end of file diff --git a/spec/unit/vmpooler_spec.rb b/spec/unit/vmpooler_spec.rb index 19aed91..84d43ed 100644 --- a/spec/unit/vmpooler_spec.rb +++ b/spec/unit/vmpooler_spec.rb @@ -45,10 +45,30 @@ describe 'Vmpooler' do end context 'when config file is set' do - it 'should use the file' do + before(:each) do ENV['VMPOOLER_CONFIG_FILE'] = config_file + end + it 'should use the file' do expect(Vmpooler.config[:pools]).to eq(config[:pools]) end + it 'merges one extra file, results in two providers' do + ENV['EXTRA_CONFIG'] = File.join(fixtures_dir, 'extra_config1.yaml') + expect(Vmpooler.config[:providers].keys).to include(:dummy) + expect(Vmpooler.config[:providers].keys).to include(:alice) + end + it 'merges two extra file, results in three providers and an extra pool' do + extra1 = File.join(fixtures_dir, 'extra_config1.yaml') + extra2 = File.join(fixtures_dir, 'extra_config2.yaml') + ENV['EXTRA_CONFIG'] = "#{extra1},#{extra2}" + expect(Vmpooler.config[:providers].keys).to include(:dummy) + expect(Vmpooler.config[:providers].keys).to include(:alice) + expect(Vmpooler.config[:providers].keys).to include(:bob) + merged_pools = [{"name"=>"pool03", "provider"=>"dummy", "ready_ttl"=>5, "size"=>5}, + {"name"=>"pool04", "provider"=>"dummy", "ready_ttl"=>5, "size"=>5}, + {"name"=>"pool05", "provider"=>"dummy", "ready_ttl"=>5, "size"=>5}] + expect(Vmpooler.config[:pools]).to eq(merged_pools) + expect(Vmpooler.config[:config]).not_to be_nil #merge does not deleted existing keys + end end end end diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 1f3299e..144ae0b 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] s.add_dependency 'concurrent-ruby', '~> 1.1' s.add_dependency 'connection_pool', '~> 2.2' + s.add_dependency 'deep_merge', '~> 1.2' s.add_dependency 'net-ldap', '~> 0.16' s.add_dependency 'nokogiri', '~> 1.10' s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.20.1' From 3a508a3afb54a163d241023b3e191d471faecde7 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Thu, 23 Dec 2021 14:08:18 -0600 Subject: [PATCH 004/184] (maint) do not raise an error when base provider create_template_delta_disks called --- lib/vmpooler/providers/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb index 25d39ef..1147aa8 100644 --- a/lib/vmpooler/providers/base.rb +++ b/lib/vmpooler/providers/base.rb @@ -242,7 +242,7 @@ module Vmpooler # returns # nil when successful. Raises error when encountered def create_template_delta_disks(_pool) - raise("#{self.class.name} does not implement create_template_delta_disks") + puts("#{self.class.name} does not implement create_template_delta_disks") end # inputs From ddbd522d5c18bbd65fead097456685270959304e Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 29 Dec 2021 13:27:24 -0600 Subject: [PATCH 005/184] Bump version to 2.2.0 In preparation for release --- lib/vmpooler/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vmpooler/version.rb b/lib/vmpooler/version.rb index ab53724..00af653 100644 --- a/lib/vmpooler/version.rb +++ b/lib/vmpooler/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Vmpooler - VERSION = '2.1.0' + VERSION = '2.2.0' end From f7eaeedbfc1f320c86019f96edd5d2d127740e5e Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Wed, 19 Jan 2022 14:59:12 -0500 Subject: [PATCH 006/184] Update dev tooling and related docs --- .dockerignore | 1 - .github/dependabot.yml | 6 + .github/workflows/lightstep.yml | 29 ---- .github/workflows/release.yml | 4 +- .github/workflows/testing.yml | 5 +- .gitignore | 2 - .lightstep.yml | 10 -- Gemfile.lock | 194 ++++++++++++++++++++++++++ README.md | 86 ++++++------ Vagrantfile | 2 + docker/Dockerfile | 31 ++--- docker/Dockerfile-aio | 39 ------ docker/Dockerfile_local | 36 ----- docker/Gemfile | 5 + docker/Gemfile.lock | 205 +++++++++++++++++++++++++++ docker/docker-compose.yml | 17 ++- docker/docker-entrypoint.sh | 2 +- docker/update-gemfile-lock.sh | 8 ++ docs/dev-setup.md | 236 -------------------------------- docs/vagrant.md | 45 ------ update-gemfile-lock.sh | 6 + 21 files changed, 506 insertions(+), 463 deletions(-) delete mode 100644 .github/workflows/lightstep.yml delete mode 100644 .lightstep.yml create mode 100644 Gemfile.lock delete mode 100644 docker/Dockerfile-aio delete mode 100644 docker/Dockerfile_local create mode 100644 docker/Gemfile create mode 100644 docker/Gemfile.lock create mode 100755 docker/update-gemfile-lock.sh delete mode 100644 docs/dev-setup.md delete mode 100644 docs/vagrant.md create mode 100755 update-gemfile-lock.sh diff --git a/.dockerignore b/.dockerignore index 1b242ef..a5157f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ **/*.md **/*example **/Dockerfile* -Gemfile.lock Rakefile Vagrantfile coverage diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8f8016..eb1db81 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,9 @@ updates: interval: daily time: "13:00" open-pull-requests-limit: 10 +- package-ecosystem: bundler + directory: "/docker/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 diff --git a/.github/workflows/lightstep.yml b/.github/workflows/lightstep.yml deleted file mode 100644 index 5e6a92f..0000000 --- a/.github/workflows/lightstep.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Verify Pre-Deploy Status - -on: pull_request - -jobs: - build: - runs-on: ubuntu-latest - - steps: - # Checkout repo - - name: Checkout - uses: actions/checkout@v2 - - # Run checks - - name: Lightstep Pre-Deploy Check - uses: lightstep/lightstep-action-predeploy@v0.2.6 - id: lightstep-predeploy - with: - lightstep_api_key: ${{ secrets.LIGHTSTEP_API_KEY }} - pagerduty_api_token: ${{ secrets.PAGERDUTY_API_TOKEN }} - - # Output status as a comment - - name: Add a Comment - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: ${{ steps.lightstep-predeploy.outputs.lightstep_predeploy_md }} - check_for_duplicate_msg: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5014b67..3d01a8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,10 +20,10 @@ jobs: draft: false prerelease: false generateReleaseNotes: true - - name: Install Ruby 2.5.8 + - name: Install Ruby jruby-9.2.12.0 uses: ruby/setup-ruby@v1 with: - ruby-version: '2.5.8' + ruby-version: 'jruby-9.2.12.0' - name: Build gem run: gem build *.gemspec - name: Publish gem diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 501403f..ec96927 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -18,7 +18,8 @@ jobs: strategy: matrix: ruby-version: - - '2.5.8' + # - '2.5.8' + - 'jruby-9.2.12.0' steps: - uses: actions/checkout@v2 - name: Set up Ruby @@ -34,7 +35,7 @@ jobs: strategy: matrix: ruby-version: - - '2.5.8' + # - '2.5.8' - 'jruby-9.2.12.0' steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 6117961..700a888 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,5 @@ vendor/ .dccache .ruby-version Gemfile.local -Gemfile.lock results.xml /vmpooler.yaml - diff --git a/.lightstep.yml b/.lightstep.yml deleted file mode 100644 index c3d2ebc..0000000 --- a/.lightstep.yml +++ /dev/null @@ -1,10 +0,0 @@ -organization: Puppet Inc -project: puppet-inc-prod -conditions: - # API /status error >0% - - SZcJVQfy -integrations: - pagerduty: - # VMPooler service - service: P714ID4 - diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..b46f84e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,194 @@ +PATH + remote: . + specs: + vmpooler (2.2.0) + concurrent-ruby (~> 1.1) + connection_pool (~> 2.2) + deep_merge (~> 1.2) + net-ldap (~> 0.16) + nokogiri (~> 1.10) + opentelemetry-exporter-jaeger (= 0.20.1) + opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) + opentelemetry-instrumentation-redis (= 0.21.2) + opentelemetry-instrumentation-sinatra (= 0.19.3) + opentelemetry-resource_detectors (= 0.19.1) + opentelemetry-sdk (~> 1.0, >= 1.0.2) + pickup (~> 0.0.11) + prometheus-client (~> 2.0) + puma (~> 5.0, >= 5.0.4) + rack (~> 2.2) + rake (~> 13.0) + redis (~> 4.1) + sinatra (~> 2.0) + spicy-proton (~> 2.1) + statsd-ruby (~> 1.4) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + bindata (2.4.10) + builder (3.2.4) + climate_control (1.0.1) + coderay (1.1.3) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) + deep_merge (1.2.2) + diff-lcs (1.5.0) + docile (1.4.0) + faraday (1.9.3) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.3) + multipart-post (>= 1.2, < 3) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.15.5-java) + google-cloud-env (1.5.0) + faraday (>= 0.17.3, < 2.0) + method_source (1.0.0) + mock_redis (0.29.0) + ruby2_keywords + multipart-post (2.1.1) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) + net-ldap (0.17.0) + nio4r (2.5.8-java) + nokogiri (1.12.5-java) + racc (~> 1.4) + opentelemetry-api (1.0.1) + opentelemetry-common (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-jaeger (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.2) + opentelemetry-sdk (~> 1.0) + thrift + opentelemetry-instrumentation-base (0.19.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-concurrent_ruby (0.19.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-redis (0.21.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-sinatra (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-resource_detectors (0.19.1) + google-cloud-env + opentelemetry-sdk + opentelemetry-sdk (1.0.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.8.0) + opentelemetry-api (~> 1.0) + parallel (1.21.0) + parser (3.1.0.0) + ast (~> 2.4.1) + pickup (0.0.11) + prometheus-client (2.1.0) + pry (0.14.1-java) + coderay (~> 1.1) + method_source (~> 1.0) + spoon (~> 0.0) + puma (5.5.2-java) + nio4r (~> 2.0) + racc (1.6.0-java) + rack (2.2.3) + rack-protection (2.1.0) + rack + rack-test (1.1.0) + rack (>= 1.0, < 3) + rainbow (3.1.1) + rake (13.0.6) + redis (4.5.1) + regexp_parser (2.2.0) + rexml (3.2.5) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.3) + rubocop (1.1.0) + parallel (~> 1.10) + parser (>= 2.7.1.5) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8) + rexml + rubocop-ast (>= 1.0.1) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 2.0) + rubocop-ast (1.15.1) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) + ruby2_keywords (0.0.5) + simplecov (0.21.2) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.3) + sinatra (2.1.0) + mustermann (~> 1.0) + rack (~> 2.2) + rack-protection (= 2.1.0) + tilt (~> 2.0) + spicy-proton (2.1.13) + bindata (~> 2.3) + spoon (0.0.6) + ffi + statsd-ruby (1.5.0) + thor (1.2.1) + thrift (0.15.0) + tilt (2.0.10) + unicode-display_width (1.8.0) + yarjuf (2.0.0) + builder + rspec (~> 3) + +PLATFORMS + universal-java-1.8 + +DEPENDENCIES + climate_control (>= 0.2.0) + mock_redis (>= 0.17.0) + pry + rack-test (>= 0.6) + rspec (>= 3.2) + rubocop (~> 1.1.0) + simplecov (>= 0.11.2) + thor (~> 1.0, >= 1.0.1) + vmpooler! + yarjuf (>= 2.0) + +BUNDLED WITH + 2.3.5 diff --git a/README.md b/README.md index a24d816..ece9e84 100644 --- a/README.md +++ b/README.md @@ -14,19 +14,23 @@ As of version 2.0.0, all providers other than the dummy one are now separate gem ## Installation -### Prerequisites - -VMPooler is available as a gem. To use the gem run `gem install vmpooler` or add it to your Gemfile and install via bundler. You will also need to install any needed providers in the same manner. +The recommended method of installation is via the Helm chart located in [puppetlabs/vmpooler-deployment](https://github.com/puppetlabs/vmpooler-deployment). That repository also provides Docker images of VMPooler. ### Dependencies +#### Redis + VMPooler requires a [Redis](http://redis.io/) server. This is the data store used for VMPooler's inventory and queuing services. -### Configuration +#### Other gems + +VMPooler itself and the dev environment talked about below require additional Ruby gems to function. You can update the currently required ones for VMPooler by running `./update-gemfile-lock.sh`. The gems for the dev environment can be updated by running `./docker/update-gemfile-lock.sh`. These scripts will utilize the container on the FROM line of the Dockerfile to update the Gemfile.lock in the root of this repo and in the docker folder, respectively. + +## Configuration Configuration for VMPooler may be provided via environment variables, or a configuration file. -#### Note on JRuby 9.2.11.x +### Note on JRuby 9.2.11.x We have found when running VMPooler on JRuby 9.2.11.x we occasionally encounter a stack overflow error that causes the pool\_manager application component to fail and stop doing work. To address this issue on JRuby 9.2.11.x we recommend setting the JRuby option `invokedynamic.yield=false`. To set this with JRuby 9.2.11.1 you can specify the environment variable `JRUBY_OPTS` with the value `-Xinvokedynamic.yield=false`. @@ -71,39 +75,7 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian- See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for additional configuration options and parameters or for supporting multiple providers. -### Running via Docker - -A [Dockerfile](/docker/Dockerfile) is included in this repository to allow running VMPooler inside a Docker container. A configuration file can be used via volume mapping, and specifying the destination as the configuration file via environment variables, or the application can be configured with environment variables alone. The Dockerfile provides an entrypoint so you may choose whether to run API, or manager services. The default behavior will run both. To build and run: - -```bash -docker build -t vmpooler . && docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler -``` - -To run only the API and dashboard: - -```bash -docker run -p 80:4567 -it vmpooler api -``` - -To run only the manager component: - -```bash -docker run -it vmpooler manager -``` - -### docker-compose - -A docker-compose file is provided to support running VMPooler easily via docker-compose. This is useful for development because your local code is used to build the gem used in the docker-compose environment. - -```bash -docker-compose -f docker/docker-compose.yml up -``` - -### Running Docker inside Vagrant - -A Vagrantfile is included in this repository. Please see [vagrant instructions](docs/vagrant.md) for details. - -## API and Dashboard +## Components VMPooler provides an API and web front-end (dashboard) on port `:4567`. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), to specify an alternative port to listen on. @@ -129,13 +101,45 @@ A dashboard is provided to offer real-time statistics and historical graphs. It - [vagrant-vmpooler](https://github.com/briancain/vagrant-vmpooler): Use Vagrant to create and manage your VMPooler instances. -## Development and further documentation +## Development + +### docker-compose + +A docker-compose file is provided to support running VMPooler and associated tools locally. This is useful for development because your local code is used to build the gem used in the docker-compose environment. The compose environment also pulls in the latest providers via git. Details of this setup are stored in the `docker/` folder. + +```bash +docker-compose -f docker/docker-compose.yml build && \ +docker-compose -f docker/docker-compose.yml up +``` + +### Running docker-compose inside Vagrant + +A Vagrantfile is included in this repository so as to provide a reproducible development environment. + +```bash +vagrant up +vagrant ssh +cd /vagrant +docker-compose -f docker/docker-compose.yml build && \ +docker-compose -f docker/docker-compose.yml up +``` + +The Vagrant environment also contains multiple rubies you can utilize for spec test and the like. You can see a list of the pre-installed ones when you log in as part of the message of the day. For more information about setting up a development instance of VMPooler or other subjects, see the [docs/](docs) directory. -### Build status +### URLs when using docker-compose -[![Testing](https://github.com/puppetlabs/vmpooler/actions/workflows/testing.yml/badge.svg)](https://github.com/puppetlabs/vmpooler/actions/workflows/testing.yml) +| Endpoint | URL | +|-------------------|-----------------------------------------------------------------------| +| Redis Commander | [http://localhost:8079](http://localhost:8079) | +| API | [http://localhost:8080/api/v1]([http://localhost:8080/api/v1) | +| Dashboard | [http://localhost:8080/dashboard/](http://localhost:8080/dashboard/) | +| Metrics (API) | [http://localhost:8080/prometheus]([http://localhost:8080/prometheus) | +| Metrics (Manager) | [http://localhost:8081/prometheus]([http://localhost:8081/prometheus) | +| Jaeger | [http://localhost:8082](http://localhost:8082) | + +Additionally, the Redis instance can be accessed at `localhost:6379`. ## License diff --git a/Vagrantfile b/Vagrantfile index aa667a6..38598e4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure("2") do |config| config.vm.box = "genebean/centos-7-rvm-multi" config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose + config.vm.network "forwarded_port", guest: 6379, host: 6379 # Redis + config.vm.network "forwarded_port", guest: 8079, host: 8079 # Redis Commander config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f1d15b..856f8fd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,9 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler +# This Dockerfile is intended to be used with the +# docker-compose file in the same directory. FROM jruby:9.2-jdk -ARG vmpooler_version=0.11.3 - -COPY docker/docker-entrypoint.sh /usr/local/bin/ - -ENV LOGFILE=/dev/stdout \ - RACK_ENV=production +ENV RACK_ENV=production RUN apt-get update -qq && \ apt-get install -y --no-install-recommends make && \ @@ -23,7 +11,16 @@ RUN apt-get update -qq && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* -RUN gem install vmpooler -v ${vmpooler_version} && \ - chmod +x /usr/local/bin/docker-entrypoint.sh +COPY docker/docker-entrypoint.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +COPY docker/Gemfile* ./ + +COPY ./ ./vmpooler-source + +RUN gem install bundler && \ + bundle config set --local jobs 3 && \ + bundle install ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/Dockerfile-aio b/docker/Dockerfile-aio deleted file mode 100644 index 48a62d6..0000000 --- a/docker/Dockerfile-aio +++ /dev/null @@ -1,39 +0,0 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler - -FROM jruby:9.2.9-jdk - -RUN mkdir -p /var/lib/vmpooler - -WORKDIR /var/lib/vmpooler - -RUN echo "deb http://httpredir.debian.org/debian jessie main" >/etc/apt/sources.list.d/jessie-main.list - -RUN apt-get update -qq && \ - apt-get install -y --no-install-recommends make redis-server && \ - apt-get clean autoclean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -ADD Gemfile* /var/lib/vmpooler/ - -RUN bundle install --system - -RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby - -COPY . /var/lib/vmpooler - -ENV VMPOOLER_LOG /var/log/vmpooler.log - -CMD \ - /etc/init.d/redis-server start \ - && /var/lib/vmpooler/scripts/vmpooler_init.sh start \ - && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \ - tail -f ${VMPOOLER_LOG} diff --git a/docker/Dockerfile_local b/docker/Dockerfile_local deleted file mode 100644 index 2480e16..0000000 --- a/docker/Dockerfile_local +++ /dev/null @@ -1,36 +0,0 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler - -FROM jruby:9.2-jdk - -ENV RACK_ENV=production - -RUN apt-get update -qq && \ - apt-get install -y --no-install-recommends make && \ - apt-get clean autoclean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -COPY docker/docker-entrypoint.sh /usr/local/bin/ -COPY ./Gemfile ./ -COPY ./vmpooler.gemspec ./ -COPY ./lib/vmpooler/version.rb ./lib/vmpooler/version.rb - -RUN gem install bundler && \ - bundle config set --local jobs 3 && \ - bundle install - -COPY ./ ./ - -RUN gem build vmpooler.gemspec && \ - gem install vmpooler*.gem && \ - chmod +x /usr/local/bin/docker-entrypoint.sh - -ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/Gemfile b/docker/Gemfile new file mode 100644 index 0000000..a616bbc --- /dev/null +++ b/docker/Gemfile @@ -0,0 +1,5 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gem 'vmpooler', path: './vmpooler-source' +gem 'vmpooler-provider-gce', git: 'https://github.com/puppetlabs/vmpooler-provider-gce.git' +gem 'vmpooler-provider-vsphere', git: 'https://github.com/puppetlabs/vmpooler-provider-vsphere.git' diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock new file mode 100644 index 0000000..c461ae9 --- /dev/null +++ b/docker/Gemfile.lock @@ -0,0 +1,205 @@ +GIT + remote: https://github.com/puppetlabs/vmpooler-provider-gce.git + revision: 614c474305f707ef52b4e3e92aabccec5880fba6 + specs: + vmpooler-provider-gce (0.1.2) + google-apis-compute_v1 (~> 0.14) + google-cloud-dns (~> 0.35.1) + googleauth (~> 0.16.2) + +GIT + remote: https://github.com/puppetlabs/vmpooler-provider-vsphere.git + revision: a9662622d5574a7f9f6e97ff4c8bc82786982266 + specs: + vmpooler-provider-vsphere (1.5.0) + rbvmomi (>= 2.1, < 4.0) + +PATH + remote: vmpooler-source + specs: + vmpooler (2.2.0) + concurrent-ruby (~> 1.1) + connection_pool (~> 2.2) + deep_merge (~> 1.2) + net-ldap (~> 0.16) + nokogiri (~> 1.10) + opentelemetry-exporter-jaeger (= 0.20.1) + opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) + opentelemetry-instrumentation-redis (= 0.21.2) + opentelemetry-instrumentation-sinatra (= 0.19.3) + opentelemetry-resource_detectors (= 0.19.1) + opentelemetry-sdk (~> 1.0, >= 1.0.2) + pickup (~> 0.0.11) + prometheus-client (~> 2.0) + puma (~> 5.0, >= 5.0.4) + rack (~> 2.2) + rake (~> 13.0) + redis (~> 4.1) + sinatra (~> 2.0) + spicy-proton (~> 2.1) + statsd-ruby (~> 1.4) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.0) + public_suffix (>= 2.0.2, < 5.0) + bindata (2.4.10) + builder (3.2.4) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) + declarative (0.0.20) + deep_merge (1.2.2) + faraday (1.9.3) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.3) + multipart-post (>= 1.2, < 3) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + google-apis-compute_v1 (0.23.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-core (0.4.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + rexml + webrick + google-apis-dns_v1 (0.17.0) + google-apis-core (>= 0.4, < 2.a) + google-cloud-core (1.6.0) + google-cloud-env (~> 1.0) + google-cloud-errors (~> 1.0) + google-cloud-dns (0.35.1) + google-apis-dns_v1 (~> 0.1) + google-cloud-core (~> 1.6) + googleauth (>= 0.16.2, < 2.a) + zonefile (~> 1.04) + google-cloud-env (1.5.0) + faraday (>= 0.17.3, < 2.0) + google-cloud-errors (1.2.0) + googleauth (0.16.2) + faraday (>= 0.17.3, < 2.0) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.14) + httpclient (2.8.3) + json (2.6.1-java) + jwt (2.3.0) + memoist (0.16.2) + mini_mime (1.1.2) + multi_json (1.15.0) + multipart-post (2.1.1) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) + net-ldap (0.17.0) + nio4r (2.5.8-java) + nokogiri (1.12.5-java) + racc (~> 1.4) + opentelemetry-api (1.0.1) + opentelemetry-common (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-jaeger (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.2) + opentelemetry-sdk (~> 1.0) + thrift + opentelemetry-instrumentation-base (0.19.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-concurrent_ruby (0.19.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-redis (0.21.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-sinatra (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-resource_detectors (0.19.1) + google-cloud-env + opentelemetry-sdk + opentelemetry-sdk (1.0.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.8.0) + opentelemetry-api (~> 1.0) + optimist (3.0.1) + os (1.1.4) + pickup (0.0.11) + prometheus-client (2.1.0) + public_suffix (4.0.6) + puma (5.5.2-java) + nio4r (~> 2.0) + racc (1.6.0-java) + rack (2.2.3) + rack-protection (2.1.0) + rack + rake (13.0.6) + rbvmomi (3.0.0) + builder (~> 3.2) + json (~> 2.3) + nokogiri (~> 1.10) + optimist (~> 3.0) + redis (4.5.1) + representable (3.1.1) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) + retriable (3.1.2) + rexml (3.2.5) + ruby2_keywords (0.0.5) + signet (0.16.0) + addressable (~> 2.8) + faraday (>= 0.17.3, < 2.0) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + sinatra (2.1.0) + mustermann (~> 1.0) + rack (~> 2.2) + rack-protection (= 2.1.0) + tilt (~> 2.0) + spicy-proton (2.1.13) + bindata (~> 2.3) + statsd-ruby (1.5.0) + thrift (0.15.0) + tilt (2.0.10) + trailblazer-option (0.1.2) + uber (0.1.0) + webrick (1.7.0) + zonefile (1.06) + +PLATFORMS + universal-java-1.8 + +DEPENDENCIES + vmpooler! + vmpooler-provider-gce! + vmpooler-provider-vsphere! + +BUNDLED WITH + 2.3.5 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 21b179d..917eb4a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,7 +4,7 @@ services: vmpooler-api: build: context: ../ - dockerfile: docker/Dockerfile_local + dockerfile: docker/Dockerfile volumes: - type: bind source: ${PWD}/vmpooler.yaml @@ -28,7 +28,7 @@ services: vmpooler-manager: build: context: ../ - dockerfile: docker/Dockerfile_local + dockerfile: docker/Dockerfile volumes: - type: bind source: ${PWD}/vmpooler.yaml @@ -57,6 +57,19 @@ services: - "6379:6379" networks: - redis-net + redis-commander: + container_name: redis-commander + hostname: redis-commander + image: rediscommander/redis-commander:latest + restart: always + environment: + - REDIS_HOSTS=local:redislocal:6379 + ports: + - "8079:8081" + networks: + - redis-net + depends_on: + - redislocal jaeger-aio: image: jaegertracing/all-in-one:1.18 ports: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 5da89df..07d11de 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -set -- vmpooler "$@" +set -- bundle exec vmpooler "$@" exec "$@" diff --git a/docker/update-gemfile-lock.sh b/docker/update-gemfile-lock.sh new file mode 100755 index 0000000..6fc8163 --- /dev/null +++ b/docker/update-gemfile-lock.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +docker run -it --rm \ + -v $(pwd)/docker:/app \ + -v $(pwd):/app/vmpooler-source \ + $(grep ^FROM docker/Dockerfile |cut -d ' ' -f2) \ + /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends make && cd /app && gem install bundler && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' +rmdir docker/vmpooler-source diff --git a/docs/dev-setup.md b/docs/dev-setup.md deleted file mode 100644 index 5999f09..0000000 --- a/docs/dev-setup.md +++ /dev/null @@ -1,236 +0,0 @@ -# Setting up a vmpooler development environment - -## Docker is the preferred development environment - -The docker compose file is the easiest way to get vmpooler running with any local code changes. The docker compose file expects to find a vmpooler.yaml configuration file in the root vmpooler directory. The file is mapped into the running container for the vmpooler application. This file primarily contains the pools configuration. Nearly all other configuration can be supplied with environment variables. - -## Requirements for local installation directly on your system (not recommended) - -* Supported on OSX, Windows and Linux - -* Ruby or JRuby - - Note - It is recommended to user Bundler instead of installing gems into the system repository - -* A local Redis server - - Either a containerized instance of Redis or a local version is fine. - -## Setup source and ruby - -* Clone repository, either from your own fork or the original source - -* Perform a bundle install - -``` -~/ > git clone https://github.com/puppetlabs/vmpooler.git -Cloning into 'vmpooler'... -remote: Counting objects: 3411, done. -... - -~/ > cd vmpooler - -~/vmpooler/ > bundle install -Fetching gem metadata from https://rubygems.org/......... -Fetching version metadata from https://rubygems.org/.. -Resolving dependencies... -Installing rake 12.0.0 -... -Bundle complete! 16 Gemfile dependencies, 37 gems now installed. -``` - -## Setup environment variables - -### `VMPOOLER_DEBUG` - -Setting the `VMPOOLER_DEBUG` environment variable will instruct vmpooler to: - -* Output log messages to STDOUT - -* Allow the use of the dummy authentication method - -* Add interrupt traps so you can stop vmpooler when run interactively - -Linux, OSX -```bash -~/vmpooler/ > export VMPOOLER_DEBUG=true -``` - -Windows (PowerShell) -```powershell -C:\vmpooler > $ENV:VMPOOLER_DEBUG = 'true' -``` - - -### `VMPOOLER_CONFIG` - -When `VMPOOLER_CONFIG` is set, vmpooler will read its configuration from the content of the environment variable. - -Note that this variable does not point to a different configuration file, but stores the contents of a configuration file. You may use `VMPOOLER_CONFIG_FILE` instead to specify a filename. - - -### `VMPOOLER_CONFIG_FILE` - -When `VMPOOLER_CONFIG_FILE` is set, vmpooler will read its configuration from the file specified in the environment variable. - -Note that this variable points to a different configuration file, unlike `VMPOOLER_CONFIG`. - - -## Setup vmpooler Configuration - -You can create a `vmpooler.yaml` file, set the `VMPOOLER_CONFIG` environment variable with the equivalent content, or set the `VMPOOLER_CONFIG_FILE` environment variable with the name of another configuration file to use. `VMPOOLER_CONFIG` takes precedence over `VMPOOLER_CONFIG_FILE`. - -Example minimal configuration file: -```yaml - ---- -:providers: - :dummy: - -:redis: - server: 'localhost' - -:auth: - provider: dummy - -:tagfilter: - url: '(.*)\/' - -:config: - site_name: 'vmpooler' - # Need to change this on Windows - logfile: '/var/log/vmpooler.log' - task_limit: 10 - timeout: 15 - vm_lifetime: 12 - vm_lifetime_auth: 24 - allowed_tags: - - 'created_by' - - 'project' - domain: 'example.com' - prefix: 'poolvm-' - -# Uncomment the lines below to suppress metrics to STDOUT -# :statsd: -# server: 'localhost' -# prefix: 'vmpooler' -# port: 8125 - -:pools: - - name: 'pool01' - size: 5 - provider: dummy - - name: 'pool02' - size: 5 - provider: dummy -``` - -## Running vmpooler locally - -* Run `bundle exec ruby vmpooler` - - If using JRuby, you may need to use `bundle exec jruby vmpooler` - -You should see output similar to: -``` -~/vmpooler/ > bundle exec ruby vmpooler -[2017-06-16 14:50:31] starting vmpooler -[2017-06-16 14:50:31] [!] Creating provider 'dummy' -[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection pool of size 1 with timeout 10 -[2017-06-16 14:50:31] [*] [disk_manager] starting worker thread -[2017-06-16 14:50:31] [*] [snapshot_manager] starting worker thread -[2017-06-16 14:50:31] [*] [pool01] starting worker thread -[2017-06-16 14:50:31] [*] [pool02] starting worker thread -[2017-06-16 14:50:31] [dummy] ConnPool - Creating a connection object ID 1784 -== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma -*** SIGUSR2 not implemented, signal based restart unavailable! -*** SIGUSR1 not implemented, signal based restart unavailable! -*** SIGHUP not implemented, signal based logs reopening unavailable! -Puma starting in single mode... -* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller -* Min threads: 0, max threads: 16 -* Environment: development -* Listening on tcp://0.0.0.0:4567 -Use Ctrl-C to stop -[2017-06-16 14:50:31] [!] [pool02] is empty -[2017-06-16 14:50:31] [!] [pool01] is empty -[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nexs1w50m4djap5' -[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-r543eibo4b6tjer' -[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-neqmu7wj7aukyjy' -[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-nsdnrhhy22lnemo' -[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-r543eibo4b6tjer' is being cloned from '' -[2017-06-16 14:50:31] [ ] [pool01] 'poolvm-neqmu7wj7aukyjy' is being cloned from '' -[2017-06-16 14:50:31] [ ] [pool02] 'poolvm-nexs1w50m4djap5' is being cloned from '' -[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-edzlp954lyiozli' -[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nb0uci0yrwbxr6x' -[2017-06-16 14:50:31] [ ] [pool02] Starting to clone 'poolvm-y2yxgnovaneymvy' -[2017-06-16 14:50:31] [ ] [pool01] Starting to clone 'poolvm-nur59d25s1y8jko' -... -``` - -### Common Errors - -* Forget to set VMPOOLER_DEBUG environment variable - -vmpooler will fail to start with an error similar to below -``` -~/vmpooler/ > bundle exec ruby vmpooler - -~/vmpooler/lib/vmpooler.rb:44:in `config': Dummy authentication should not be used outside of debug mode; please set environment variable VMPOOLER_DEBUG to 'true' if you want to use dummy authentication (RuntimeError) - from vmpooler:8:in `
' -... -``` - -* Error in vmpooler configuration - -If there is an error in the vmpooler configuration file, or any other fatal error in the Pool Manager, vmpooler will appear to be running but no log information is displayed. This is due to the error not being displayed until you press `Ctrl-C` and then suddenly you can see the cause of the issue. - -For example, when running vmpooler on Windows, but with a unix style filename for the vmpooler log - -```powershell -C:\vmpooler > bundle exec ruby vmpooler -[2017-06-16 14:49:57] starting vmpooler -== Sinatra (v1.4.8) has taken the stage on 4567 for production with backup from Puma -*** SIGUSR2 not implemented, signal based restart unavailable! -*** SIGUSR1 not implemented, signal based restart unavailable! -*** SIGHUP not implemented, signal based logs reopening unavailable! -Puma starting in single mode... -* Version 3.9.1 (ruby 2.3.1-p112), codename: Private Caller -* Min threads: 0, max threads: 16 -* Environment: development -* Listening on tcp://0.0.0.0:4567 -Use Ctrl-C to stop - -# [ NOTHING ELSE IS LOGGED ] -``` - -Once `Ctrl-C` is pressed the error is shown - -```powershell -... -== Sinatra has ended his set (crowd applauds) -Shutting down. -C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `initialize': No such file or directory @ rb_sysopen - /var/log/vmpooler.log (Errno::ENOENT) - from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open' - from C:/tools/ruby2.3.1x64/lib/ruby/2.3.0/open-uri.rb:37:in `open' - from C:/vmpooler/lib/vmpooler/logger.rb:17:in `log' - from C:/vmpooler/lib/vmpooler/pool_manager.rb:709:in `execute!' - from vmpooler:26:in `block in
' -``` - -## Default vmpooler URLs - -| Endpoint | URL | -|-----------|----------------------------------------------------------------------| -| Dashboard | [http://localhost:4567/dashboard/](http://localhost:4567/dashboard/) | -| API | [http://localhost:4567/api/v1]([http://localhost:4567/api/v1) | - -## Use the vmpooler API locally - -Once a local vmpooler instance is running you can use any tool you need to interact with the API. The dummy authentication provider will allow a user to connect if the username and password are not the same: - -* Authentication is successful for username `Alice` with password `foo` - -* Authentication will fail for username `Alice` with password `Alice` - -Like normal vmpooler, tokens will be created for the user and can be used for regular vmpooler operations. diff --git a/docs/vagrant.md b/docs/vagrant.md deleted file mode 100644 index 58332b9..0000000 --- a/docs/vagrant.md +++ /dev/null @@ -1,45 +0,0 @@ -A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer. -To use it run: - -``` -vagrant up -vagrant ssh -docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler -``` - -To run vmpooler with the example dummy provider you can replace the above docker command with this: - -``` -docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler -``` - -Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/). - -### Running directly in Vagrant - -You can also run vmpooler directly in the Vagrant box. To do so run this: - -``` -vagrant up -vagrant ssh -cd /vagrant - -# Do this if using the dummy provider -export VMPOOLER_DEBUG=true -cp vmpooler.yaml.dummy-example vmpooler.yaml - -# vmpooler needs a redis server. -sudo yum -y install redis -sudo systemctl start redis - -# Optional: Choose your ruby version or use jruby -# ruby 2.4.x is used by default -rvm list -rvm use jruby-9.1.7.0 - -gem install bundler -bundle install -bundle exec ruby vmpooler -``` - -When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/). diff --git a/update-gemfile-lock.sh b/update-gemfile-lock.sh new file mode 100755 index 0000000..528add0 --- /dev/null +++ b/update-gemfile-lock.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +docker run -it --rm \ + -v $(pwd):/app \ + $(grep ^FROM docker/Dockerfile |cut -d ' ' -f2) \ + /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends make && cd /app && gem install bundler && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' From 72c82cf084700ec34bd618f95be107a06c198ff5 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Thu, 20 Jan 2022 11:42:43 -0500 Subject: [PATCH 007/184] Add OTel HttpClient Instrumentation This will be useful for the GCE provider. --- Gemfile.lock | 5 +++++ docker/Gemfile.lock | 5 +++++ lib/vmpooler.rb | 2 ++ vmpooler.gemspec | 1 + 4 files changed, 13 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index b46f84e..b8fe64a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,6 +9,7 @@ PATH nokogiri (~> 1.10) opentelemetry-exporter-jaeger (= 0.20.1) opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) + opentelemetry-instrumentation-http_client (= 0.19.3) opentelemetry-instrumentation-redis (= 0.21.2) opentelemetry-instrumentation-sinatra (= 0.19.3) opentelemetry-resource_detectors (= 0.19.1) @@ -85,6 +86,10 @@ GEM opentelemetry-instrumentation-concurrent_ruby (0.19.2) opentelemetry-api (~> 1.0) opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-http_client (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) opentelemetry-instrumentation-redis (0.21.2) opentelemetry-api (~> 1.0) opentelemetry-common (~> 0.19.3) diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index c461ae9..2ebb516 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -25,6 +25,7 @@ PATH nokogiri (~> 1.10) opentelemetry-exporter-jaeger (= 0.20.1) opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) + opentelemetry-instrumentation-http_client (= 0.19.3) opentelemetry-instrumentation-redis (= 0.21.2) opentelemetry-instrumentation-sinatra (= 0.19.3) opentelemetry-resource_detectors (= 0.19.1) @@ -130,6 +131,10 @@ GEM opentelemetry-instrumentation-concurrent_ruby (0.19.2) opentelemetry-api (~> 1.0) opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-http_client (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) opentelemetry-instrumentation-redis (0.21.2) opentelemetry-api (~> 1.0) opentelemetry-common (~> 0.19.3) diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index c68e4ef..d666943 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -17,6 +17,7 @@ module Vmpooler # Dependencies for tracing require 'opentelemetry-instrumentation-concurrent_ruby' + require 'opentelemetry-instrumentation-http_client' require 'opentelemetry-instrumentation-redis' require 'opentelemetry-instrumentation-sinatra' require 'opentelemetry-sdk' @@ -265,6 +266,7 @@ module Vmpooler OpenTelemetry::SDK.configure do |c| c.use 'OpenTelemetry::Instrumentation::Sinatra' c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby' + c.use 'OpenTelemetry::Instrumentation::HttpClient' c.use 'OpenTelemetry::Instrumentation::Redis' c.add_span_processor(span_processor) diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 144ae0b..49e2375 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -24,6 +24,7 @@ Gem::Specification.new do |s| s.add_dependency 'nokogiri', '~> 1.10' s.add_dependency 'opentelemetry-exporter-jaeger', '= 0.20.1' s.add_dependency 'opentelemetry-instrumentation-concurrent_ruby', '= 0.19.2' + s.add_dependency 'opentelemetry-instrumentation-http_client', '= 0.19.3' s.add_dependency 'opentelemetry-instrumentation-redis', '= 0.21.2' s.add_dependency 'opentelemetry-instrumentation-sinatra', '= 0.19.3' s.add_dependency 'opentelemetry-resource_detectors', '= 0.19.1' From 2ad9b8c549edd60ac2647cde3ba107ed4866b920 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Wed, 30 Mar 2022 09:01:25 -0500 Subject: [PATCH 008/184] (maint) Fix deprecation warning for redis ruby library The syntax for pipelined has changed and the old syntax will be removed in v5.0.0 Fixing the syntax now since the block syntax has been supported for a while now. --- lib/vmpooler/api/helpers.rb | 18 ++++---- lib/vmpooler/pool_manager.rb | 86 ++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 0f216ab..60d1db3 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -147,12 +147,12 @@ module Vmpooler def export_tags(backend, hostname, tags) tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do - backend.pipelined do + backend.pipelined do |pipeline| tags.each_pair do |tag, value| next if value.nil? or value.empty? - backend.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value) - backend.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value) + pipeline.hset("vmpooler__vm__#{hostname}", "tag:#{tag}", value) + pipeline.hset("vmpooler__tag__#{Date.today}", "#{hostname}:#{tag}", value) end end end @@ -201,9 +201,9 @@ module Vmpooler tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do # using pipelined is much faster than querying each of the pools and adding them # as we get the result. - res = backend.pipelined do + res = backend.pipelined do |pipeline| pools.each do |pool| - backend.scard(key + pool['name']) + pipeline.scard(key + pool['name']) end end res.inject(0) { |m, x| m + x }.to_i @@ -217,9 +217,9 @@ module Vmpooler # using pipelined is much faster than querying each of the pools and adding them # as we get the result. temp_hash = {} - res = backend.pipelined do + res = backend.pipelined do |pipeline| pools.each do |pool| - backend.scard(key + pool['name']) + pipeline.scard(key + pool['name']) end end pools.each_with_index do |pool, i| @@ -236,9 +236,9 @@ module Vmpooler # using pipelined is much faster than querying each of the pools and adding them # as we get the result. temp_hash = {} - res = backend.pipelined do + res = backend.pipelined do |pipeline| pools.each do |pool| - backend.hget(key, pool['name']) + pipeline.hget(key, pool['name']) end end pools.each_with_index do |pool, i| diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 69c3a18..2ca99af 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -148,15 +148,15 @@ module Vmpooler end pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') - redis.pipelined do - redis.hset("vmpooler__active__#{pool}", vm, Time.now) - redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) + redis.pipelined do |pipeline| + pipeline.hset("vmpooler__active__#{pool}", vm, Time.now) + pipeline.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) if ondemandrequest_hash['token:token'] - redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) - redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) - redis.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i) + pipeline.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) + pipeline.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) + pipeline.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i) end - redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm) + pipeline.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm) end move_vm_queue(pool, vm, 'pending', 'running', redis) check_ondemand_request_ready(request_id, redis) @@ -164,12 +164,12 @@ module Vmpooler redis.smove("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm) end - redis.pipelined do - redis.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself? - redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now) + redis.pipelined do |pipeline| + pipeline.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself? + pipeline.hset("vmpooler__vm__#{vm}", 'ready', Time.now) # last boot time is displayed in API, and used by alarming script - redis.hset('vmpooler__lastboot', pool, Time.now) + pipeline.hset('vmpooler__lastboot', pool, Time.now) end $metrics.timing("time_to_ready_state.#{pool}", finish) @@ -418,9 +418,9 @@ module Vmpooler finish = format('%