From 4f880497b7b927973a525b38300a0a9dbc82fc45 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:15:04 -0800 Subject: [PATCH 1/9] Alphabetize existing helpers --- lib/vmpooler/api/v1.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index f47ed9c..3bb91cd 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -13,14 +13,14 @@ module Vmpooler hostname end - def validate_date_str(date_str) - /^\d{4}-\d{2}-\d{2}$/ === date_str - end - def mean(list) s = list.map(&:to_f).reduce(:+).to_f (s > 0 && list.length > 0) ? s / list.length.to_f : 0 end + + def validate_date_str(date_str) + /^\d{4}-\d{2}-\d{2}$/ === date_str + end end get "#{api_prefix}/status/?" do From c1099a82731f9a66003618885d6f34a0da4c4220 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:21:06 -0800 Subject: [PATCH 2/9] Add 'get_clone_times' helper --- lib/vmpooler/api/v1.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 3bb91cd..ad32828 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -5,6 +5,10 @@ module Vmpooler api_prefix = "/api/v#{api_version}" helpers do + def get_clone_times(date) + $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) + end + def hostname_shorten(hostname) if $config[:config]['domain'] && hostname =~ /^\w+\.#{$config[:config]['domain']}$/ hostname = hostname[/[^\.]+/] @@ -88,7 +92,7 @@ module Vmpooler result[:clone][:count][:total] = $redis.hlen('vmpooler__clone__' + Date.today.to_s).to_i if result[:clone][:count][:total] > 0 - clone_times = $redis.hvals('vmpooler__clone__' + Date.today.to_s).map(&:to_f) + clone_times = get_clone_times(Date.today) result[:clone][:duration][:average] = (clone_times.reduce(:+).to_f / result[:clone][:count][:total]).round(1) result[:clone][:duration][:min], result[:clone][:duration][:max] = clone_times.minmax @@ -174,7 +178,7 @@ module Vmpooler # fetch clone times from redis for this date. convert the results # to float (numeric). - clone_times = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) + clone_times = get_clone_times(date) unless clone_times.nil? or clone_times.length == 0 clone_time = clone_times.reduce(:+).to_f From 0c3f84e07256e95fac80c56e02e9b3ca622c76b2 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:31:56 -0800 Subject: [PATCH 3/9] Slight rework of data param validation --- lib/vmpooler/api/v1.rb | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index ad32828..2ea377a 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -124,29 +124,22 @@ module Vmpooler daily: [] # used for daily info } - from_param, to_param = params[:from], params[:to] + from_param = params[:from] || Date.today.to_s + to_param = params[:to] || Date.today.to_s - # Check date formats. - # It is ok if to_param is nil, we will assume this means 'today' - if to_param.nil? - to_param = Date.today.to_s - elsif !validate_date_str(to_param.to_s) - halt 400, 'Invalid "to" date format, must match YYYY-MM-DD' - end - - # It is also ok if from_param is nil, we will assume this also means 'today' - if from_param.nil? - from_param = Date.today.to_s - elsif !validate_date_str(from_param.to_s) - halt 400, 'Invalid "from" date format, must match YYYY-MM-DD' + # Validate date formats + [from_param, to_param].each do |param| + if !validate_date_str(param.to_s) + halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." + end end from_date, to_date = Date.parse(from_param), Date.parse(to_param) if to_date < from_date - halt 400, 'Date range is invalid, "to" cannot come before "from".' + halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' elsif from_date > Date.today - halt 400, 'Date range is invalid, "from" must be in the past.' + halt 400, 'Date range is invalid, \'from\' must be in the past.' end # total clone time for entire duration requested. From fd0e5cb438bbdad56007180fa79943ab3564349d Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:41:10 -0800 Subject: [PATCH 4/9] Move queue metric-gathering to 'get_queue_metrics' helper --- lib/vmpooler/api/v1.rb | 47 +++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 2ea377a..5ccc446 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -9,6 +9,32 @@ module Vmpooler $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) end + def get_queue_metrics() + queue = { + pending: 0, + cloning: 0, + booting: 0, + ready: 0, + running: 0, + completed: 0, + total: 0 + } + + $config[:pools].each do |pool| + queue[:pending] += $redis.scard('vmpooler__pending__' + pool['name']).to_i + queue[:ready] += $redis.scard('vmpooler__ready__' + pool['name']).to_i + queue[:running] += $redis.scard('vmpooler__running__' + pool['name']).to_i + queue[:completed] += $redis.scard('vmpooler__completed__' + pool['name']).to_i + end + + queue[:cloning] = $redis.get('vmpooler__tasks__clone').to_i + queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i + queue[:booting] = 0 if queue[:booting] < 0 + queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i + + queue + end + def hostname_shorten(hostname) if $config[:config]['domain'] && hostname =~ /^\w+\.#{$config[:config]['domain']}$/ hostname = hostname[/[^\.]+/] @@ -49,18 +75,11 @@ module Vmpooler count: { total: 0 } - }, - queue: { - pending: 0, - cloning: 0, - booting: 0, - ready: 0, - running: 0, - completed: 0, - total: 0 } } + result[:queue] = get_queue_metrics() + $config[:pools].each do |pool| pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i @@ -71,11 +90,6 @@ module Vmpooler result[:status][:empty] ||= [] result[:status][:empty].push(pool['name']) end - - result[:queue][:pending] += $redis.scard('vmpooler__pending__' + pool['name']).to_i - result[:queue][:ready] += $redis.scard('vmpooler__ready__' + pool['name']).to_i - result[:queue][:running] += $redis.scard('vmpooler__running__' + pool['name']).to_i - result[:queue][:completed] += $redis.scard('vmpooler__completed__' + pool['name']).to_i end if result[:status][:empty] @@ -85,11 +99,6 @@ module Vmpooler result[:capacity][:percent] = ((result[:capacity][:current].to_f / result[:capacity][:total].to_f) * 100.0).round(1) if result[:capacity][:total] > 0 - result[:queue][:cloning] = $redis.get('vmpooler__tasks__clone').to_i - result[:queue][:booting] = result[:queue][:pending].to_i - result[:queue][:cloning].to_i - result[:queue][:booting] = 0 if result[:queue][:booting] < 0 - result[:queue][:total] = result[:queue][:pending].to_i + result[:queue][:ready].to_i + result[:queue][:running].to_i + result[:queue][:completed].to_i - result[:clone][:count][:total] = $redis.hlen('vmpooler__clone__' + Date.today.to_s).to_i if result[:clone][:count][:total] > 0 clone_times = get_clone_times(Date.today) From 1397ff93c17bd3bedfe2fbb7a62040d90319aef1 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:49:16 -0800 Subject: [PATCH 5/9] Move capacity metric-gathering to 'get_capacity_metrics' helper --- lib/vmpooler/api/v1.rb | 45 +++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 5ccc446..989a1e1 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -5,6 +5,27 @@ module Vmpooler api_prefix = "/api/v#{api_version}" helpers do + def get_capacity_metrics() + capacity = { + current: 0, + total: 0, + percent: 0 + } + + $config[:pools].each do |pool| + pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i + + capacity[:current] += pool['capacity'] + capacity[:total] += pool['size'].to_i + end + + if capacity[:total] > 0 + capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1) + end + + capacity + end + def get_clone_times(date) $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) end @@ -61,11 +82,6 @@ module Vmpooler ok: true, message: 'Battle station fully armed and operational.' }, - capacity: { - current: 0, - total: 0, - percent: 0 - }, clone: { duration: { average: 0, @@ -78,27 +94,20 @@ module Vmpooler } } + result[:capacity] = get_capacity_metrics() result[:queue] = get_queue_metrics() + # Check for empty pools $config[:pools].each do |pool| - pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i - - result[:capacity][:current] += pool['capacity'] - result[:capacity][:total] += pool['size'].to_i - - if (pool['capacity'] == 0) + if $redis.scard('vmpooler__ready__' + pool['name']).to_i == 0 result[:status][:empty] ||= [] result[:status][:empty].push(pool['name']) + + result[:status][:ok] = false + result[:status][:message] = "Found #{result[:status][:empty].length} empty pools." end end - if result[:status][:empty] - result[:status][:ok] = false - result[:status][:message] = "Found #{result[:status][:empty].length} empty pools." - end - - result[:capacity][:percent] = ((result[:capacity][:current].to_f / result[:capacity][:total].to_f) * 100.0).round(1) if result[:capacity][:total] > 0 - result[:clone][:count][:total] = $redis.hlen('vmpooler__clone__' + Date.today.to_s).to_i if result[:clone][:count][:total] > 0 clone_times = get_clone_times(Date.today) From 201897214b2a450777bc7d46f42e6656205c0acc Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 09:58:11 -0800 Subject: [PATCH 6/9] Move clone metric-gathering to 'get_clone_metrics' helper --- lib/vmpooler/api/v1.rb | 43 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 989a1e1..0bad34a 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -26,6 +26,30 @@ module Vmpooler capacity end + def get_clone_metrics(date) + clone = { + duration: { + average: 0, + min: 0, + max: 0 + }, + count: { + total: 0 + } + } + + clone[:count][:total] = $redis.hlen('vmpooler__clone__' + date).to_i + + if clone[:count][:total] > 0 + clone_times = get_clone_times(date) + + clone[:duration][:average] = (clone_times.reduce(:+).to_f / clone[:count][:total]).round(1) + clone[:duration][:min], clone[:duration][:max] = clone_times.minmax + end + + clone + end + def get_clone_times(date) $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) end @@ -81,21 +105,12 @@ module Vmpooler status: { ok: true, message: 'Battle station fully armed and operational.' - }, - clone: { - duration: { - average: 0, - min: 0, - max: 0 - }, - count: { - total: 0 - } } } result[:capacity] = get_capacity_metrics() result[:queue] = get_queue_metrics() + result[:clone] = get_clone_metrics(Date.today.to_s) # Check for empty pools $config[:pools].each do |pool| @@ -108,14 +123,6 @@ module Vmpooler end end - result[:clone][:count][:total] = $redis.hlen('vmpooler__clone__' + Date.today.to_s).to_i - if result[:clone][:count][:total] > 0 - clone_times = get_clone_times(Date.today) - - result[:clone][:duration][:average] = (clone_times.reduce(:+).to_f / result[:clone][:count][:total]).round(1) - result[:clone][:duration][:min], result[:clone][:duration][:max] = clone_times.minmax - end - result[:status][:uptime] = (Time.now - $config[:uptime]).round(1) if $config[:uptime] JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) From 5453ec6206fcc910f2d40bc93d845009589ed37f Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 10:28:19 -0800 Subject: [PATCH 7/9] Use new helper methods in /summary route --- lib/vmpooler/api/v1.rb | 92 +++++++++++------------------------------- 1 file changed, 24 insertions(+), 68 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 0bad34a..4af76a3 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -31,7 +31,8 @@ module Vmpooler duration: { average: 0, min: 0, - max: 0 + max: 0, + total: 0 }, count: { total: 0 @@ -43,7 +44,8 @@ module Vmpooler if clone[:count][:total] > 0 clone_times = get_clone_times(date) - clone[:duration][:average] = (clone_times.reduce(:+).to_f / clone[:count][:total]).round(1) + clone[:duration][:total] = clone_times.reduce(:+).to_f + clone[:duration][:average] = (clone[:duration][:total] / clone[:count][:total]).round(1) clone[:duration][:min], clone[:duration][:max] = clone_times.minmax end @@ -146,7 +148,7 @@ module Vmpooler total: 0 } }, - daily: [] # used for daily info + daily: [] } from_param = params[:from] || Date.today.to_s @@ -167,81 +169,35 @@ module Vmpooler halt 400, 'Date range is invalid, \'from\' must be in the past.' end - # total clone time for entire duration requested. - min_max_clone_times = [] # min/max clone times from each day. - total_clones_per_day = [] # total clones from each day - total_clone_dur_day = [] # total clone durations from each day - - # generate sequence/range for from_date to to_date (inclusive). - # for each date, calculate the day's clone total & average, as well - # as add to total/overall values. (from_date..to_date).each do |date| - me = { - date: date.to_s, - clone: { - duration: { - average: 0, - min: 0, - max: 0, - total: 0 - }, - count: { - total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i - } - } + daily = { + date: date.to_s, + clone: get_clone_metrics(date.to_s) } - # add to daily clone count array - total_clones_per_day.push(me[:clone][:count][:total]) - - # fetch clone times from redis for this date. convert the results - # to float (numeric). - clone_times = get_clone_times(date) - - unless clone_times.nil? or clone_times.length == 0 - clone_time = clone_times.reduce(:+).to_f - # add clone time for this date for later processing - total_clone_dur_day.push(clone_time) - - me[:clone][:duration][:total] = clone_time - - # min and max clone durations for the day. - mi, ma = clone_times.minmax - me[:clone][:duration][:min], me[:clone][:duration][:max] = mi, ma - min_max_clone_times.push(mi, ma) - - # if clone_total > 0, calculate clone_average. - # this prevents divide by 0 problems. - if me[:clone][:count][:total] > 0 - me[:clone][:duration][:average] = mean(clone_times) - else - me[:clone][:duration][:average] = 0 - end - end - - # add to daily array - result[:daily].push(me) + result[:daily].push(daily) end - # again, calc clone_average if we had clones. - unless total_clones_per_day.nil? or total_clones_per_day.length == 0 - # totals - result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i - result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f + daily_clone_counts = [] + daily_clone_durations = [] - # averages and other things. - if result[:clone][:count][:total] != 0 + result[:daily].each do |daily| + daily_clone_counts.push(daily[:clone][:count][:total]) + daily_clone_durations.push(daily[:clone][:duration][:min]) + daily_clone_durations.push(daily[:clone][:duration][:max]) + + result[:clone][:count][:total] += daily[:clone][:count][:total] + result[:clone][:duration][:total] += daily[:clone][:duration][:total] + + if result[:clone][:count][:total] > 0 result[:clone][:duration][:average] = result[:clone][:duration][:total] / result[:clone][:count][:total] end - - if min_max_clone_times.length > 0 - result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax - end - - result[:clone][:count][:min], result[:clone][:count][:max] = total_clones_per_day.minmax - result[:clone][:count][:average] = mean(total_clones_per_day) end + result[:clone][:count][:min], result[:clone][:count][:max] = daily_clone_counts.minmax + result[:clone][:count][:average] = mean(daily_clone_counts) + result[:clone][:duration][:min], result[:clone][:duration][:max] = daily_clone_durations.minmax + JSON.pretty_generate(result) end From 35e78aacb524af263a7139334509590fd7ce65c0 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 11:12:23 -0800 Subject: [PATCH 8/9] Use 'date_str' instead of 'date' --- lib/vmpooler/api/v1.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 4af76a3..92e3a09 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -26,7 +26,7 @@ module Vmpooler capacity end - def get_clone_metrics(date) + def get_clone_metrics(date_str) clone = { duration: { average: 0, @@ -39,10 +39,10 @@ module Vmpooler } } - clone[:count][:total] = $redis.hlen('vmpooler__clone__' + date).to_i + clone[:count][:total] = $redis.hlen('vmpooler__clone__' + date_str).to_i if clone[:count][:total] > 0 - clone_times = get_clone_times(date) + clone_times = get_clone_times(date_str) clone[:duration][:total] = clone_times.reduce(:+).to_f clone[:duration][:average] = (clone[:duration][:total] / clone[:count][:total]).round(1) @@ -52,8 +52,8 @@ module Vmpooler clone end - def get_clone_times(date) - $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) + def get_clone_times(date_str) + $redis.hvals('vmpooler__clone__' + date_str).map(&:to_f) end def get_queue_metrics() From e347e2b6b4337d78dfc9dab908e09584a0bb3c5a Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 4 Mar 2015 11:28:31 -0800 Subject: [PATCH 9/9] Don't include daily_clone durations in average if nothing was cloned --- lib/vmpooler/api/v1.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 92e3a09..5189d8d 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -183,8 +183,11 @@ module Vmpooler result[:daily].each do |daily| daily_clone_counts.push(daily[:clone][:count][:total]) - daily_clone_durations.push(daily[:clone][:duration][:min]) - daily_clone_durations.push(daily[:clone][:duration][:max]) + + if daily[:clone][:count][:total] > 0 + daily_clone_durations.push(daily[:clone][:duration][:min]) + daily_clone_durations.push(daily[:clone][:duration][:max]) + end result[:clone][:count][:total] += daily[:clone][:count][:total] result[:clone][:duration][:total] += daily[:clone][:duration][:total]