From cdde7d1122b9a02fa4ba271572ed11a193a7a42e Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 23 Feb 2015 17:32:34 -0800 Subject: [PATCH 1/5] (QENG-1873) Match status endpoint These changes are to match the status endpoints JSON payload. This also introduces both clone timings and information on total clones. The field "day" is renamed to "date" to accomodate when from/to will include times. --- lib/vmpooler/api.rb | 70 +++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 442fbc5..ec0996b 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -35,6 +35,11 @@ module Vmpooler def validate_date_str(date_str) /^\d{4}-\d{2}-\d{2}$/ === date_str end + + def mean(list) + s = list.map(&:to_f).reduce(:+).to_f + (s > 0 && list.length > 0) ? s / list.length.to_f : 0 + end end get '/' do @@ -205,8 +210,17 @@ module Vmpooler get '/summary/?' do result = { - clone_total: 0, - clone_average: 0, + clone: { + duration: { + average: 0, + min: 0, + max: 0 + }, + average: 0, + min: 0, + max: 0, + total: 0 + }, daily: [] # used for daily info } @@ -235,36 +249,54 @@ module Vmpooler # total clone time for entire duration requested. total_clone_time = 0 + min_max_clone_times = [] # min/max clone times from each day. + total_clones_per_day = [] # total clones from each day # generate sequence/range for from_date to to_date (inclusive). # for each date, calculate the day's clone total & average, as well # as add to total/overall values. (from_date..to_date).each do |date| me = { - day: date.to_s, - clone_total: $redis.hlen('vmpooler__clone__' + date.to_s), - clone_average: 0 + date: date.to_s, + clone: { + average: 0, + min: 0, + max: 0, + total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i, + } } - result[:clone_total] += me[:clone_total] + result[:clone][:total] += me[:clone][:total] + total_clones_per_day.push(me[:clone][:total]) - clone_time = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f).reduce(:+) - total_clone_time += clone_time.to_f + clone_times = $redis.hvals('vmpooler__clone__' + date.to_s) - # if clone_total > 0, calculate clone_average. - # this prevents divide by 0 problems. - if me[:clone_total] > 0 - me[:clone_average] = clone_time / me[:clone_total] - else - me[:clone_average] = 0 + unless clone_times.nil? or clone_times.empty? + clone_time = clone_times.map(&:to_f).reduce(:+).to_f + total_clone_time += clone_time + + me[:clone][:min], me[:clone][:max] = clone_times.minmax + min_max_clone_times.push(me[:clone][:min]) + min_max_clone_times.push(me[:clone][:max]) + + # if clone_total > 0, calculate clone_average. + # this prevents divide by 0 problems. + if me[:clone][:total] > 0 + me[:clone][:average] = clone_time / me[:clone][:total] + else + me[:clone][:average] = 0 + end + + # add to daily array + result[:daily].push(me) end - - # add to daily array - result[:daily].push(me) end # again, calc clone_average if we had clones. - if result[:clone_total] > 0 - result[:clone_average] = total_clone_time / result[:clone_total] + if result[:clone][:total] > 0 + result[:clone][:timings][:average] = total_clone_time / result[:clone][:total] + result[:clone][:timings][:min], result[:clone][:timings][:max] = min_max_clone_times.minmax + result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax + result[:clone][:average] = mean(total_clones_per_day) end content_type :json From ee55720b42b27d704d4d999a400d78de3eb0d521 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 08:52:33 -0800 Subject: [PATCH 2/5] (QENG-1873) Fix Key Name Issue And minmax This commit fixes the problem of lingering :timings that were renamed to :duration. This should resolve a NilClass failure. This also converts string results from redis to float so that future minmax calls will treat them as numerics instead of strings. --- lib/vmpooler/api.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index ec0996b..222302a 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -268,10 +268,10 @@ module Vmpooler result[:clone][:total] += me[:clone][:total] total_clones_per_day.push(me[:clone][:total]) - clone_times = $redis.hvals('vmpooler__clone__' + date.to_s) + clone_times = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) unless clone_times.nil? or clone_times.empty? - clone_time = clone_times.map(&:to_f).reduce(:+).to_f + clone_time = clone_times.reduce(:+).to_f total_clone_time += clone_time me[:clone][:min], me[:clone][:max] = clone_times.minmax @@ -293,8 +293,8 @@ module Vmpooler # again, calc clone_average if we had clones. if result[:clone][:total] > 0 - result[:clone][:timings][:average] = total_clone_time / result[:clone][:total] - result[:clone][:timings][:min], result[:clone][:timings][:max] = min_max_clone_times.minmax + result[:clone][:duration][:average] = total_clone_time / result[:clone][:total] + result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax result[:clone][:average] = mean(total_clones_per_day) end From b627f4779632e4a89a9dc3d38157baaab0249b61 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:23:58 -0800 Subject: [PATCH 3/5] (QENG-1873) Add count Section to summary Result This adds a :count key to both the overall and daily results. The payload is now divided in to :duration and :count values for summary data. The daily results do not include anything under :count except the total number of VMs cloned for the day. The overall :count includes total, including minimum, maximum, and average VMs cloned for the date range requested. --- lib/vmpooler/api.rb | 70 ++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 222302a..9ce012c 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -214,12 +214,15 @@ module Vmpooler duration: { average: 0, min: 0, - max: 0 + max: 0, + total: 0 }, - average: 0, - min: 0, - max: 0, - total: 0 + count: { + average: 0, + min: 0, + max: 0, + total: 0 + } }, daily: [] # used for daily info } @@ -248,9 +251,9 @@ module Vmpooler end # total clone time for entire duration requested. - total_clone_time = 0 min_max_clone_times = [] # min/max clone times from each day. total_clones_per_day = [] # total clones from each day + total_clone_dur_day = [] # total clone durations from each day # generate sequence/range for from_date to to_date (inclusive). # for each date, calculate the day's clone total & average, as well @@ -259,44 +262,59 @@ module Vmpooler me = { date: date.to_s, clone: { - average: 0, - min: 0, - max: 0, - total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i, + duration: { + average: 0, + min: 0, + max: 0, + total: 0 + }, + count: { + total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i + } } } - result[:clone][:total] += me[:clone][:total] - total_clones_per_day.push(me[:clone][:total]) + # add to daily clone count array + total_clones_per_day.push(me[:clone][:count][:total]) + + # fetch clone times from redis for this date. convert the results + # to float (numeric). clone_times = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) - unless clone_times.nil? or clone_times.empty? + unless clone_times.nil? or clone_times.length == 0 clone_time = clone_times.reduce(:+).to_f - total_clone_time += clone_time + # add clone time for this date for later processing + total_clone_dur_day.push(clone_time) - me[:clone][:min], me[:clone][:max] = clone_times.minmax - min_max_clone_times.push(me[:clone][:min]) - min_max_clone_times.push(me[:clone][:max]) + me[:clone][:duration][:total] = clone_time + + # min and max clone durations for the day. + mi, ma = clone_times.minmax + me[:clone][:duration][:min], me[:clone][:duration][:max] = mi, ma + min_max_clone_times.push(mi, ma) # if clone_total > 0, calculate clone_average. # this prevents divide by 0 problems. - if me[:clone][:total] > 0 - me[:clone][:average] = clone_time / me[:clone][:total] + if me[:clone][:count][:total] > 0 + me[:clone][:duration][:average] = clone_time / me[:clone][:duration][:total] else - me[:clone][:average] = 0 + me[:clone][:duration][:average] = 0 end - # add to daily array - result[:daily].push(me) end + + # add to daily array + result[:daily].push(me) end # again, calc clone_average if we had clones. - if result[:clone][:total] > 0 - result[:clone][:duration][:average] = total_clone_time / result[:clone][:total] + unless total_clones_per_day.nil? or total_clones_per_day.length == 0 + result[:clone][:duration][:average] = result[:clone][:duration][:total] / result[:clone][:count][:total] result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax - result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax - result[:clone][:average] = mean(total_clones_per_day) + result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f + result[:clone][:count][:min], result[:clone][:count][:max] = total_clones_per_day.minmax + result[:clone][:count][:average] = mean(total_clones_per_day) + result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i end content_type :json From cc1e0e10759a78b809672c72f3cd0ed638a18adb Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:26:50 -0800 Subject: [PATCH 4/5] (QENG-1873) Fix /0 Error Calculate total count before trying to use it in average calculation. --- lib/vmpooler/api.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 9ce012c..96306f2 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -309,12 +309,15 @@ module Vmpooler # again, calc clone_average if we had clones. unless total_clones_per_day.nil? or total_clones_per_day.length == 0 + # totals + result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i + result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f + + # averages and other things. result[:clone][:duration][:average] = result[:clone][:duration][:total] / result[:clone][:count][:total] result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax - result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f result[:clone][:count][:min], result[:clone][:count][:max] = total_clones_per_day.minmax result[:clone][:count][:average] = mean(total_clones_per_day) - result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i end content_type :json From 65493b1c7fa1ebea2408ddb47ddbd24e880d0079 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:32:02 -0800 Subject: [PATCH 5/5] (QENG-1873) Fix Average Duration Calculation Use the mean helper function to calculate average clone duration. --- lib/vmpooler/api.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 96306f2..2e2e693 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -296,7 +296,7 @@ module Vmpooler # if clone_total > 0, calculate clone_average. # this prevents divide by 0 problems. if me[:clone][:count][:total] > 0 - me[:clone][:duration][:average] = clone_time / me[:clone][:duration][:total] + me[:clone][:duration][:average] = mean(clone_times) else me[:clone][:duration][:average] = 0 end