From ef333ca65f56387d051733f08783fdde3417e5aa Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 23 Feb 2015 17:32:34 -0800 Subject: [PATCH 1/5] (QENG-1873) Match status endpoint These changes are to match the status endpoints JSON payload. This also introduces both clone timings and information on total clones. The field "day" is renamed to "date" to accomodate when from/to will include times. --- lib/vmpooler/api.rb | 70 +++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 442fbc5..ec0996b 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -35,6 +35,11 @@ module Vmpooler def validate_date_str(date_str) /^\d{4}-\d{2}-\d{2}$/ === date_str end + + def mean(list) + s = list.map(&:to_f).reduce(:+).to_f + (s > 0 && list.length > 0) ? s / list.length.to_f : 0 + end end get '/' do @@ -205,8 +210,17 @@ module Vmpooler get '/summary/?' do result = { - clone_total: 0, - clone_average: 0, + clone: { + duration: { + average: 0, + min: 0, + max: 0 + }, + average: 0, + min: 0, + max: 0, + total: 0 + }, daily: [] # used for daily info } @@ -235,36 +249,54 @@ module Vmpooler # total clone time for entire duration requested. total_clone_time = 0 + min_max_clone_times = [] # min/max clone times from each day. + total_clones_per_day = [] # total clones from each day # generate sequence/range for from_date to to_date (inclusive). # for each date, calculate the day's clone total & average, as well # as add to total/overall values. (from_date..to_date).each do |date| me = { - day: date.to_s, - clone_total: $redis.hlen('vmpooler__clone__' + date.to_s), - clone_average: 0 + date: date.to_s, + clone: { + average: 0, + min: 0, + max: 0, + total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i, + } } - result[:clone_total] += me[:clone_total] + result[:clone][:total] += me[:clone][:total] + total_clones_per_day.push(me[:clone][:total]) - clone_time = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f).reduce(:+) - total_clone_time += clone_time.to_f + clone_times = $redis.hvals('vmpooler__clone__' + date.to_s) - # if clone_total > 0, calculate clone_average. - # this prevents divide by 0 problems. - if me[:clone_total] > 0 - me[:clone_average] = clone_time / me[:clone_total] - else - me[:clone_average] = 0 + unless clone_times.nil? or clone_times.empty? + clone_time = clone_times.map(&:to_f).reduce(:+).to_f + total_clone_time += clone_time + + me[:clone][:min], me[:clone][:max] = clone_times.minmax + min_max_clone_times.push(me[:clone][:min]) + min_max_clone_times.push(me[:clone][:max]) + + # if clone_total > 0, calculate clone_average. + # this prevents divide by 0 problems. + if me[:clone][:total] > 0 + me[:clone][:average] = clone_time / me[:clone][:total] + else + me[:clone][:average] = 0 + end + + # add to daily array + result[:daily].push(me) end - - # add to daily array - result[:daily].push(me) end # again, calc clone_average if we had clones. - if result[:clone_total] > 0 - result[:clone_average] = total_clone_time / result[:clone_total] + if result[:clone][:total] > 0 + result[:clone][:timings][:average] = total_clone_time / result[:clone][:total] + result[:clone][:timings][:min], result[:clone][:timings][:max] = min_max_clone_times.minmax + result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax + result[:clone][:average] = mean(total_clones_per_day) end content_type :json From 57520ae995f6f41973bf990a168482b14dff4ed6 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 08:52:33 -0800 Subject: [PATCH 2/5] (QENG-1873) Fix Key Name Issue And minmax This commit fixes the problem of lingering :timings that were renamed to :duration. This should resolve a NilClass failure. This also converts string results from redis to float so that future minmax calls will treat them as numerics instead of strings. --- lib/vmpooler/api.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index ec0996b..222302a 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -268,10 +268,10 @@ module Vmpooler result[:clone][:total] += me[:clone][:total] total_clones_per_day.push(me[:clone][:total]) - clone_times = $redis.hvals('vmpooler__clone__' + date.to_s) + clone_times = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) unless clone_times.nil? or clone_times.empty? - clone_time = clone_times.map(&:to_f).reduce(:+).to_f + clone_time = clone_times.reduce(:+).to_f total_clone_time += clone_time me[:clone][:min], me[:clone][:max] = clone_times.minmax @@ -293,8 +293,8 @@ module Vmpooler # again, calc clone_average if we had clones. if result[:clone][:total] > 0 - result[:clone][:timings][:average] = total_clone_time / result[:clone][:total] - result[:clone][:timings][:min], result[:clone][:timings][:max] = min_max_clone_times.minmax + result[:clone][:duration][:average] = total_clone_time / result[:clone][:total] + result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax result[:clone][:average] = mean(total_clones_per_day) end From 5db5df826f6d059793c810bc4aea66d17fb6ca05 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:23:58 -0800 Subject: [PATCH 3/5] (QENG-1873) Add count Section to summary Result This adds a :count key to both the overall and daily results. The payload is now divided in to :duration and :count values for summary data. The daily results do not include anything under :count except the total number of VMs cloned for the day. The overall :count includes total, including minimum, maximum, and average VMs cloned for the date range requested. --- lib/vmpooler/api.rb | 70 ++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 222302a..9ce012c 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -214,12 +214,15 @@ module Vmpooler duration: { average: 0, min: 0, - max: 0 + max: 0, + total: 0 }, - average: 0, - min: 0, - max: 0, - total: 0 + count: { + average: 0, + min: 0, + max: 0, + total: 0 + } }, daily: [] # used for daily info } @@ -248,9 +251,9 @@ module Vmpooler end # total clone time for entire duration requested. - total_clone_time = 0 min_max_clone_times = [] # min/max clone times from each day. total_clones_per_day = [] # total clones from each day + total_clone_dur_day = [] # total clone durations from each day # generate sequence/range for from_date to to_date (inclusive). # for each date, calculate the day's clone total & average, as well @@ -259,44 +262,59 @@ module Vmpooler me = { date: date.to_s, clone: { - average: 0, - min: 0, - max: 0, - total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i, + duration: { + average: 0, + min: 0, + max: 0, + total: 0 + }, + count: { + total: $redis.hlen('vmpooler__clone__' + date.to_s).to_i + } } } - result[:clone][:total] += me[:clone][:total] - total_clones_per_day.push(me[:clone][:total]) + # add to daily clone count array + total_clones_per_day.push(me[:clone][:count][:total]) + + # fetch clone times from redis for this date. convert the results + # to float (numeric). clone_times = $redis.hvals('vmpooler__clone__' + date.to_s).map(&:to_f) - unless clone_times.nil? or clone_times.empty? + unless clone_times.nil? or clone_times.length == 0 clone_time = clone_times.reduce(:+).to_f - total_clone_time += clone_time + # add clone time for this date for later processing + total_clone_dur_day.push(clone_time) - me[:clone][:min], me[:clone][:max] = clone_times.minmax - min_max_clone_times.push(me[:clone][:min]) - min_max_clone_times.push(me[:clone][:max]) + me[:clone][:duration][:total] = clone_time + + # min and max clone durations for the day. + mi, ma = clone_times.minmax + me[:clone][:duration][:min], me[:clone][:duration][:max] = mi, ma + min_max_clone_times.push(mi, ma) # if clone_total > 0, calculate clone_average. # this prevents divide by 0 problems. - if me[:clone][:total] > 0 - me[:clone][:average] = clone_time / me[:clone][:total] + if me[:clone][:count][:total] > 0 + me[:clone][:duration][:average] = clone_time / me[:clone][:duration][:total] else - me[:clone][:average] = 0 + me[:clone][:duration][:average] = 0 end - # add to daily array - result[:daily].push(me) end + + # add to daily array + result[:daily].push(me) end # again, calc clone_average if we had clones. - if result[:clone][:total] > 0 - result[:clone][:duration][:average] = total_clone_time / result[:clone][:total] + unless total_clones_per_day.nil? or total_clones_per_day.length == 0 + result[:clone][:duration][:average] = result[:clone][:duration][:total] / result[:clone][:count][:total] result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax - result[:clone][:min], result[:clone][:max] = total_clones_per_day.minmax - result[:clone][:average] = mean(total_clones_per_day) + result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f + result[:clone][:count][:min], result[:clone][:count][:max] = total_clones_per_day.minmax + result[:clone][:count][:average] = mean(total_clones_per_day) + result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i end content_type :json From 85ac329894624031fc1a6fadfbc6d806272f5373 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:26:50 -0800 Subject: [PATCH 4/5] (QENG-1873) Fix /0 Error Calculate total count before trying to use it in average calculation. --- lib/vmpooler/api.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 9ce012c..96306f2 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -309,12 +309,15 @@ module Vmpooler # again, calc clone_average if we had clones. unless total_clones_per_day.nil? or total_clones_per_day.length == 0 + # totals + result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i + result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f + + # averages and other things. result[:clone][:duration][:average] = result[:clone][:duration][:total] / result[:clone][:count][:total] result[:clone][:duration][:min], result[:clone][:duration][:max] = min_max_clone_times.minmax - result[:clone][:duration][:total] = total_clone_dur_day.reduce(:+).to_f result[:clone][:count][:min], result[:clone][:count][:max] = total_clones_per_day.minmax result[:clone][:count][:average] = mean(total_clones_per_day) - result[:clone][:count][:total] = total_clones_per_day.reduce(:+).to_i end content_type :json From 90d6ac2f1bba7addc8ed3877108bb551a161ea1e Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 24 Feb 2015 09:32:02 -0800 Subject: [PATCH 5/5] (QENG-1873) Fix Average Duration Calculation Use the mean helper function to calculate average clone duration. --- lib/vmpooler/api.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 96306f2..2e2e693 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -296,7 +296,7 @@ module Vmpooler # if clone_total > 0, calculate clone_average. # this prevents divide by 0 problems. if me[:clone][:count][:total] > 0 - me[:clone][:duration][:average] = clone_time / me[:clone][:duration][:total] + me[:clone][:duration][:average] = mean(clone_times) else me[:clone][:duration][:average] = 0 end