From d938a50ee805eefd709a1b9edc27886f3349efed Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 3 Jun 2015 13:07:48 -0700 Subject: [PATCH 1/3] Add get_tag_summary and get_task_summary helpers --- lib/vmpooler/api/helpers.rb | 127 +++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 82f87ff..a97455b 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -147,7 +147,9 @@ module Vmpooler queue end - def get_tag_metrics(backend, date_str) + def get_tag_metrics(backend, date_str, opts = {}) + opts = {:only => false}.merge(opts) + tags = {} backend.hgetall('vmpooler__tag__' + date_str).each do |key, value| @@ -158,6 +160,10 @@ module Vmpooler hostname, tag = key.split(':', 2) end + if opts[:only] + next unless tag == opts[:only] + end + tags[tag] ||= {} tags[tag][value] ||= 0 tags[tag][value] += 1 @@ -169,8 +175,38 @@ module Vmpooler tags end + def get_tag_summary(backend, from_date, to_date, opts = {}) + opts = {:only => false}.merge(opts) + + result = { + tag: {}, + daily: [] + } + + (from_date..to_date).each do |date| + daily = { + date: date.to_s, + tag: get_tag_metrics(backend, date.to_s, opts) + } + result[:daily].push(daily) + end + + result[:daily].each do |daily| + daily[:tag].each_key do |tag| + result[:tag][tag] ||= {} + + daily[:tag][tag].each do |key, value| + result[:tag][tag][key] ||= 0 + result[:tag][tag][key] += value + end + end + end + + result + end + def get_task_metrics(backend, task_str, date_str, opts = {}) - opts = {:bypool => false}.merge(opts) + opts = {:bypool => false, :only => false}.merge(opts) task = { duration: { @@ -226,9 +262,96 @@ module Vmpooler task[:duration][:min], task[:duration][:max] = task_times.minmax end + if opts[:only] + task.each_key do |key| + task.delete(key) unless key.to_s == opts[:only] + end + end + task end + def get_task_summary(backend, task_str, from_date, to_date, opts = {}) + opts = {:bypool => false, :only => false}.merge(opts) + + task_sym = task_str.to_sym + + result = { + task_sym => {}, + daily: [] + } + + (from_date..to_date).each do |date| + daily = { + date: date.to_s, + task_sym => get_task_metrics(backend, task_str, date.to_s, opts) + } + result[:daily].push(daily) + end + + daily_task = {} + daily_task_bypool = {} if opts[:bypool] == true + + result[:daily].each do |daily| + daily[task_sym].each_key do |type| + result[task_sym][type] ||= {} + daily_task[type] ||= {} + + ['min', 'max'].each do |key| + if daily[task_sym][type][key] + daily_task[type][:data] ||= [] + daily_task[type][:data].push(daily[task_sym][type][key]) + end + end + + result[task_sym][type][:total] ||= 0 + result[task_sym][type][:total] += daily[task_sym][type][:total] + + if opts[:bypool] == true + result[task_sym][type][:pool] ||= {} + daily_task_bypool[type] ||= {} + + next unless daily[task_sym][type][:pool] + + daily[task_sym][type][:pool].each_key do |pool| + result[task_sym][type][:pool][pool] ||= {} + daily_task_bypool[type][pool] ||= {} + + ['min', 'max'].each do |key| + if daily[task_sym][type][:pool][pool][key.to_sym] + daily_task_bypool[type][pool][:data] ||= [] + daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym]) + end + end + + result[task_sym][type][:pool][pool][:total] ||= 0 + result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total] + end + end + end + end + + result[task_sym].each_key do |type| + if daily_task[type][:data] + result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax + result[task_sym][type][:average] = mean(daily_task[type][:data]) + end + + if opts[:bypool] == true + result[task_sym].each_key do |type| + result[task_sym][type][:pool].each_key do |pool| + if daily_task_bypool[type][pool][:data] + result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax + result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data]) + end + end + end + end + end + + result + end + end end end From ce05c94677c3aa63e84f5596c9eb730f32c20174 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 3 Jun 2015 13:08:21 -0700 Subject: [PATCH 2/3] Generate summaries from helpers; individual routes - '/summary*' routes are now generated from helper methods - many '/summary/...' combinations now possible - '/summary/tag' - '/summary/tag/beaker_version' - '/summary/boot' - '/summary/boot/duration' - '/summary/clone' - '/summary/clone/count?from=2015-06-01' - etc. --- lib/vmpooler/api/v1.rb | 158 ++++++++++++++--------------------------- 1 file changed, 55 insertions(+), 103 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 9bc2a37..2ac2dae 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -67,39 +67,6 @@ module Vmpooler content_type :json result = { - boot: { - duration: { - average: 0, - min: 0, - max: 0, - total: 0, - pool: {} - }, - count: { - average: 0, - min: 0, - max: 0, - total: 0, - pool: {} - } - }, - clone: { - duration: { - average: 0, - min: 0, - max: 0, - total: 0, - pool: {} - }, - count: { - average: 0, - min: 0, - max: 0, - total: 0, - pool: {} - } - }, - tag: {}, daily: [] } @@ -121,90 +88,75 @@ module Vmpooler halt 400, 'Date range is invalid, \'from\' must be in the past.' end - (from_date..to_date).each do |date| - daily = { - date: date.to_s, - boot: get_task_metrics(backend, 'boot', date.to_s, :bypool => true), - clone: get_task_metrics(backend, 'clone', date.to_s, :bypool => true), - tag: get_tag_metrics(backend, date.to_s) - } + boot = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true) + clone = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true) + tag = get_tag_summary(backend, from_date, to_date) - result[:daily].push(daily) + result[:boot] = boot[:boot] + result[:clone] = clone[:clone] + result[:tag] = tag[:tag] + + daily = {} + + boot[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:boot] = day[:boot] end - [:boot, :clone].each do |task| - daily_counts = [] - daily_counts_bypool = {} - daily_durations = [] - daily_durations_bypool = {} + clone[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:clone] = day[:clone] + end - result[:daily].each do |daily| - if daily[task][:count][:pool] - daily[task][:count][:pool].each_key do |pool| - daily_counts_bypool[pool] ||= [] - daily_counts_bypool[pool].push(daily[task][:count][:pool][pool][:total]) + tag[:daily].each do |day| + daily[day[:date]] ||= {} + daily[day[:date]][:tag] = day[:tag] + end - if daily[task][:count][:pool][pool][:total] > 0 - daily_durations_bypool[pool] ||= [] - daily_durations_bypool[pool].push(daily[task][:duration][:pool][pool][:min]) - daily_durations_bypool[pool].push(daily[task][:duration][:pool][pool][:max]) - end + daily.each_key do |day| + result[:daily].push({ + date: day, + boot: daily[day][:boot], + clone: daily[day][:clone], + tag: daily[day][:tag] + }) + end - result[task][:count][:pool][pool] ||= {} - result[task][:count][:pool][pool][:total] ||= 0 - result[task][:count][:pool][pool][:total] += daily[task][:count][:pool][pool][:total] + JSON.pretty_generate(result) + end - result[task][:duration][:pool][pool] ||= {} - result[task][:duration][:pool][pool][:total] ||= 0 - result[task][:duration][:pool][pool][:total] += daily[task][:duration][:pool][pool][:total] - end - end + get "#{api_prefix}/summary/:route/?:key?/?" do + content_type :json - daily_counts.push(daily[task][:count][:total]) - if daily[task][:count][:total] > 0 - daily_durations.push(daily[task][:duration][:min]) - daily_durations.push(daily[task][:duration][:max]) - end + result = {} - result[task][:count][:total] += daily[task][:count][:total] - result[task][:duration][:total] += daily[task][:duration][:total] - end + from_param = params[:from] || Date.today.to_s + to_param = params[:to] || Date.today.to_s - if result[task][:count][:total] > 0 - result[task][:duration][:average] = result[task][:duration][:total] / result[task][:count][:total] ##?? - end - - result[task][:count][:min], result[task][:count][:max] = daily_counts.minmax - result[task][:count][:average] = mean(daily_counts) - - if daily_durations.length > 0 - result[task][:duration][:min], result[task][:duration][:max] = daily_durations.minmax - end - - daily_counts_bypool.each_key do |pool| - result[task][:count][:pool][pool][:min], result[task][:count][:pool][pool][:max] = daily_counts_bypool[pool].minmax - result[task][:count][:pool][pool][:average] = mean(daily_counts_bypool[pool]) - - if daily_durations_bypool[pool].length > 0 - result[task][:duration][:pool][pool] ||= {} - result[task][:duration][:pool][pool][:min], result[task][:duration][:pool][pool][:max] = daily_durations_bypool[pool].minmax - end - - if result[task][:count][:pool][pool][:total] > 0 - result[task][:duration][:pool][pool][:average] = result[task][:duration][:pool][pool][:total] / result[task][:count][:pool][pool][:total] - end + # Validate date formats + [from_param, to_param].each do |param| + if !validate_date_str(param.to_s) + halt 400, "Invalid date format '#{param}', must match YYYY-MM-DD." end end - result[:daily].each do |daily| - daily[:tag].each_key do |tag| - result[:tag][tag] ||= {} + from_date, to_date = Date.parse(from_param), Date.parse(to_param) - daily[:tag][tag].each do |key, value| - result[:tag][tag][key] ||= 0 - result[:tag][tag][key] += value - end - end + if to_date < from_date + halt 400, 'Date range is invalid, \'to\' cannot come before \'from\'.' + elsif from_date > Date.today + halt 400, 'Date range is invalid, \'from\' must be in the past.' + end + + case params[:route] + when 'boot' + result = get_task_summary(backend, 'boot', from_date, to_date, :bypool => true, :only => params[:key]) + when 'clone' + result = get_task_summary(backend, 'clone', from_date, to_date, :bypool => true, :only => params[:key]) + when 'tag' + result = get_tag_summary(backend, from_date, to_date, :only => params[:key]) + else + halt 404, JSON.pretty_generate({ 'ok' => false }) end JSON.pretty_generate(result) From d3f4f6fb7774e1adad8afcb64005bb3e66beb38e Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 3 Jun 2015 13:11:11 -0700 Subject: [PATCH 3/3] Rerouting for new /summary routes --- lib/vmpooler/api/reroute.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/vmpooler/api/reroute.rb b/lib/vmpooler/api/reroute.rb index 0592de4..81fe0ab 100644 --- a/lib/vmpooler/api/reroute.rb +++ b/lib/vmpooler/api/reroute.rb @@ -11,6 +11,10 @@ module Vmpooler call env.merge("PATH_INFO" => "/api/v#{api_version}/summary") end + get '/summary/:route/?:key?/?' do + call env.merge("PATH_INFO" => "/api/v#{api_version}/summary/#{params[:route]}/#{params[:key]}") + end + post '/token/?' do call env.merge("PATH_INFO" => "/api/v#{api_version}/token") end