diff --git a/lib/vmpooler/metrics.rb b/lib/vmpooler/metrics.rb index 241dc18..98c17f9 100644 --- a/lib/vmpooler/metrics.rb +++ b/lib/vmpooler/metrics.rb @@ -1,24 +1,24 @@ # frozen_string_literal: true +require 'vmpooler/metrics/statsd' +require 'vmpooler/metrics/graphite' +require 'vmpooler/metrics/promstats' +require 'vmpooler/metrics/dummy_statsd' + module Vmpooler class Metrics # static class instantiate appropriate metrics object. def self.init(logger, params) if params[:statsd] - metrics = Vmpooler::Statsd.new(logger, params[:statsd]) + metrics = Vmpooler::Metrics::Statsd.new(logger, params[:statsd]) elsif params[:graphite] - metrics = Vmpooler::Graphite.new(logger, params[:graphite]) + metrics = Vmpooler::Metrics::Graphite.new(logger, params[:graphite]) elsif params[:prometheus] - metrics = Vmpooler::Promstats.new(logger, params[:prometheus]) + metrics = Vmpooler::Metrics::Promstats.new(logger, params[:prometheus]) else - metrics = Vmpooler::DummyStatsd.new + metrics = Vmpooler::Metrics::DummyStatsd.new end metrics end end end - -require 'vmpooler/metrics/statsd' -require 'vmpooler/metrics/dummy_statsd' -require 'vmpooler/metrics/graphite' -require 'vmpooler/metrics/promstats' diff --git a/lib/vmpooler/metrics/dummy_statsd.rb b/lib/vmpooler/metrics/dummy_statsd.rb index 2cc4ac6..96f2eaa 100644 --- a/lib/vmpooler/metrics/dummy_statsd.rb +++ b/lib/vmpooler/metrics/dummy_statsd.rb @@ -1,22 +1,24 @@ # frozen_string_literal: true module Vmpooler - class DummyStatsd < Metrics - attr_reader :server, :port, :prefix + class Metrics + class DummyStatsd < Metrics + attr_reader :server, :port, :prefix - def initialize(*) - end + def initialize(*) + end - def increment(*) - true - end + def increment(*) + true + end - def gauge(*) - true - end + def gauge(*) + true + end - def timing(*) - true + def timing(*) + true + end end end end diff --git a/lib/vmpooler/metrics/graphite.rb b/lib/vmpooler/metrics/graphite.rb index e94e77a..128805d 100644 --- a/lib/vmpooler/metrics/graphite.rb +++ b/lib/vmpooler/metrics/graphite.rb @@ -3,43 +3,45 @@ require 'rubygems' unless defined?(Gem) module Vmpooler - class Graphite < Metrics - attr_reader :server, :port, :prefix + class Metrics + class Graphite < Metrics + attr_reader :server, :port, :prefix - def initialize(logger, params = {}) - raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? + def initialize(logger, params = {}) + raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? - @server = params['server'] - @port = params['port'] || 2003 - @prefix = params['prefix'] || 'vmpooler' - @logger = logger - end - - def increment(label) - log label, 1 - end - - def gauge(label, value) - log label, value - end - - def timing(label, duration) - log label, duration - end - - def log(path, value) - Thread.new do - socket = TCPSocket.new(server, port) - begin - socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}" - ensure - socket.close - end + @server = params['server'] + @port = params['port'] || 2003 + @prefix = params['prefix'] || 'vmpooler' + @logger = logger + end + + def increment(label) + log label, 1 + end + + def gauge(label, value) + log label, value + end + + def timing(label, duration) + log label, duration + end + + def log(path, value) + Thread.new do + socket = TCPSocket.new(server, port) + begin + socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}" + ensure + socket.close + end + end + rescue Errno::EADDRNOTAVAIL => e + warn "Could not assign address to graphite server #{server}: #{e}" + rescue StandardError => e + @logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}") end - rescue Errno::EADDRNOTAVAIL => e - warn "Could not assign address to graphite server #{server}: #{e}" - rescue StandardError => e - @logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}") end end end diff --git a/lib/vmpooler/metrics/promstats.rb b/lib/vmpooler/metrics/promstats.rb index 9a9e7a6..e385d17 100644 --- a/lib/vmpooler/metrics/promstats.rb +++ b/lib/vmpooler/metrics/promstats.rb @@ -3,304 +3,319 @@ require 'prometheus/client' module Vmpooler - class Promstats < Metrics - attr_reader :prefix, :endpoint, :metrics_prefix + class Metrics + class Promstats < Metrics + attr_reader :prefix, :endpoint, :metrics_prefix - # Constants for Metric Types - M_COUNTER = 1 - M_GAUGE = 2 - M_SUMMARY = 3 - M_HISTOGRAM = 4 + # Constants for Metric Types + M_COUNTER = 1 + M_GAUGE = 2 + M_SUMMARY = 3 + M_HISTOGRAM = 4 - # Customised Bucket set to use for the Pooler clone times set to more appropriate intervals. - POOLER_TIME_BUCKETS = [1.0, 2.5, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0].freeze - # Same for Redis connection times - this is the same as the current Prometheus Default. - # https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14 - REDIS_CONNECT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze + # Customised Bucket set to use for the Pooler clone times set to more appropriate intervals. + POOLER_TIME_BUCKETS = [1.0, 2.5, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0].freeze + # Same for redis connection times - this is the same as the current Prometheus Default. + # https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14 + REDIS_CONNECT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze - @p_metrics = {} + @p_metrics = {} - def initialize(logger, params = {}) - @prefix = params['prefix'] || 'vmpooler' - @metrics_prefix = params['metrics_prefix'] || 'vmpooler' - @endpoint = params['endpoint'] || '/prometheus' - @logger = logger + def initialize(logger, params = {}) + @prefix = params['prefix'] || 'vmpooler' + @metrics_prefix = params['metrics_prefix'] || 'vmpooler' + @endpoint = params['endpoint'] || '/prometheus' + @logger = logger - # Setup up prometheus registry and data structures - @prometheus = Prometheus::Client.registry - end + # Setup up prometheus registry and data structures + @prometheus = Prometheus::Client.registry + end - # Metrics structure used to register the metrics and also translate/interpret the incoming metrics. - def vmpooler_metrics_table - { - errors: { - mtype: M_COUNTER, - docstring: 'Count of Errors for pool', - prom_metric_prefix: "#{@metrics_prefix}_errors", - metric_suffixes: { - markedasfailed: 'Timeout waiting for instance to initialise', - duplicatehostname: 'Unable to create instance due to duplicate hostname' + # Metrics structure used to register the metrics and also translate/interpret the incoming metrics. + def vmpooler_metrics_table + { + errors: { + mtype: M_COUNTER, + docstring: 'Count of errors for pool', + prom_metric_prefix: "#{@metrics_prefix}_errors", + metric_suffixes: { + markedasfailed: 'timeout waiting for instance to initialise', + duplicatehostname: 'unable to create instance due to duplicate hostname', + staledns: 'unable to create instance due to duplicate DNS record' + }, + param_labels: %i[template_name] }, - param_labels: %i[template_name] - }, - user: { - mtype: M_COUNTER, - docstring: 'Number of Pool Instances this user created created', - prom_metric_prefix: "#{@metrics_prefix}_user", - param_labels: %i[user poolname] - }, - usage_jenkins_instance: { - mtype: M_COUNTER, - docstring: 'Pools by Jenkins Instance usage', - prom_metric_prefix: "#{@metrics_prefix}_usage_jenkins_instance", - param_labels: %i[jenkins_instance value_stream poolname] - }, - usage_branch_project: { - mtype: M_COUNTER, - docstring: 'Pools by branch/project usage', - prom_metric_prefix: "#{@metrics_prefix}_usage_branch_project", - param_labels: %i[branch project poolname] - }, - usage_job_component: { - mtype: M_COUNTER, - docstring: 'Pools by job/component usage', - prom_metric_prefix: "#{@metrics_prefix}_usage_job_component", - param_labels: %i[job_name component_to_test poolname] - }, - checkout: { - mtype: M_COUNTER, - docstring: 'Pool Checkout counts', - prom_metric_prefix: "#{@metrics_prefix}_checkout", - metric_suffixes: { - nonresponsive: 'Checkout Failed - Non Responsive Machine', - empty: 'Checkout Failed - no machine', - success: 'Successful Checkout', - invalid: 'Checkout Failed - Invalid Template' + user: { + mtype: M_COUNTER, + docstring: 'Number of pool instances this user created created', + prom_metric_prefix: "#{@metrics_prefix}_user", + param_labels: %i[user poolname] }, - param_labels: %i[poolname] - }, - config: { - mtype: M_COUNTER, - docstring: 'vmpooler Pool Configuration Request', - prom_metric_prefix: "#{@metrics_prefix}_config", - metric_suffixes: { invalid: 'Invalid' }, - param_labels: %i[poolname] - }, - poolreset: { - mtype: M_COUNTER, - docstring: 'Pool Reset Counter', - prom_metric_prefix: "#{@metrics_prefix}_poolreset", - metric_suffixes: { invalid: 'Invalid Pool' }, - param_labels: %i[poolname] - }, - connect: { - mtype: M_COUNTER, - docstring: 'vmpooler Connect (to vSphere)', - prom_metric_prefix: "#{@metrics_prefix}_connect", - metric_suffixes: { - open: 'Connect Succeeded', - fail: 'Connect Failed' + usage_litmus: { + mtype: M_COUNTER, + docstring: 'Pools by Litmus job usage', + prom_metric_prefix: "#{@metrics_prefix}_usage_litmus", + param_labels: %i[user poolname] }, - param_labels: [] - }, - migrate_from: { - mtype: M_COUNTER, - docstring: 'vmpooler Machine Migrated from', - prom_metric_prefix: "#{@metrics_prefix}_migrate_from", - param_labels: %i[host_name] - }, - migrate_to: { - mtype: M_COUNTER, - docstring: 'vmpooler Machine Migrated to', - prom_metric_prefix: "#{@metrics_prefix}_migrate_to", - param_labels: %i[host_name] - }, - ready: { - mtype: M_GAUGE, - docstring: 'vmpooler Number of Machines in Ready State', - prom_metric_prefix: "#{@metrics_prefix}_ready", - param_labels: %i[poolname] - }, - running: { - mtype: M_GAUGE, - docstring: 'vmpooler Number of Machines Running', - prom_metric_prefix: "#{@metrics_prefix}_running", - param_labels: %i[poolname] - }, - connection_available: { - mtype: M_GAUGE, - docstring: 'vmpooler Redis Connections Available', - prom_metric_prefix: "#{@metrics_prefix}_connection_available", - param_labels: %i[type provider] - }, - time_to_ready_state: { - mtype: M_HISTOGRAM, - buckets: POOLER_TIME_BUCKETS, - docstring: 'Time taken for machine to read ready state for pool', - prom_metric_prefix: "#{@metrics_prefix}_time_to_ready_state", - param_labels: %i[poolname] - }, - migrate: { - mtype: M_HISTOGRAM, - buckets: POOLER_TIME_BUCKETS, - docstring: 'vmpooler Time taken to migrate machine for pool', - prom_metric_prefix: "#{@metrics_prefix}_migrate", - param_labels: %i[poolname] - }, - clone: { - mtype: M_HISTOGRAM, - buckets: POOLER_TIME_BUCKETS, - docstring: 'vmpooler Time taken to Clone Machine', - prom_metric_prefix: "#{@metrics_prefix}_clone", - param_labels: %i[poolname] - }, - destroy: { - mtype: M_HISTOGRAM, - buckets: POOLER_TIME_BUCKETS, - docstring: 'vmpooler Time taken to Destroy Machine', - prom_metric_prefix: "#{@metrics_prefix}_destroy", - param_labels: %i[poolname] - }, - connection_waited: { - mtype: M_HISTOGRAM, - buckets: REDIS_CONNECT_BUCKETS, - docstring: 'vmpooler Redis Connection Wait Time', - prom_metric_prefix: "#{@metrics_prefix}_connection_waited", - param_labels: %i[type provider] + usage_jenkins_instance: { + mtype: M_COUNTER, + docstring: 'Pools by Jenkins instance usage', + prom_metric_prefix: "#{@metrics_prefix}_usage_jenkins_instance", + param_labels: %i[jenkins_instance value_stream poolname] + }, + usage_branch_project: { + mtype: M_COUNTER, + docstring: 'Pools by branch/project usage', + prom_metric_prefix: "#{@metrics_prefix}_usage_branch_project", + param_labels: %i[branch project poolname] + }, + usage_job_component: { + mtype: M_COUNTER, + docstring: 'Pools by job/component usage', + prom_metric_prefix: "#{@metrics_prefix}_usage_job_component", + param_labels: %i[job_name component_to_test poolname] + }, + checkout: { + mtype: M_COUNTER, + docstring: 'Pool checkout counts', + prom_metric_prefix: "#{@metrics_prefix}_checkout", + metric_suffixes: { + nonresponsive: 'checkout failed - non responsive machine', + empty: 'checkout failed - no machine', + success: 'successful checkout', + invalid: 'checkout failed - invalid template' + }, + param_labels: %i[poolname] + }, + config: { + mtype: M_COUNTER, + docstring: 'vmpooler pool configuration request', + prom_metric_prefix: "#{@metrics_prefix}_config", + metric_suffixes: { invalid: 'Invalid' }, + param_labels: %i[poolname] + }, + poolreset: { + mtype: M_COUNTER, + docstring: 'Pool reset counter', + prom_metric_prefix: "#{@metrics_prefix}_poolreset", + metric_suffixes: { invalid: 'Invalid Pool' }, + param_labels: %i[poolname] + }, + connect: { + mtype: M_COUNTER, + docstring: 'vmpooler connect (to vSphere)', + prom_metric_prefix: "#{@metrics_prefix}_connect", + metric_suffixes: { + open: 'Connect Succeeded', + fail: 'Connect Failed' + }, + param_labels: [] + }, + migrate_from: { + mtype: M_COUNTER, + docstring: 'vmpooler machine migrated from', + prom_metric_prefix: "#{@metrics_prefix}_migrate_from", + param_labels: %i[host_name] + }, + migrate_to: { + mtype: M_COUNTER, + docstring: 'vmpooler machine migrated to', + prom_metric_prefix: "#{@metrics_prefix}_migrate_to", + param_labels: %i[host_name] + }, + api_vm: { + mtype: M_COUNTER, + docstring: 'Total number of HTTP request/sub-operations handled by the Rack application under the /vm endpoint', + prom_metric_prefix: "#{@metrics_prefix}_http_requests_vm_total", + param_labels: %i[method subpath operation] + }, + ready: { + mtype: M_GAUGE, + docstring: 'vmpooler number of machines in ready State', + prom_metric_prefix: "#{@metrics_prefix}_ready", + param_labels: %i[poolname] + }, + running: { + mtype: M_GAUGE, + docstring: 'vmpooler number of machines running', + prom_metric_prefix: "#{@metrics_prefix}_running", + param_labels: %i[poolname] + }, + connection_available: { + mtype: M_GAUGE, + docstring: 'vmpooler redis connections available', + prom_metric_prefix: "#{@metrics_prefix}_connection_available", + param_labels: %i[type provider] + }, + time_to_ready_state: { + mtype: M_HISTOGRAM, + buckets: POOLER_TIME_BUCKETS, + docstring: 'Time taken for machine to read ready state for pool', + prom_metric_prefix: "#{@metrics_prefix}_time_to_ready_state", + param_labels: %i[poolname] + }, + migrate: { + mtype: M_HISTOGRAM, + buckets: POOLER_TIME_BUCKETS, + docstring: 'vmpooler time taken to migrate machine for pool', + prom_metric_prefix: "#{@metrics_prefix}_migrate", + param_labels: %i[poolname] + }, + clone: { + mtype: M_HISTOGRAM, + buckets: POOLER_TIME_BUCKETS, + docstring: 'vmpooler time taken to clone machine', + prom_metric_prefix: "#{@metrics_prefix}_clone", + param_labels: %i[poolname] + }, + destroy: { + mtype: M_HISTOGRAM, + buckets: POOLER_TIME_BUCKETS, + docstring: 'vmpooler time taken to destroy machine', + prom_metric_prefix: "#{@metrics_prefix}_destroy", + param_labels: %i[poolname] + }, + connection_waited: { + mtype: M_HISTOGRAM, + buckets: REDIS_CONNECT_BUCKETS, + docstring: 'vmpooler redis connection wait time', + prom_metric_prefix: "#{@metrics_prefix}_connection_waited", + param_labels: %i[type provider] + } } - } - end - - # Helper to add individual prom metric. - # Allow Histograms to specify the bucket size. - def add_prometheus_metric(metric_spec, name, docstring) - case metric_spec[:mtype] - when M_COUNTER - metric_class = Prometheus::Client::Counter - when M_GAUGE - metric_class = Prometheus::Client::Gauge - when M_SUMMARY - metric_class = Prometheus::Client::Summary - when M_HISTOGRAM - metric_class = Prometheus::Client::Histogram - else - raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}") end - if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets) - prom_metric = metric_class.new( - name.to_sym, - docstring: docstring, - labels: metric_spec[:param_labels] + [:vmpooler_instance], - buckets: metric_spec[:buckets], - preset_labels: { vmpooler_instance: @prefix } - ) - else - prom_metric = metric_class.new( - name.to_sym, - docstring: docstring, - labels: metric_spec[:param_labels] + [:vmpooler_instance], - preset_labels: { vmpooler_instance: @prefix } - ) - end - @prometheus.register(prom_metric) - end - - # Top level method to register all the prometheus metrics. - - def setup_prometheus_metrics - @p_metrics = vmpooler_metrics_table - @p_metrics.each do |_name, metric_spec| - if metric_spec.key? :metric_suffixes - # Iterate thru the suffixes if provided to register multiple counters here. - metric_spec[:metric_suffixes].each do |metric_suffix| - add_prometheus_metric( - metric_spec, - "#{metric_spec[:prom_metric_prefix]}_#{metric_suffix[0]}", - "#{metric_spec[:docstring]} #{metric_suffix[1]}" - ) - end + # Helper to add individual prom metric. + # Allow Histograms to specify the bucket size. + def add_prometheus_metric(metric_spec, name, docstring) + case metric_spec[:mtype] + when M_COUNTER + metric_class = Prometheus::Client::Counter + when M_GAUGE + metric_class = Prometheus::Client::Gauge + when M_SUMMARY + metric_class = Prometheus::Client::Summary + when M_HISTOGRAM + metric_class = Prometheus::Client::Histogram else - # No Additional counter suffixes so register this as metric. - add_prometheus_metric( - metric_spec, - metric_spec[:prom_metric_prefix], - metric_spec[:docstring] + raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}") + end + + if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets) + prom_metric = metric_class.new( + name.to_sym, + docstring: docstring, + labels: metric_spec[:param_labels] + [:vmpooler_instance], + buckets: metric_spec[:buckets], + preset_labels: { vmpooler_instance: @prefix } + ) + else + prom_metric = metric_class.new( + name.to_sym, + docstring: docstring, + labels: metric_spec[:param_labels] + [:vmpooler_instance], + preset_labels: { vmpooler_instance: @prefix } ) end - end - end - - # locate a metric and check/interpet the sub-fields. - def find_metric(label) - sublabels = label.split('.') - metric_key = sublabels.shift.to_sym - raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key - - metric = @p_metrics[metric_key].clone - - if metric.key? :metric_suffixes - metric_subkey = sublabels.shift.to_sym - raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym - - metric[:metric_name] = "#{metric[:prom_metric_prefix]}_#{metric_subkey}" - else - metric[:metric_name] = metric[:prom_metric_prefix] + @prometheus.register(prom_metric) end - # Check if we are looking for a parameter value at last element. - if metric.key? :param_labels - metric[:labels] = {} - # Special case processing here - if there is only one parameter label then make sure - # we append all of the remaining contents of the metric with "." separators to ensure - # we get full nodenames (e.g. for Migration to node operations) - if metric[:param_labels].length == 1 - metric[:labels][metric[:param_labels].first] = sublabels.join('.') - else - metric[:param_labels].reverse_each do |param_label| - metric[:labels][param_label] = sublabels.pop(1).first + # Top level method to register all the prometheus metrics. + + def setup_prometheus_metrics + @p_metrics = vmpooler_metrics_table + @p_metrics.each do |_name, metric_spec| + if metric_spec.key? :metric_suffixes + # Iterate thru the suffixes if provided to register multiple counters here. + metric_spec[:metric_suffixes].each do |metric_suffix| + add_prometheus_metric( + metric_spec, + "#{metric_spec[:prom_metric_prefix]}_#{metric_suffix[0]}", + "#{metric_spec[:docstring]} #{metric_suffix[1]}" + ) + end + else + # No Additional counter suffixes so register this as metric. + add_prometheus_metric( + metric_spec, + metric_spec[:prom_metric_prefix], + metric_spec[:docstring] + ) end end end - metric - end - # Helper to get lab metrics. - def get(label) - metric = find_metric(label) - [metric, @prometheus.get(metric[:metric_name])] - end + # locate a metric and check/interpet the sub-fields. + def find_metric(label) + sublabels = label.split('.') + metric_key = sublabels.shift.to_sym + raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key - # Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation. - def increment(label) - begin - counter_metric, c = get(label) - c.increment(labels: counter_metric[:labels]) - rescue StandardError => e - @logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}") - end - end + metric = @p_metrics[metric_key].clone - def gauge(label, value) - begin - unless value.nil? - gauge_metric, g = get(label) - g.set(value.to_i, labels: gauge_metric[:labels]) + if metric.key? :metric_suffixes + metric_subkey = sublabels.shift.to_sym + raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym + + metric[:metric_name] = "#{metric[:prom_metric_prefix]}_#{metric_subkey}" + else + metric[:metric_name] = metric[:prom_metric_prefix] end - rescue StandardError => e - @logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}") - end - end - def timing(label, duration) - begin - # https://prometheus.io/docs/practices/histograms/ - unless duration.nil? - histogram_metric, hm = get(label) - hm.observe(duration.to_f, labels: histogram_metric[:labels]) + # Check if we are looking for a parameter value at last element. + if metric.key? :param_labels + metric[:labels] = {} + # Special case processing here - if there is only one parameter label then make sure + # we append all of the remaining contents of the metric with "." separators to ensure + # we get full nodenames (e.g. for Migration to node operations) + if metric[:param_labels].length == 1 + metric[:labels][metric[:param_labels].first] = sublabels.join('.') + else + metric[:param_labels].reverse_each do |param_label| + metric[:labels][param_label] = sublabels.pop(1).first + end + end + end + metric + end + + # Helper to get lab metrics. + def get(label) + metric = find_metric(label) + [metric, @prometheus.get(metric[:metric_name])] + end + + # Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation. + def increment(label) + begin + counter_metric, c = get(label) + c.increment(labels: counter_metric[:labels]) + rescue StandardError => e + @logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}") + end + end + + def gauge(label, value) + begin + unless value.nil? + gauge_metric, g = get(label) + g.set(value.to_i, labels: gauge_metric[:labels]) + end + rescue StandardError => e + @logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}") + end + end + + def timing(label, duration) + begin + # https://prometheus.io/docs/practices/histograms/ + unless duration.nil? + histogram_metric, hm = get(label) + hm.observe(duration.to_f, labels: histogram_metric[:labels]) + end + rescue StandardError => e + @logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}") end - rescue StandardError => e - @logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}") end end end diff --git a/lib/vmpooler/metrics/statsd.rb b/lib/vmpooler/metrics/statsd.rb index def7b58..85b705d 100644 --- a/lib/vmpooler/metrics/statsd.rb +++ b/lib/vmpooler/metrics/statsd.rb @@ -4,35 +4,37 @@ require 'rubygems' unless defined?(Gem) require 'statsd' module Vmpooler - class Statsd < Metrics - attr_reader :server, :port, :prefix + class Metrics + class Statsd < Metrics + attr_reader :server, :port, :prefix - def initialize(logger, params = {}) - raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? + def initialize(logger, params = {}) + raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? - host = params['server'] - @port = params['port'] || 8125 - @prefix = params['prefix'] || 'vmpooler' - @server = ::Statsd.new(host, @port) - @logger = logger - end + host = params['server'] + @port = params['port'] || 8125 + @prefix = params['prefix'] || 'vmpooler' + @server = ::Statsd.new(host, @port) + @logger = logger + end - def increment(label) - server.increment(prefix + '.' + label) - rescue StandardError => e - @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") - end + def increment(label) + server.increment(prefix + '.' + label) + rescue StandardError => e + @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") + end - def gauge(label, value) - server.gauge(prefix + '.' + label, value) - rescue StandardError => e - @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") - end + def gauge(label, value) + server.gauge(prefix + '.' + label, value) + rescue StandardError => e + @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") + end - def timing(label, duration) - server.timing(prefix + '.' + label, duration) - rescue StandardError => e - @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") + def timing(label, duration) + server.timing(prefix + '.' + label, duration) + rescue StandardError => e + @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") + end end end end diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v1/config_spec.rb index 7462307..655a2cc 100644 --- a/spec/integration/api/v1/config_spec.rb +++ b/spec/integration/api/v1/config_spec.rb @@ -34,7 +34,7 @@ describe Vmpooler::API::V1 do describe '/config/pooltemplate' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:current_time) { Time.now } diff --git a/spec/integration/api/v1/ondemandvm_spec.rb b/spec/integration/api/v1/ondemandvm_spec.rb index f9b31b2..6ab0076 100644 --- a/spec/integration/api/v1/ondemandvm_spec.rb +++ b/spec/integration/api/v1/ondemandvm_spec.rb @@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do describe '/ondemandvm' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:config) { { config: { diff --git a/spec/integration/api/v1/poolreset.rb b/spec/integration/api/v1/poolreset.rb index 043989e..35e2ba4 100644 --- a/spec/integration/api/v1/poolreset.rb +++ b/spec/integration/api/v1/poolreset.rb @@ -31,7 +31,7 @@ describe Vmpooler::API::V1 do describe '/poolreset' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:current_time) { Time.now } diff --git a/spec/integration/api/v1/vm_hostname_spec.rb b/spec/integration/api/v1/vm_hostname_spec.rb index 18fcec3..25201a8 100644 --- a/spec/integration/api/v1/vm_hostname_spec.rb +++ b/spec/integration/api/v1/vm_hostname_spec.rb @@ -21,7 +21,7 @@ describe Vmpooler::API::V1 do describe '/vm/:hostname' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:config) { { diff --git a/spec/integration/api/v1/vm_spec.rb b/spec/integration/api/v1/vm_spec.rb index 7f5c29a..15b6074 100644 --- a/spec/integration/api/v1/vm_spec.rb +++ b/spec/integration/api/v1/vm_spec.rb @@ -17,7 +17,7 @@ describe Vmpooler::API::V1 do describe '/vm' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:config) { { config: { diff --git a/spec/integration/api/v1/vm_template_spec.rb b/spec/integration/api/v1/vm_template_spec.rb index da83116..839b427 100644 --- a/spec/integration/api/v1/vm_template_spec.rb +++ b/spec/integration/api/v1/vm_template_spec.rb @@ -17,7 +17,7 @@ describe Vmpooler::API::V1 do describe '/vm/:template' do let(:prefix) { '/api/v1' } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:config) { { config: { diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index ed7d0ae..446d1ba 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -10,7 +10,7 @@ RSpec::Matchers.define :a_pool_with_name_of do |value| end describe 'Pool Manager' do let(:logger) { MockLogger.new } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } diff --git a/spec/unit/promstats_spec.rb b/spec/unit/promstats_spec.rb index d721cce..62c8f4a 100644 --- a/spec/unit/promstats_spec.rb +++ b/spec/unit/promstats_spec.rb @@ -5,12 +5,12 @@ require 'spec_helper' describe 'prometheus' do logger = MockLogger.new params = { 'prefix': 'test', 'metrics_prefix': 'mtest', 'endpoint': 'eptest' } - subject = Vmpooler::Promstats.new(logger, params) + subject = Vmpooler::Metrics::Promstats.new(logger, params) let(:logger) { MockLogger.new } describe '#initialise' do it 'returns a Metrics object' do - expect(Vmpooler::Promstats.new(logger)).to be_a(Vmpooler::Metrics) + expect(Vmpooler::Metrics::Promstats.new(logger)).to be_a(Vmpooler::Metrics) end end @@ -142,6 +142,13 @@ describe 'prometheus' do po.get(labels: metric[:labels]) }.by(1) end + it 'Increments errors.staledns.#{pool_name}' do + pool_name = 'test-pool' + expect { subject.increment("errors.staledns.#{pool_name}") }.to change { + metric, po = subject.get("errors.staledns.#{pool_name}") + po.get(labels: metric[:labels]) + }.by(1) + end it 'Increments user.#{user}.#{poolname}' do user = 'myuser' poolname = 'test-pool' @@ -150,6 +157,14 @@ describe 'prometheus' do po.get(labels: metric[:labels]) }.by(1) end + it 'Increments usage_litmus.#{user}.#{poolname}' do + user = 'myuser' + poolname = 'test-pool' + expect { subject.increment("usage_litmus.#{user}.#{poolname}") }.to change { + metric, po = subject.get("usage_litmus.#{user}.#{poolname}") + po.get(labels: metric[:labels]) + }.by(1) + end it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}' do jenkins_instance = 'jenkins_test_instance' value_stream = 'notional_value' @@ -177,7 +192,6 @@ describe 'prometheus' do po.get(labels: metric[:labels]) }.by(1) end - it 'Increments connect.open' do expect { subject.increment('connect.open') }.to change { metric, po = subject.get('connect.open') @@ -204,6 +218,16 @@ describe 'prometheus' do po.get(labels: metric[:labels]) }.by(1) end + it 'Increments label api_vm.#{method}.#{subpath}.#{operation}' do + method = 'get' + subpath = 'template' + operation = 'something' + expect { subject.increment("api_vm.#{method}.#{subpath}.#{operation}") }.to change { + metric, po = subject.get("api_vm.#{method}.#{subpath}.#{operation}") + po.get(labels: metric[:labels]) + }.by(1) + end + end describe '#gauge' do diff --git a/spec/unit/providers/base_spec.rb b/spec/unit/providers/base_spec.rb index 27b3d94..1cc2090 100644 --- a/spec/unit/providers/base_spec.rb +++ b/spec/unit/providers/base_spec.rb @@ -6,7 +6,7 @@ require 'vmpooler/providers/base' describe 'Vmpooler::PoolManager::Provider::Base' do let(:logger) { MockLogger.new } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:config) { {} } let(:provider_name) { 'base' } let(:provider_options) { { 'param' => 'value' } } diff --git a/spec/unit/providers/dummy_spec.rb b/spec/unit/providers/dummy_spec.rb index a22d2e6..f4e4849 100644 --- a/spec/unit/providers/dummy_spec.rb +++ b/spec/unit/providers/dummy_spec.rb @@ -3,7 +3,7 @@ require 'vmpooler/providers/dummy' describe 'Vmpooler::PoolManager::Provider::Dummy' do let(:logger) { MockLogger.new } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:pool_name) { 'pool1' } let(:other_pool_name) { 'pool2' } let(:vm_name) { 'vm1' } diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb index 90146b9..a2970ac 100644 --- a/spec/unit/providers/vsphere_spec.rb +++ b/spec/unit/providers/vsphere_spec.rb @@ -41,7 +41,7 @@ end describe 'Vmpooler::PoolManager::Provider::VSphere' do let(:logger) { MockLogger.new } - let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metrics) { Vmpooler::Metrics::DummyStatsd.new } let(:poolname) { 'pool1'} let(:provider_options) { { 'param' => 'value' } } let(:datacenter_name) { 'MockDC' }