puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 10:08:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

(POOLER-160) Revise Metrics Classwork

Browse source 
Review changes suggested to revise the Metrics related files into a more
logical class structure.

Also fixup grammar typos in docs strings and any trailing metrics that
have been recently added to vmpooler.
This commit is contained in:
John O'Connor 2020-06-18 21:27:08 +01:00
parent cb955a1bed
commit 8ed8c43970
16 changed files with 409 additions and 364 deletions
Download patch file Download diff file Expand all files Collapse all files

18
lib/vmpooler/metrics.rb
View file

@ -1,24 +1,24 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'vmpooler/metrics/statsd'
require 'vmpooler/metrics/graphite'
require 'vmpooler/metrics/promstats'
require 'vmpooler/metrics/dummy_statsd'
module Vmpooler module Vmpooler
class Metrics class Metrics
# static class instantiate appropriate metrics object. # static class instantiate appropriate metrics object.
def self.init(logger, params) def self.init(logger, params)
if params[:statsd] if params[:statsd]
metrics = Vmpooler::Statsd.new(logger, params[:statsd]) metrics = Vmpooler::Metrics::Statsd.new(logger, params[:statsd])
elsif params[:graphite] elsif params[:graphite]
metrics = Vmpooler::Graphite.new(logger, params[:graphite]) metrics = Vmpooler::Metrics::Graphite.new(logger, params[:graphite])
elsif params[:prometheus] elsif params[:prometheus]
metrics = Vmpooler::Promstats.new(logger, params[:prometheus]) metrics = Vmpooler::Metrics::Promstats.new(logger, params[:prometheus])
else else
metrics = Vmpooler::DummyStatsd.new metrics = Vmpooler::Metrics::DummyStatsd.new
end end
metrics metrics
end end
end end
end end
require 'vmpooler/metrics/statsd'
require 'vmpooler/metrics/dummy_statsd'
require 'vmpooler/metrics/graphite'
require 'vmpooler/metrics/promstats'

26
lib/vmpooler/metrics/dummy_statsd.rb
View file

@ -1,22 +1,24 @@
# frozen_string_literal: true # frozen_string_literal: true
module Vmpooler module Vmpooler
class DummyStatsd < Metrics class Metrics
attr_reader :server, :port, :prefix class DummyStatsd < Metrics
attr_reader :server, :port, :prefix
def initialize(*) def initialize(*)
end end
def increment(*) def increment(*)
true true
end end
def gauge(*) def gauge(*)
true true
end end
def timing(*) def timing(*)
true true
end
end end
end end
end end

70
lib/vmpooler/metrics/graphite.rb
View file

@ -3,43 +3,45 @@
require 'rubygems' unless defined?(Gem) require 'rubygems' unless defined?(Gem)
module Vmpooler module Vmpooler
class Graphite < Metrics class Metrics
attr_reader :server, :port, :prefix class Graphite < Metrics
attr_reader :server, :port, :prefix
def initialize(logger, params = {}) def initialize(logger, params = {})
raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
@server = params['server'] @server = params['server']
@port = params['port'] || 2003 @port = params['port'] || 2003
@prefix = params['prefix'] || 'vmpooler' @prefix = params['prefix'] || 'vmpooler'
@logger = logger @logger = logger
end end
def increment(label) def increment(label)
log label, 1 log label, 1
end end
def gauge(label, value) def gauge(label, value)
log label, value log label, value
end end
def timing(label, duration) def timing(label, duration)
log label, duration log label, duration
end end
def log(path, value) def log(path, value)
Thread.new do Thread.new do
socket = TCPSocket.new(server, port) socket = TCPSocket.new(server, port)
begin begin
socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}" socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
ensure ensure
socket.close socket.close
end end
end
rescue Errno::EADDRNOTAVAIL => e
warn "Could not assign address to graphite server #{server}: #{e}"
rescue StandardError => e
@logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}")
end end
rescue Errno::EADDRNOTAVAIL => e
warn "Could not assign address to graphite server #{server}: #{e}"
rescue StandardError => e
@logger.log('s', "[!] Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}")
end end
end end
end end

559
lib/vmpooler/metrics/promstats.rb
View file

@ -3,304 +3,319 @@
require 'prometheus/client' require 'prometheus/client'
module Vmpooler module Vmpooler
class Promstats < Metrics class Metrics
attr_reader :prefix, :endpoint, :metrics_prefix class Promstats < Metrics
attr_reader :prefix, :endpoint, :metrics_prefix
# Constants for Metric Types # Constants for Metric Types
M_COUNTER = 1 M_COUNTER = 1
M_GAUGE = 2 M_GAUGE = 2
M_SUMMARY = 3 M_SUMMARY = 3
M_HISTOGRAM = 4 M_HISTOGRAM = 4
# Customised Bucket set to use for the Pooler clone times set to more appropriate intervals. # Customised Bucket set to use for the Pooler clone times set to more appropriate intervals.
POOLER_TIME_BUCKETS = [1.0, 2.5, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0].freeze POOLER_TIME_BUCKETS = [1.0, 2.5, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0].freeze
# Same for Redis connection times - this is the same as the current Prometheus Default. # Same for redis connection times - this is the same as the current Prometheus Default.
# https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14 # https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/client/histogram.rb#L14
REDIS_CONNECT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze REDIS_CONNECT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze
@p_metrics = {} @p_metrics = {}
def initialize(logger, params = {}) def initialize(logger, params = {})
@prefix = params['prefix'] || 'vmpooler' @prefix = params['prefix'] || 'vmpooler'
@metrics_prefix = params['metrics_prefix'] || 'vmpooler' @metrics_prefix = params['metrics_prefix'] || 'vmpooler'
@endpoint = params['endpoint'] || '/prometheus' @endpoint = params['endpoint'] || '/prometheus'
@logger = logger @logger = logger
# Setup up prometheus registry and data structures # Setup up prometheus registry and data structures
@prometheus = Prometheus::Client.registry @prometheus = Prometheus::Client.registry
end end
# Metrics structure used to register the metrics and also translate/interpret the incoming metrics. # Metrics structure used to register the metrics and also translate/interpret the incoming metrics.
def vmpooler_metrics_table def vmpooler_metrics_table
{ {
errors: { errors: {
mtype: M_COUNTER, mtype: M_COUNTER,
docstring: 'Count of Errors for pool', docstring: 'Count of errors for pool',
prom_metric_prefix: "#{@metrics_prefix}_errors", prom_metric_prefix: "#{@metrics_prefix}_errors",
metric_suffixes: { metric_suffixes: {
markedasfailed: 'Timeout waiting for instance to initialise', markedasfailed: 'timeout waiting for instance to initialise',
duplicatehostname: 'Unable to create instance due to duplicate hostname' duplicatehostname: 'unable to create instance due to duplicate hostname',
staledns: 'unable to create instance due to duplicate DNS record'
},
param_labels: %i[template_name]
}, },
param_labels: %i[template_name] user: {
}, mtype: M_COUNTER,
user: { docstring: 'Number of pool instances this user created created',
mtype: M_COUNTER, prom_metric_prefix: "#{@metrics_prefix}_user",
docstring: 'Number of Pool Instances this user created created', param_labels: %i[user poolname]
prom_metric_prefix: "#{@metrics_prefix}_user",
param_labels: %i[user poolname]
},
usage_jenkins_instance: {
mtype: M_COUNTER,
docstring: 'Pools by Jenkins Instance usage',
prom_metric_prefix: "#{@metrics_prefix}_usage_jenkins_instance",
param_labels: %i[jenkins_instance value_stream poolname]
},
usage_branch_project: {
mtype: M_COUNTER,
docstring: 'Pools by branch/project usage',
prom_metric_prefix: "#{@metrics_prefix}_usage_branch_project",
param_labels: %i[branch project poolname]
},
usage_job_component: {
mtype: M_COUNTER,
docstring: 'Pools by job/component usage',
prom_metric_prefix: "#{@metrics_prefix}_usage_job_component",
param_labels: %i[job_name component_to_test poolname]
},
checkout: {
mtype: M_COUNTER,
docstring: 'Pool Checkout counts',
prom_metric_prefix: "#{@metrics_prefix}_checkout",
metric_suffixes: {
nonresponsive: 'Checkout Failed - Non Responsive Machine',
empty: 'Checkout Failed - no machine',
success: 'Successful Checkout',
invalid: 'Checkout Failed - Invalid Template'
}, },
param_labels: %i[poolname] usage_litmus: {
}, mtype: M_COUNTER,
config: { docstring: 'Pools by Litmus job usage',
mtype: M_COUNTER, prom_metric_prefix: "#{@metrics_prefix}_usage_litmus",
docstring: 'vmpooler Pool Configuration Request', param_labels: %i[user poolname]
prom_metric_prefix: "#{@metrics_prefix}_config",
metric_suffixes: { invalid: 'Invalid' },
param_labels: %i[poolname]
},
poolreset: {
mtype: M_COUNTER,
docstring: 'Pool Reset Counter',
prom_metric_prefix: "#{@metrics_prefix}_poolreset",
metric_suffixes: { invalid: 'Invalid Pool' },
param_labels: %i[poolname]
},
connect: {
mtype: M_COUNTER,
docstring: 'vmpooler Connect (to vSphere)',
prom_metric_prefix: "#{@metrics_prefix}_connect",
metric_suffixes: {
open: 'Connect Succeeded',
fail: 'Connect Failed'
}, },
param_labels: [] usage_jenkins_instance: {
}, mtype: M_COUNTER,
migrate_from: { docstring: 'Pools by Jenkins instance usage',
mtype: M_COUNTER, prom_metric_prefix: "#{@metrics_prefix}_usage_jenkins_instance",
docstring: 'vmpooler Machine Migrated from', param_labels: %i[jenkins_instance value_stream poolname]
prom_metric_prefix: "#{@metrics_prefix}_migrate_from", },
param_labels: %i[host_name] usage_branch_project: {
}, mtype: M_COUNTER,
migrate_to: { docstring: 'Pools by branch/project usage',
mtype: M_COUNTER, prom_metric_prefix: "#{@metrics_prefix}_usage_branch_project",
docstring: 'vmpooler Machine Migrated to', param_labels: %i[branch project poolname]
prom_metric_prefix: "#{@metrics_prefix}_migrate_to", },
param_labels: %i[host_name] usage_job_component: {
}, mtype: M_COUNTER,
ready: { docstring: 'Pools by job/component usage',
mtype: M_GAUGE, prom_metric_prefix: "#{@metrics_prefix}_usage_job_component",
docstring: 'vmpooler Number of Machines in Ready State', param_labels: %i[job_name component_to_test poolname]
prom_metric_prefix: "#{@metrics_prefix}_ready", },
param_labels: %i[poolname] checkout: {
}, mtype: M_COUNTER,
running: { docstring: 'Pool checkout counts',
mtype: M_GAUGE, prom_metric_prefix: "#{@metrics_prefix}_checkout",
docstring: 'vmpooler Number of Machines Running', metric_suffixes: {
prom_metric_prefix: "#{@metrics_prefix}_running", nonresponsive: 'checkout failed - non responsive machine',
param_labels: %i[poolname] empty: 'checkout failed - no machine',
}, success: 'successful checkout',
connection_available: { invalid: 'checkout failed - invalid template'
mtype: M_GAUGE, },
docstring: 'vmpooler Redis Connections Available', param_labels: %i[poolname]
prom_metric_prefix: "#{@metrics_prefix}_connection_available", },
param_labels: %i[type provider] config: {
}, mtype: M_COUNTER,
time_to_ready_state: { docstring: 'vmpooler pool configuration request',
mtype: M_HISTOGRAM, prom_metric_prefix: "#{@metrics_prefix}_config",
buckets: POOLER_TIME_BUCKETS, metric_suffixes: { invalid: 'Invalid' },
docstring: 'Time taken for machine to read ready state for pool', param_labels: %i[poolname]
prom_metric_prefix: "#{@metrics_prefix}_time_to_ready_state", },
param_labels: %i[poolname] poolreset: {
}, mtype: M_COUNTER,
migrate: { docstring: 'Pool reset counter',
mtype: M_HISTOGRAM, prom_metric_prefix: "#{@metrics_prefix}_poolreset",
buckets: POOLER_TIME_BUCKETS, metric_suffixes: { invalid: 'Invalid Pool' },
docstring: 'vmpooler Time taken to migrate machine for pool', param_labels: %i[poolname]
prom_metric_prefix: "#{@metrics_prefix}_migrate", },
param_labels: %i[poolname] connect: {
}, mtype: M_COUNTER,
clone: { docstring: 'vmpooler connect (to vSphere)',
mtype: M_HISTOGRAM, prom_metric_prefix: "#{@metrics_prefix}_connect",
buckets: POOLER_TIME_BUCKETS, metric_suffixes: {
docstring: 'vmpooler Time taken to Clone Machine', open: 'Connect Succeeded',
prom_metric_prefix: "#{@metrics_prefix}_clone", fail: 'Connect Failed'
param_labels: %i[poolname] },
}, param_labels: []
destroy: { },
mtype: M_HISTOGRAM, migrate_from: {
buckets: POOLER_TIME_BUCKETS, mtype: M_COUNTER,
docstring: 'vmpooler Time taken to Destroy Machine', docstring: 'vmpooler machine migrated from',
prom_metric_prefix: "#{@metrics_prefix}_destroy", prom_metric_prefix: "#{@metrics_prefix}_migrate_from",
param_labels: %i[poolname] param_labels: %i[host_name]
}, },
connection_waited: { migrate_to: {
mtype: M_HISTOGRAM, mtype: M_COUNTER,
buckets: REDIS_CONNECT_BUCKETS, docstring: 'vmpooler machine migrated to',
docstring: 'vmpooler Redis Connection Wait Time', prom_metric_prefix: "#{@metrics_prefix}_migrate_to",
prom_metric_prefix: "#{@metrics_prefix}_connection_waited", param_labels: %i[host_name]
param_labels: %i[type provider] },
api_vm: {
mtype: M_COUNTER,
docstring: 'Total number of HTTP request/sub-operations handled by the Rack application under the /vm endpoint',
prom_metric_prefix: "#{@metrics_prefix}_http_requests_vm_total",
param_labels: %i[method subpath operation]
},
ready: {
mtype: M_GAUGE,
docstring: 'vmpooler number of machines in ready State',
prom_metric_prefix: "#{@metrics_prefix}_ready",
param_labels: %i[poolname]
},
running: {
mtype: M_GAUGE,
docstring: 'vmpooler number of machines running',
prom_metric_prefix: "#{@metrics_prefix}_running",
param_labels: %i[poolname]
},
connection_available: {
mtype: M_GAUGE,
docstring: 'vmpooler redis connections available',
prom_metric_prefix: "#{@metrics_prefix}_connection_available",
param_labels: %i[type provider]
},
time_to_ready_state: {
mtype: M_HISTOGRAM,
buckets: POOLER_TIME_BUCKETS,
docstring: 'Time taken for machine to read ready state for pool',
prom_metric_prefix: "#{@metrics_prefix}_time_to_ready_state",
param_labels: %i[poolname]
},
migrate: {
mtype: M_HISTOGRAM,
buckets: POOLER_TIME_BUCKETS,
docstring: 'vmpooler time taken to migrate machine for pool',
prom_metric_prefix: "#{@metrics_prefix}_migrate",
param_labels: %i[poolname]
},
clone: {
mtype: M_HISTOGRAM,
buckets: POOLER_TIME_BUCKETS,
docstring: 'vmpooler time taken to clone machine',
prom_metric_prefix: "#{@metrics_prefix}_clone",
param_labels: %i[poolname]
},
destroy: {
mtype: M_HISTOGRAM,
buckets: POOLER_TIME_BUCKETS,
docstring: 'vmpooler time taken to destroy machine',
prom_metric_prefix: "#{@metrics_prefix}_destroy",
param_labels: %i[poolname]
},
connection_waited: {
mtype: M_HISTOGRAM,
buckets: REDIS_CONNECT_BUCKETS,
docstring: 'vmpooler redis connection wait time',
prom_metric_prefix: "#{@metrics_prefix}_connection_waited",
param_labels: %i[type provider]
}
} }
}
end
# Helper to add individual prom metric.
# Allow Histograms to specify the bucket size.
def add_prometheus_metric(metric_spec, name, docstring)
case metric_spec[:mtype]
when M_COUNTER
metric_class = Prometheus::Client::Counter
when M_GAUGE
metric_class = Prometheus::Client::Gauge
when M_SUMMARY
metric_class = Prometheus::Client::Summary
when M_HISTOGRAM
metric_class = Prometheus::Client::Histogram
else
raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}")
end end
if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets) # Helper to add individual prom metric.
prom_metric = metric_class.new( # Allow Histograms to specify the bucket size.
name.to_sym, def add_prometheus_metric(metric_spec, name, docstring)
docstring: docstring, case metric_spec[:mtype]
labels: metric_spec[:param_labels] + [:vmpooler_instance], when M_COUNTER
buckets: metric_spec[:buckets], metric_class = Prometheus::Client::Counter
preset_labels: { vmpooler_instance: @prefix } when M_GAUGE
) metric_class = Prometheus::Client::Gauge
else when M_SUMMARY
prom_metric = metric_class.new( metric_class = Prometheus::Client::Summary
name.to_sym, when M_HISTOGRAM
docstring: docstring, metric_class = Prometheus::Client::Histogram
labels: metric_spec[:param_labels] + [:vmpooler_instance],
preset_labels: { vmpooler_instance: @prefix }
)
end
@prometheus.register(prom_metric)
end
# Top level method to register all the prometheus metrics.
def setup_prometheus_metrics
@p_metrics = vmpooler_metrics_table
@p_metrics.each do |_name, metric_spec|
if metric_spec.key? :metric_suffixes
# Iterate thru the suffixes if provided to register multiple counters here.
metric_spec[:metric_suffixes].each do |metric_suffix|
add_prometheus_metric(
metric_spec,
"#{metric_spec[:prom_metric_prefix]}_#{metric_suffix[0]}",
"#{metric_spec[:docstring]} #{metric_suffix[1]}"
)
end
else else
# No Additional counter suffixes so register this as metric. raise("Unable to register metric #{name} with metric type #{metric_spec[:mtype]}")
add_prometheus_metric( end
metric_spec,
metric_spec[:prom_metric_prefix], if (metric_spec[:mtype] == M_HISTOGRAM) && (metric_spec.key? :buckets)
metric_spec[:docstring] prom_metric = metric_class.new(
name.to_sym,
docstring: docstring,
labels: metric_spec[:param_labels] + [:vmpooler_instance],
buckets: metric_spec[:buckets],
preset_labels: { vmpooler_instance: @prefix }
)
else
prom_metric = metric_class.new(
name.to_sym,
docstring: docstring,
labels: metric_spec[:param_labels] + [:vmpooler_instance],
preset_labels: { vmpooler_instance: @prefix }
) )
end end
end @prometheus.register(prom_metric)
end
# locate a metric and check/interpet the sub-fields.
def find_metric(label)
sublabels = label.split('.')
metric_key = sublabels.shift.to_sym
raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key
metric = @p_metrics[metric_key].clone
if metric.key? :metric_suffixes
metric_subkey = sublabels.shift.to_sym
raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym
metric[:metric_name] = "#{metric[:prom_metric_prefix]}_#{metric_subkey}"
else
metric[:metric_name] = metric[:prom_metric_prefix]
end end
# Check if we are looking for a parameter value at last element. # Top level method to register all the prometheus metrics.
if metric.key? :param_labels
metric[:labels] = {} def setup_prometheus_metrics
# Special case processing here - if there is only one parameter label then make sure @p_metrics = vmpooler_metrics_table
# we append all of the remaining contents of the metric with "." separators to ensure @p_metrics.each do |_name, metric_spec|
# we get full nodenames (e.g. for Migration to node operations) if metric_spec.key? :metric_suffixes
if metric[:param_labels].length == 1 # Iterate thru the suffixes if provided to register multiple counters here.
metric[:labels][metric[:param_labels].first] = sublabels.join('.') metric_spec[:metric_suffixes].each do |metric_suffix|
else add_prometheus_metric(
metric[:param_labels].reverse_each do |param_label| metric_spec,
metric[:labels][param_label] = sublabels.pop(1).first "#{metric_spec[:prom_metric_prefix]}_#{metric_suffix[0]}",
"#{metric_spec[:docstring]} #{metric_suffix[1]}"
)
end
else
# No Additional counter suffixes so register this as metric.
add_prometheus_metric(
metric_spec,
metric_spec[:prom_metric_prefix],
metric_spec[:docstring]
)
end end
end end
end end
metric
end
# Helper to get lab metrics. # locate a metric and check/interpet the sub-fields.
def get(label) def find_metric(label)
metric = find_metric(label) sublabels = label.split('.')
[metric, @prometheus.get(metric[:metric_name])] metric_key = sublabels.shift.to_sym
end raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key
# Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation. metric = @p_metrics[metric_key].clone
def increment(label)
begin
counter_metric, c = get(label)
c.increment(labels: counter_metric[:labels])
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}")
end
end
def gauge(label, value) if metric.key? :metric_suffixes
begin metric_subkey = sublabels.shift.to_sym
unless value.nil? raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym
gauge_metric, g = get(label)
g.set(value.to_i, labels: gauge_metric[:labels]) metric[:metric_name] = "#{metric[:prom_metric_prefix]}_#{metric_subkey}"
else
metric[:metric_name] = metric[:prom_metric_prefix]
end end
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}")
end
end
def timing(label, duration) # Check if we are looking for a parameter value at last element.
begin if metric.key? :param_labels
# https://prometheus.io/docs/practices/histograms/ metric[:labels] = {}
unless duration.nil? # Special case processing here - if there is only one parameter label then make sure
histogram_metric, hm = get(label) # we append all of the remaining contents of the metric with "." separators to ensure
hm.observe(duration.to_f, labels: histogram_metric[:labels]) # we get full nodenames (e.g. for Migration to node operations)
if metric[:param_labels].length == 1
metric[:labels][metric[:param_labels].first] = sublabels.join('.')
else
metric[:param_labels].reverse_each do |param_label|
metric[:labels][param_label] = sublabels.pop(1).first
end
end
end
metric
end
# Helper to get lab metrics.
def get(label)
metric = find_metric(label)
[metric, @prometheus.get(metric[:metric_name])]
end
# Note - Catch and log metrics failures so they can be noted, but don't interrupt vmpooler operation.
def increment(label)
begin
counter_metric, c = get(label)
c.increment(labels: counter_metric[:labels])
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging metric #{label} increment : #{e}")
end
end
def gauge(label, value)
begin
unless value.nil?
gauge_metric, g = get(label)
g.set(value.to_i, labels: gauge_metric[:labels])
end
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging gauge #{label}, value #{value}: #{e}")
end
end
def timing(label, duration)
begin
# https://prometheus.io/docs/practices/histograms/
unless duration.nil?
histogram_metric, hm = get(label)
hm.observe(duration.to_f, labels: histogram_metric[:labels])
end
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}")
end end
rescue StandardError => e
@logger.log('s', "[!] prometheus error logging timing event label #{label}, duration #{duration}: #{e}")
end end
end end
end end

50
lib/vmpooler/metrics/statsd.rb
View file

@ -4,35 +4,37 @@ require 'rubygems' unless defined?(Gem)
require 'statsd' require 'statsd'
module Vmpooler module Vmpooler
class Statsd < Metrics class Metrics
attr_reader :server, :port, :prefix class Statsd < Metrics
attr_reader :server, :port, :prefix
def initialize(logger, params = {}) def initialize(logger, params = {})
raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty? raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
host = params['server'] host = params['server']
@port = params['port'] || 8125 @port = params['port'] || 8125
@prefix = params['prefix'] || 'vmpooler' @prefix = params['prefix'] || 'vmpooler'
@server = ::Statsd.new(host, @port) @server = ::Statsd.new(host, @port)
@logger = logger @logger = logger
end end
def increment(label) def increment(label)
server.increment(prefix + '.' + label) server.increment(prefix + '.' + label)
rescue StandardError => e rescue StandardError => e
@logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
end end
def gauge(label, value) def gauge(label, value)
server.gauge(prefix + '.' + label, value) server.gauge(prefix + '.' + label, value)
rescue StandardError => e rescue StandardError => e
@logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
end end
def timing(label, duration) def timing(label, duration)
server.timing(prefix + '.' + label, duration) server.timing(prefix + '.' + label, duration)
rescue StandardError => e rescue StandardError => e
@logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}") @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
end
end end
end end
end end

2
spec/integration/api/v1/config_spec.rb
View file

@ -34,7 +34,7 @@ describe Vmpooler::API::V1 do
describe '/config/pooltemplate' do describe '/config/pooltemplate' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:current_time) { Time.now } let(:current_time) { Time.now }

2
spec/integration/api/v1/ondemandvm_spec.rb
View file

@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do
describe '/ondemandvm' do describe '/ondemandvm' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:config) { let(:config) {
{ {
config: { config: {

2
spec/integration/api/v1/poolreset.rb
View file

@ -31,7 +31,7 @@ describe Vmpooler::API::V1 do
describe '/poolreset' do describe '/poolreset' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:current_time) { Time.now } let(:current_time) { Time.now }

2
spec/integration/api/v1/vm_hostname_spec.rb
View file

@ -21,7 +21,7 @@ describe Vmpooler::API::V1 do
describe '/vm/:hostname' do describe '/vm/:hostname' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:config) { let(:config) {
{ {

2
spec/integration/api/v1/vm_spec.rb
View file

@ -17,7 +17,7 @@ describe Vmpooler::API::V1 do
describe '/vm' do describe '/vm' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:config) { let(:config) {
{ {
config: { config: {

2
spec/integration/api/v1/vm_template_spec.rb
View file

@ -17,7 +17,7 @@ describe Vmpooler::API::V1 do
describe '/vm/:template' do describe '/vm/:template' do
let(:prefix) { '/api/v1' } let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:config) { let(:config) {
{ {
config: { config: {

2
spec/unit/pool_manager_spec.rb
View file

@ -10,7 +10,7 @@ RSpec::Matchers.define :a_pool_with_name_of do |value|
end end
describe 'Pool Manager' do describe 'Pool Manager' do
let(:logger) { MockLogger.new } let(:logger) { MockLogger.new }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:pool) { 'pool1' } let(:pool) { 'pool1' }
let(:vm) { 'vm1' } let(:vm) { 'vm1' }
let(:timeout) { 5 } let(:timeout) { 5 }

30
spec/unit/promstats_spec.rb
View file

@ -5,12 +5,12 @@ require 'spec_helper'
describe 'prometheus' do describe 'prometheus' do
logger = MockLogger.new logger = MockLogger.new
params = { 'prefix': 'test', 'metrics_prefix': 'mtest', 'endpoint': 'eptest' } params = { 'prefix': 'test', 'metrics_prefix': 'mtest', 'endpoint': 'eptest' }
subject = Vmpooler::Promstats.new(logger, params) subject = Vmpooler::Metrics::Promstats.new(logger, params)
let(:logger) { MockLogger.new } let(:logger) { MockLogger.new }
describe '#initialise' do describe '#initialise' do
it 'returns a Metrics object' do it 'returns a Metrics object' do
expect(Vmpooler::Promstats.new(logger)).to be_a(Vmpooler::Metrics) expect(Vmpooler::Metrics::Promstats.new(logger)).to be_a(Vmpooler::Metrics)
end end
end end
@ -142,6 +142,13 @@ describe 'prometheus' do
po.get(labels: metric[:labels]) po.get(labels: metric[:labels])
}.by(1) }.by(1)
end end
it 'Increments errors.staledns.#{pool_name}' do
pool_name = 'test-pool'
expect { subject.increment("errors.staledns.#{pool_name}") }.to change {
metric, po = subject.get("errors.staledns.#{pool_name}")
po.get(labels: metric[:labels])
}.by(1)
end
it 'Increments user.#{user}.#{poolname}' do it 'Increments user.#{user}.#{poolname}' do
user = 'myuser' user = 'myuser'
poolname = 'test-pool' poolname = 'test-pool'
@ -150,6 +157,14 @@ describe 'prometheus' do
po.get(labels: metric[:labels]) po.get(labels: metric[:labels])
}.by(1) }.by(1)
end end
it 'Increments usage_litmus.#{user}.#{poolname}' do
user = 'myuser'
poolname = 'test-pool'
expect { subject.increment("usage_litmus.#{user}.#{poolname}") }.to change {
metric, po = subject.get("usage_litmus.#{user}.#{poolname}")
po.get(labels: metric[:labels])
}.by(1)
end
it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}' do it 'Increments label usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}' do
jenkins_instance = 'jenkins_test_instance' jenkins_instance = 'jenkins_test_instance'
value_stream = 'notional_value' value_stream = 'notional_value'
@ -177,7 +192,6 @@ describe 'prometheus' do
po.get(labels: metric[:labels]) po.get(labels: metric[:labels])
}.by(1) }.by(1)
end end
it 'Increments connect.open' do it 'Increments connect.open' do
expect { subject.increment('connect.open') }.to change { expect { subject.increment('connect.open') }.to change {
metric, po = subject.get('connect.open') metric, po = subject.get('connect.open')
@ -204,6 +218,16 @@ describe 'prometheus' do
po.get(labels: metric[:labels]) po.get(labels: metric[:labels])
}.by(1) }.by(1)
end end
it 'Increments label api_vm.#{method}.#{subpath}.#{operation}' do
method = 'get'
subpath = 'template'
operation = 'something'
expect { subject.increment("api_vm.#{method}.#{subpath}.#{operation}") }.to change {
metric, po = subject.get("api_vm.#{method}.#{subpath}.#{operation}")
po.get(labels: metric[:labels])
}.by(1)
end
end end
describe '#gauge' do describe '#gauge' do

2
spec/unit/providers/base_spec.rb
View file

@ -6,7 +6,7 @@ require 'vmpooler/providers/base'
describe 'Vmpooler::PoolManager::Provider::Base' do describe 'Vmpooler::PoolManager::Provider::Base' do
let(:logger) { MockLogger.new } let(:logger) { MockLogger.new }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:config) { {} } let(:config) { {} }
let(:provider_name) { 'base' } let(:provider_name) { 'base' }
let(:provider_options) { { 'param' => 'value' } } let(:provider_options) { { 'param' => 'value' } }

2
spec/unit/providers/dummy_spec.rb
View file

@ -3,7 +3,7 @@ require 'vmpooler/providers/dummy'
describe 'Vmpooler::PoolManager::Provider::Dummy' do describe 'Vmpooler::PoolManager::Provider::Dummy' do
let(:logger) { MockLogger.new } let(:logger) { MockLogger.new }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:pool_name) { 'pool1' } let(:pool_name) { 'pool1' }
let(:other_pool_name) { 'pool2' } let(:other_pool_name) { 'pool2' }
let(:vm_name) { 'vm1' } let(:vm_name) { 'vm1' }

2
spec/unit/providers/vsphere_spec.rb
View file

@ -41,7 +41,7 @@ end
describe 'Vmpooler::PoolManager::Provider::VSphere' do describe 'Vmpooler::PoolManager::Provider::VSphere' do
let(:logger) { MockLogger.new } let(:logger) { MockLogger.new }
let(:metrics) { Vmpooler::DummyStatsd.new } let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
let(:poolname) { 'pool1'} let(:poolname) { 'pool1'}
let(:provider_options) { { 'param' => 'value' } } let(:provider_options) { { 'param' => 'value' } }
let(:datacenter_name) { 'MockDC' } let(:datacenter_name) { 'MockDC' }