mirror of
https://github.com/puppetlabs/vmpooler.git
synced 2026-01-27 02:18:41 -05:00
Merge pull request #159 from puppetlabs/dont-call-statsd-a-comeback
[QENG-4075] Land statsd support after earlier ci.next changes
This commit is contained in:
commit
063d45c60b
15 changed files with 320 additions and 87 deletions
2
Gemfile
2
Gemfile
|
|
@ -12,10 +12,12 @@ gem 'rbvmomi', '>= 1.8'
|
||||||
gem 'redis', '>= 3.2'
|
gem 'redis', '>= 3.2'
|
||||||
gem 'sinatra', '>= 1.4'
|
gem 'sinatra', '>= 1.4'
|
||||||
gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3
|
gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3
|
||||||
|
gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd'
|
||||||
|
|
||||||
# Test deps
|
# Test deps
|
||||||
group :test do
|
group :test do
|
||||||
gem 'rack-test', '>= 0.6'
|
gem 'rack-test', '>= 0.6'
|
||||||
gem 'rspec', '>= 3.2'
|
gem 'rspec', '>= 3.2'
|
||||||
|
gem 'simplecov', '>= 0.11.2'
|
||||||
gem 'yarjuf', '>= 2.0'
|
gem 'yarjuf', '>= 2.0'
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ module Vmpooler
|
||||||
require 'yaml'
|
require 'yaml'
|
||||||
require 'set'
|
require 'set'
|
||||||
|
|
||||||
%w( api graphite logger pool_manager vsphere_helper ).each do |lib|
|
%w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib|
|
||||||
begin
|
begin
|
||||||
require "vmpooler/#{lib}"
|
require "vmpooler/#{lib}"
|
||||||
rescue LoadError
|
rescue LoadError
|
||||||
|
|
@ -53,10 +53,6 @@ module Vmpooler
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if parsed_config[:graphite]['server']
|
|
||||||
parsed_config[:graphite]['prefix'] ||= 'vmpooler'
|
|
||||||
end
|
|
||||||
|
|
||||||
if parsed_config[:tagfilter]
|
if parsed_config[:tagfilter]
|
||||||
parsed_config[:tagfilter].keys.each do |tag|
|
parsed_config[:tagfilter].keys.each do |tag|
|
||||||
parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
|
parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
|
||||||
|
|
@ -64,7 +60,6 @@ module Vmpooler
|
||||||
end
|
end
|
||||||
|
|
||||||
parsed_config[:uptime] = Time.now
|
parsed_config[:uptime] = Time.now
|
||||||
|
|
||||||
parsed_config
|
parsed_config
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -76,11 +71,13 @@ module Vmpooler
|
||||||
Vmpooler::Logger.new logfile
|
Vmpooler::Logger.new logfile
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.new_graphite(server)
|
def self.new_metrics(params)
|
||||||
if server.nil? or server.empty? or server.length == 0
|
if params[:statsd]
|
||||||
nil
|
Vmpooler::Statsd.new(params[:statsd])
|
||||||
|
elsif params[:graphite]
|
||||||
|
Vmpooler::Graphite.new(params[:graphite])
|
||||||
else
|
else
|
||||||
Vmpooler::Graphite.new server
|
Vmpooler::DummyStatsd.new
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -42,9 +42,10 @@ module Vmpooler
|
||||||
use Vmpooler::API::Reroute
|
use Vmpooler::API::Reroute
|
||||||
use Vmpooler::API::V1
|
use Vmpooler::API::V1
|
||||||
|
|
||||||
def configure(config, redis, environment = :production)
|
def configure(config, redis, metrics, environment = :production)
|
||||||
self.settings.set :config, config
|
self.settings.set :config, config
|
||||||
self.settings.set :redis, redis
|
self.settings.set :redis, redis
|
||||||
|
self.settings.set :metrics, metrics
|
||||||
self.settings.set :environment, environment
|
self.settings.set :environment, environment
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,57 @@
|
||||||
module Vmpooler
|
module Vmpooler
|
||||||
class API
|
class API
|
||||||
class Dashboard < Sinatra::Base
|
class Dashboard < Sinatra::Base
|
||||||
|
|
||||||
|
# handle to the App's configuration information
|
||||||
|
def config
|
||||||
|
@config ||= Vmpooler::API.settings.config
|
||||||
|
end
|
||||||
|
|
||||||
|
# configuration setting for server hosting graph URLs to view
|
||||||
|
def graph_server
|
||||||
|
return @graph_server if @graph_server
|
||||||
|
|
||||||
|
if config[:graphs]
|
||||||
|
return false unless config[:graphs]['server']
|
||||||
|
@graph_server = config[:graphs]['server']
|
||||||
|
elsif config[:graphite]
|
||||||
|
return false unless config[:graphite]['server']
|
||||||
|
@graph_server = config[:graphite]['server']
|
||||||
|
else
|
||||||
|
false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# configuration setting for URL prefix for graphs to view
|
||||||
|
def graph_prefix
|
||||||
|
return @graph_prefix if @graph_prefix
|
||||||
|
|
||||||
|
if config[:graphs]
|
||||||
|
return "vmpooler" unless config[:graphs]['prefix']
|
||||||
|
@graph_prefix = config[:graphs]['prefix']
|
||||||
|
elsif config[:graphite]
|
||||||
|
return false unless config[:graphite]['prefix']
|
||||||
|
@graph_prefix = config[:graphite]['prefix']
|
||||||
|
else
|
||||||
|
false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# what is the base URL for viewable graphs?
|
||||||
|
def graph_url
|
||||||
|
return false unless graph_server && graph_prefix
|
||||||
|
@graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
|
||||||
|
end
|
||||||
|
|
||||||
|
# return a full URL to a viewable graph for a given metrics target (graphite syntax)
|
||||||
|
def graph_link(target = "")
|
||||||
|
return "" unless graph_url
|
||||||
|
graph_url + target
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
get '/dashboard/stats/vmpooler/pool/?' do
|
get '/dashboard/stats/vmpooler/pool/?' do
|
||||||
content_type :json
|
content_type :json
|
||||||
|
|
||||||
result = {}
|
result = {}
|
||||||
|
|
||||||
Vmpooler::API.settings.config[:pools].each do |pool|
|
Vmpooler::API.settings.config[:pools].each do |pool|
|
||||||
|
|
@ -13,13 +61,11 @@ module Vmpooler
|
||||||
end
|
end
|
||||||
|
|
||||||
if params[:history]
|
if params[:history]
|
||||||
if Vmpooler::API.settings.config[:graphite]['server']
|
if graph_url
|
||||||
history ||= {}
|
history ||= {}
|
||||||
|
|
||||||
begin
|
begin
|
||||||
buffer = open(
|
buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
|
||||||
'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json'
|
|
||||||
).read
|
|
||||||
history = JSON.parse(buffer)
|
history = JSON.parse(buffer)
|
||||||
|
|
||||||
history.each do |pool|
|
history.each do |pool|
|
||||||
|
|
@ -52,59 +98,47 @@ module Vmpooler
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
JSON.pretty_generate(result)
|
JSON.pretty_generate(result)
|
||||||
end
|
end
|
||||||
|
|
||||||
get '/dashboard/stats/vmpooler/running/?' do
|
get '/dashboard/stats/vmpooler/running/?' do
|
||||||
content_type :json
|
content_type :json
|
||||||
|
|
||||||
result = {}
|
result = {}
|
||||||
|
|
||||||
Vmpooler::API.settings.config[:pools].each do |pool|
|
Vmpooler::API.settings.config[:pools].each do |pool|
|
||||||
running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
|
running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
|
||||||
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
|
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
|
||||||
|
|
||||||
result[pool['major']] ||= {}
|
result[pool['major']] ||= {}
|
||||||
|
|
||||||
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
|
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
|
||||||
end
|
end
|
||||||
|
|
||||||
if params[:history]
|
if params[:history]
|
||||||
if Vmpooler::API.settings.config[:graphite]['server']
|
if graph_url
|
||||||
begin
|
begin
|
||||||
buffer = open(
|
buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
|
||||||
'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json'
|
|
||||||
).read
|
|
||||||
JSON.parse(buffer).each do |pool|
|
JSON.parse(buffer).each do |pool|
|
||||||
if pool['target'] =~ /.*\.(.*)$/
|
if pool['target'] =~ /.*\.(.*)$/
|
||||||
pool['name'] = Regexp.last_match[1]
|
pool['name'] = Regexp.last_match[1]
|
||||||
|
|
||||||
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
|
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
|
||||||
|
|
||||||
result[pool['major']]['history'] ||= Array.new
|
result[pool['major']]['history'] ||= Array.new
|
||||||
|
|
||||||
for i in 0..pool['datapoints'].length
|
for i in 0..pool['datapoints'].length
|
||||||
if
|
if
|
||||||
pool['datapoints'][i] &&
|
pool['datapoints'][i] &&
|
||||||
pool['datapoints'][i][0]
|
pool['datapoints'][i][0]
|
||||||
|
|
||||||
pool['last'] = pool['datapoints'][i][0]
|
pool['last'] = pool['datapoints'][i][0]
|
||||||
|
|
||||||
result[pool['major']]['history'][i] ||= 0
|
result[pool['major']]['history'][i] ||= 0
|
||||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
|
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
|
||||||
else
|
else
|
||||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
|
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
rescue
|
rescue
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
JSON.pretty_generate(result)
|
JSON.pretty_generate(result)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,10 @@ module Vmpooler
|
||||||
Vmpooler::API.settings.redis
|
Vmpooler::API.settings.redis
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def metrics
|
||||||
|
Vmpooler::API.settings.metrics
|
||||||
|
end
|
||||||
|
|
||||||
def config
|
def config
|
||||||
Vmpooler::API.settings.config[:config]
|
Vmpooler::API.settings.config[:config]
|
||||||
end
|
end
|
||||||
|
|
@ -34,13 +38,11 @@ module Vmpooler
|
||||||
|
|
||||||
def fetch_single_vm(template)
|
def fetch_single_vm(template)
|
||||||
vm = backend.spop('vmpooler__ready__' + template)
|
vm = backend.spop('vmpooler__ready__' + template)
|
||||||
|
|
||||||
return [vm, template] if vm
|
return [vm, template] if vm
|
||||||
|
|
||||||
aliases = Vmpooler::API.settings.config[:alias]
|
aliases = Vmpooler::API.settings.config[:alias]
|
||||||
if aliases && aliased_template = aliases[template]
|
if aliases && aliased_template = aliases[template]
|
||||||
vm = backend.spop('vmpooler__ready__' + aliased_template)
|
vm = backend.spop('vmpooler__ready__' + aliased_template)
|
||||||
|
|
||||||
return [vm, aliased_template] if vm
|
return [vm, aliased_template] if vm
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -85,14 +87,16 @@ module Vmpooler
|
||||||
failed = false
|
failed = false
|
||||||
vms = []
|
vms = []
|
||||||
|
|
||||||
payload.each do |template, count|
|
payload.each do |requested, count|
|
||||||
count.to_i.times do |_i|
|
count.to_i.times do |_i|
|
||||||
vm, name = fetch_single_vm(template)
|
vm, name = fetch_single_vm(requested)
|
||||||
if !vm
|
if !vm
|
||||||
failed = true
|
failed = true
|
||||||
|
metrics.increment('checkout.empty.' + requested)
|
||||||
break
|
break
|
||||||
else
|
else
|
||||||
vms << [ name, vm ]
|
vms << [ name, vm ]
|
||||||
|
metrics.increment('checkout.success.' + name)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -372,9 +376,18 @@ module Vmpooler
|
||||||
|
|
||||||
payload = JSON.parse(request.body.read)
|
payload = JSON.parse(request.body.read)
|
||||||
|
|
||||||
if all_templates_valid?(payload)
|
if payload
|
||||||
|
invalid = invalid_templates(payload)
|
||||||
|
if invalid.empty?
|
||||||
result = atomically_allocate_vms(payload)
|
result = atomically_allocate_vms(payload)
|
||||||
else
|
else
|
||||||
|
invalid.each do |bad_template|
|
||||||
|
metrics.increment('checkout.invalid.' + bad_template)
|
||||||
|
end
|
||||||
|
status 404
|
||||||
|
end
|
||||||
|
else
|
||||||
|
metrics.increment('checkout.invalid.unknown')
|
||||||
status 404
|
status 404
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -392,12 +405,12 @@ module Vmpooler
|
||||||
payload
|
payload
|
||||||
end
|
end
|
||||||
|
|
||||||
def all_templates_valid?(payload)
|
def invalid_templates(payload)
|
||||||
return false unless payload
|
invalid = []
|
||||||
|
payload.keys.each do |template|
|
||||||
payload.keys.all? do |templates|
|
invalid << template unless pool_exists?(template)
|
||||||
pool_exists?(templates)
|
|
||||||
end
|
end
|
||||||
|
invalid
|
||||||
end
|
end
|
||||||
|
|
||||||
post "#{api_prefix}/vm/:template/?" do
|
post "#{api_prefix}/vm/:template/?" do
|
||||||
|
|
@ -406,9 +419,18 @@ module Vmpooler
|
||||||
|
|
||||||
payload = extract_templates_from_query_params(params[:template])
|
payload = extract_templates_from_query_params(params[:template])
|
||||||
|
|
||||||
if all_templates_valid?(payload)
|
if payload
|
||||||
|
invalid = invalid_templates(payload)
|
||||||
|
if invalid.empty?
|
||||||
result = atomically_allocate_vms(payload)
|
result = atomically_allocate_vms(payload)
|
||||||
else
|
else
|
||||||
|
invalid.each do |bad_template|
|
||||||
|
metrics.increment('checkout.invalid.' + bad_template)
|
||||||
|
end
|
||||||
|
status 404
|
||||||
|
end
|
||||||
|
else
|
||||||
|
metrics.increment('checkout.invalid.unknown')
|
||||||
status 404
|
status 404
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
20
lib/vmpooler/dummy_statsd.rb
Normal file
20
lib/vmpooler/dummy_statsd.rb
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
module Vmpooler
|
||||||
|
class DummyStatsd
|
||||||
|
attr_reader :server, :port, :prefix
|
||||||
|
|
||||||
|
def initialize(params = {})
|
||||||
|
end
|
||||||
|
|
||||||
|
def increment(label)
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
def gauge(label, value)
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
def timing(label, duration)
|
||||||
|
true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -2,18 +2,41 @@ require 'rubygems' unless defined?(Gem)
|
||||||
|
|
||||||
module Vmpooler
|
module Vmpooler
|
||||||
class Graphite
|
class Graphite
|
||||||
def initialize(
|
attr_reader :server, :port, :prefix
|
||||||
s = 'graphite'
|
|
||||||
)
|
def initialize(params = {})
|
||||||
@server = s
|
if params["server"].nil? || params["server"].empty?
|
||||||
|
raise ArgumentError, "Graphite server is required. Config: #{params.inspect}"
|
||||||
|
end
|
||||||
|
|
||||||
|
@server = params["server"]
|
||||||
|
@port = params["port"] || 2003
|
||||||
|
@prefix = params["prefix"] || "vmpooler"
|
||||||
|
end
|
||||||
|
|
||||||
|
def increment(label)
|
||||||
|
log label, 1
|
||||||
|
end
|
||||||
|
|
||||||
|
def gauge(label, value)
|
||||||
|
log label, value
|
||||||
|
end
|
||||||
|
|
||||||
|
def timing(label, duration)
|
||||||
|
log label, duration
|
||||||
end
|
end
|
||||||
|
|
||||||
def log(path, value)
|
def log(path, value)
|
||||||
Thread.new do
|
Thread.new do
|
||||||
socket = TCPSocket.new(@server, 2003)
|
socket = TCPSocket.new(server, port)
|
||||||
socket.puts "#{path} #{value} #{Time.now.to_i}"
|
begin
|
||||||
|
socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
|
||||||
|
ensure
|
||||||
socket.close
|
socket.close
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
rescue => err
|
||||||
|
$stderr.puts "Failure logging #{path} to graphite server [#{server}:#{port}]: #{err}"
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,13 @@
|
||||||
module Vmpooler
|
module Vmpooler
|
||||||
class PoolManager
|
class PoolManager
|
||||||
def initialize(config, logger, redis, graphite=nil)
|
def initialize(config, logger, redis, metrics)
|
||||||
$config = config
|
$config = config
|
||||||
|
|
||||||
# Load logger library
|
# Load logger library
|
||||||
$logger = logger
|
$logger = logger
|
||||||
|
|
||||||
unless graphite.nil?
|
# metrics logging handle
|
||||||
$graphite = graphite
|
$metrics = metrics
|
||||||
end
|
|
||||||
|
|
||||||
# Connect to Redis
|
# Connect to Redis
|
||||||
$redis = redis
|
$redis = redis
|
||||||
|
|
@ -63,7 +62,7 @@ module Vmpooler
|
||||||
(host.summary.guest.hostName == vm)
|
(host.summary.guest.hostName == vm)
|
||||||
|
|
||||||
begin
|
begin
|
||||||
Socket.getaddrinfo(vm, nil)
|
Socket.getaddrinfo(vm, nil) # WTF?
|
||||||
rescue
|
rescue
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -257,10 +256,7 @@ module Vmpooler
|
||||||
|
|
||||||
$redis.decr('vmpooler__tasks__clone')
|
$redis.decr('vmpooler__tasks__clone')
|
||||||
|
|
||||||
begin
|
$metrics.timing("clone.#{vm['template']}", finish)
|
||||||
$graphite.log($config[:graphite]['prefix'] + ".clone.#{vm['template']}", finish) if defined? $graphite
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -293,8 +289,7 @@ module Vmpooler
|
||||||
finish = '%.2f' % (Time.now - start)
|
finish = '%.2f' % (Time.now - start)
|
||||||
|
|
||||||
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
||||||
|
$metrics.timing("destroy.#{pool}", finish)
|
||||||
$graphite.log($config[:graphite]['prefix'] + ".destroy.#{pool}", finish) if defined? $graphite
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -564,13 +559,8 @@ module Vmpooler
|
||||||
ready = $redis.scard('vmpooler__ready__' + pool['name'])
|
ready = $redis.scard('vmpooler__ready__' + pool['name'])
|
||||||
total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
|
total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
|
||||||
|
|
||||||
begin
|
$metrics.gauge('ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
|
||||||
if defined? $graphite
|
$metrics.gauge('running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
|
||||||
$graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
|
|
||||||
$graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
|
|
||||||
end
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
|
|
||||||
if $redis.get('vmpooler__empty__' + pool['name'])
|
if $redis.get('vmpooler__empty__' + pool['name'])
|
||||||
unless ready == 0
|
unless ready == 0
|
||||||
|
|
|
||||||
37
lib/vmpooler/statsd.rb
Normal file
37
lib/vmpooler/statsd.rb
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
require 'rubygems' unless defined?(Gem)
|
||||||
|
require 'statsd'
|
||||||
|
|
||||||
|
module Vmpooler
|
||||||
|
class Statsd
|
||||||
|
attr_reader :server, :port, :prefix
|
||||||
|
|
||||||
|
def initialize(params = {})
|
||||||
|
if params["server"].nil? || params["server"].empty?
|
||||||
|
raise ArgumentError, "Statsd server is required. Config: #{params.inspect}"
|
||||||
|
end
|
||||||
|
|
||||||
|
host = params["server"]
|
||||||
|
@port = params["port"] || 8125
|
||||||
|
@prefix = params["prefix"] || 'vmpooler'
|
||||||
|
@server = ::Statsd.new(host, @port)
|
||||||
|
end
|
||||||
|
|
||||||
|
def increment(label)
|
||||||
|
server.increment(prefix + "." + label)
|
||||||
|
rescue => err
|
||||||
|
$stderr.puts "Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def gauge(label, value)
|
||||||
|
server.gauge(prefix + "." + label, value)
|
||||||
|
rescue => err
|
||||||
|
$stderr.puts "Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def timing(label, duration)
|
||||||
|
server.timing(prefix + "." + label, duration)
|
||||||
|
rescue => err
|
||||||
|
$stderr.puts "Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -1,5 +1,10 @@
|
||||||
|
require 'simplecov'
|
||||||
|
SimpleCov.start do
|
||||||
|
add_filter '/spec/'
|
||||||
|
end
|
||||||
require 'helpers'
|
require 'helpers'
|
||||||
require 'rbvmomi'
|
require 'rbvmomi'
|
||||||
require 'rspec'
|
require 'rspec'
|
||||||
require 'vmpooler'
|
require 'vmpooler'
|
||||||
require 'redis'
|
require 'redis'
|
||||||
|
require 'vmpooler/statsd'
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do
|
||||||
|
|
||||||
describe '/vm' do
|
describe '/vm' do
|
||||||
let(:prefix) { '/api/v1' }
|
let(:prefix) { '/api/v1' }
|
||||||
|
let(:metrics) { Vmpooler::DummyStatsd.new }
|
||||||
let(:config) {
|
let(:config) {
|
||||||
{
|
{
|
||||||
config: {
|
config: {
|
||||||
|
|
@ -31,6 +31,7 @@ describe Vmpooler::API::V1 do
|
||||||
{'name' => 'pool1', 'size' => 5},
|
{'name' => 'pool1', 'size' => 5},
|
||||||
{'name' => 'pool2', 'size' => 10}
|
{'name' => 'pool2', 'size' => 10}
|
||||||
],
|
],
|
||||||
|
statsd: { 'prefix' => 'stats_prefix'},
|
||||||
alias: { 'poolone' => 'pool1' },
|
alias: { 'poolone' => 'pool1' },
|
||||||
pool_names: [ 'pool1', 'pool2', 'poolone' ]
|
pool_names: [ 'pool1', 'pool2', 'poolone' ]
|
||||||
}
|
}
|
||||||
|
|
@ -43,6 +44,7 @@ describe Vmpooler::API::V1 do
|
||||||
|
|
||||||
app.settings.set :config, config
|
app.settings.set :config, config
|
||||||
app.settings.set :redis, redis
|
app.settings.set :redis, redis
|
||||||
|
app.settings.set :metrics, metrics
|
||||||
app.settings.set :config, auth: false
|
app.settings.set :config, auth: false
|
||||||
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
|
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do
|
||||||
|
|
||||||
describe '/vm/:template' do
|
describe '/vm/:template' do
|
||||||
let(:prefix) { '/api/v1' }
|
let(:prefix) { '/api/v1' }
|
||||||
|
let(:metrics) { Vmpooler::DummyStatsd.new }
|
||||||
let(:config) {
|
let(:config) {
|
||||||
{
|
{
|
||||||
config: {
|
config: {
|
||||||
|
|
@ -31,6 +31,7 @@ describe Vmpooler::API::V1 do
|
||||||
{'name' => 'pool1', 'size' => 5},
|
{'name' => 'pool1', 'size' => 5},
|
||||||
{'name' => 'pool2', 'size' => 10}
|
{'name' => 'pool2', 'size' => 10}
|
||||||
],
|
],
|
||||||
|
statsd: { 'prefix' => 'stats_prefix'},
|
||||||
alias: { 'poolone' => 'pool1' },
|
alias: { 'poolone' => 'pool1' },
|
||||||
pool_names: [ 'pool1', 'pool2', 'poolone' ]
|
pool_names: [ 'pool1', 'pool2', 'poolone' ]
|
||||||
}
|
}
|
||||||
|
|
@ -43,6 +44,7 @@ describe Vmpooler::API::V1 do
|
||||||
|
|
||||||
app.settings.set :config, config
|
app.settings.set :config, config
|
||||||
app.settings.set :redis, redis
|
app.settings.set :redis, redis
|
||||||
|
app.settings.set :metrics, metrics
|
||||||
app.settings.set :config, auth: false
|
app.settings.set :config, auth: false
|
||||||
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
|
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -4,14 +4,14 @@ require 'time'
|
||||||
describe 'Pool Manager' do
|
describe 'Pool Manager' do
|
||||||
let(:logger) { double('logger') }
|
let(:logger) { double('logger') }
|
||||||
let(:redis) { double('redis') }
|
let(:redis) { double('redis') }
|
||||||
|
let(:metrics) { Vmpooler::DummyStatsd.new }
|
||||||
let(:config) { {} }
|
let(:config) { {} }
|
||||||
let(:graphite) { nil }
|
|
||||||
let(:pool) { 'pool1' }
|
let(:pool) { 'pool1' }
|
||||||
let(:vm) { 'vm1' }
|
let(:vm) { 'vm1' }
|
||||||
let(:timeout) { 5 }
|
let(:timeout) { 5 }
|
||||||
let(:host) { double('host') }
|
let(:host) { double('host') }
|
||||||
|
|
||||||
subject { Vmpooler::PoolManager.new(config, logger, redis, graphite) }
|
subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
|
||||||
|
|
||||||
describe '#_check_pending_vm' do
|
describe '#_check_pending_vm' do
|
||||||
let(:pool_helper) { double('pool') }
|
let(:pool_helper) { double('pool') }
|
||||||
|
|
@ -23,14 +23,12 @@ describe 'Pool Manager' do
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'host not in pool' do
|
context 'host not in pool' do
|
||||||
|
|
||||||
it 'calls fail_pending_vm' do
|
it 'calls fail_pending_vm' do
|
||||||
allow(pool_helper).to receive(:find_vm).and_return(nil)
|
allow(pool_helper).to receive(:find_vm).and_return(nil)
|
||||||
allow(redis).to receive(:hget)
|
allow(redis).to receive(:hget)
|
||||||
expect(redis).to receive(:hget).with(String, 'clone').once
|
expect(redis).to receive(:hget).with(String, 'clone').once
|
||||||
subject._check_pending_vm(vm, pool, timeout)
|
subject._check_pending_vm(vm, pool, timeout)
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'host is in pool' do
|
context 'host is in pool' do
|
||||||
|
|
@ -58,7 +56,6 @@ describe 'Pool Manager' do
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'a host without correct summary' do
|
context 'a host without correct summary' do
|
||||||
|
|
||||||
it 'does nothing when summary is nil' do
|
it 'does nothing when summary is nil' do
|
||||||
allow(host).to receive(:summary).and_return nil
|
allow(host).to receive(:summary).and_return nil
|
||||||
subject.move_pending_vm_to_ready(vm, pool, host)
|
subject.move_pending_vm_to_ready(vm, pool, host)
|
||||||
|
|
@ -114,7 +111,6 @@ describe 'Pool Manager' do
|
||||||
|
|
||||||
subject.move_pending_vm_to_ready(vm, pool, host)
|
subject.move_pending_vm_to_ready(vm, pool, host)
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -191,9 +187,7 @@ describe 'Pool Manager' do
|
||||||
|
|
||||||
subject._check_running_vm(vm, pool, timeout)
|
subject._check_running_vm(vm, pool, timeout)
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '#move_running_to_completed' do
|
describe '#move_running_to_completed' do
|
||||||
|
|
@ -240,15 +234,62 @@ describe 'Pool Manager' do
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'logging' do
|
context 'logging' do
|
||||||
|
|
||||||
it 'logs empty pool' do
|
it 'logs empty pool' do
|
||||||
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
|
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
|
||||||
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
|
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0)
|
||||||
|
|
||||||
expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
|
expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
|
||||||
subject._check_pool(config[:pools][0])
|
subject._check_pool(config[:pools][0])
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#_stats_running_ready' do
|
||||||
|
let(:pool_helper) { double('pool') }
|
||||||
|
let(:vsphere) { {pool => pool_helper} }
|
||||||
|
let(:metrics) { Vmpooler::DummyStatsd.new }
|
||||||
|
let(:config) { {
|
||||||
|
config: { task_limit: 10 },
|
||||||
|
pools: [ {'name' => 'pool1', 'size' => 5} ],
|
||||||
|
graphite: { 'prefix' => 'vmpooler' }
|
||||||
|
} }
|
||||||
|
|
||||||
|
before do
|
||||||
|
expect(subject).not_to be_nil
|
||||||
|
$vsphere = vsphere
|
||||||
|
allow(logger).to receive(:log)
|
||||||
|
allow(pool_helper).to receive(:find_folder)
|
||||||
|
allow(redis).to receive(:smembers).and_return([])
|
||||||
|
allow(redis).to receive(:set)
|
||||||
|
allow(redis).to receive(:get).with('vmpooler__tasks__clone').and_return(0)
|
||||||
|
allow(redis).to receive(:get).with('vmpooler__empty__pool1').and_return(nil)
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'metrics' do
|
||||||
|
subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
|
||||||
|
|
||||||
|
it 'increments metrics' do
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(1)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5)
|
||||||
|
|
||||||
|
expect(metrics).to receive(:gauge).with('ready.pool1', 1)
|
||||||
|
expect(metrics).to receive(:gauge).with('running.pool1', 5)
|
||||||
|
subject._check_pool(config[:pools][0])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'increments metrics when ready with 0 when pool empty' do
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5)
|
||||||
|
|
||||||
|
expect(metrics).to receive(:gauge).with('ready.pool1', 0)
|
||||||
|
expect(metrics).to receive(:gauge).with('running.pool1', 5)
|
||||||
|
subject._check_pool(config[:pools][0])
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -319,5 +360,4 @@ describe 'Pool Manager' do
|
||||||
subject._check_snapshot_queue
|
subject._check_snapshot_queue
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
||||||
8
vmpooler
8
vmpooler
|
|
@ -8,11 +8,12 @@ require 'lib/vmpooler'
|
||||||
config = Vmpooler.config
|
config = Vmpooler.config
|
||||||
redis_host = config[:redis]['server']
|
redis_host = config[:redis]['server']
|
||||||
logger_file = config[:config]['logfile']
|
logger_file = config[:config]['logfile']
|
||||||
graphite = config[:graphite]['server'] ? config[:graphite]['server'] : nil
|
|
||||||
|
metrics = Vmpooler.new_metrics(config)
|
||||||
|
|
||||||
api = Thread.new {
|
api = Thread.new {
|
||||||
thr = Vmpooler::API.new
|
thr = Vmpooler::API.new
|
||||||
thr.helpers.configure(config, Vmpooler.new_redis(redis_host))
|
thr.helpers.configure(config, Vmpooler.new_redis(redis_host), metrics)
|
||||||
thr.helpers.execute!
|
thr.helpers.execute!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -21,9 +22,8 @@ manager = Thread.new {
|
||||||
config,
|
config,
|
||||||
Vmpooler.new_logger(logger_file),
|
Vmpooler.new_logger(logger_file),
|
||||||
Vmpooler.new_redis(redis_host),
|
Vmpooler.new_redis(redis_host),
|
||||||
Vmpooler.new_graphite(graphite)
|
metrics
|
||||||
).execute!
|
).execute!
|
||||||
}
|
}
|
||||||
|
|
||||||
[api, manager].each { |t| t.join }
|
[api, manager].each { |t| t.join }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -53,20 +53,79 @@
|
||||||
:redis:
|
:redis:
|
||||||
server: 'redis.company.com'
|
server: 'redis.company.com'
|
||||||
|
|
||||||
|
|
||||||
|
# :graphs:
|
||||||
|
#
|
||||||
|
# This section contains the server and prefix information for a graphite-
|
||||||
|
# compatible web front-end where graphs may be viewed. This is used by the
|
||||||
|
# vmpooler dashboard to retrieve statistics and graphs for a given instance.
|
||||||
|
#
|
||||||
|
# NOTE: This is not the endpoint for publishing metrics data. See `graphite:`
|
||||||
|
# and `statsd:` below.
|
||||||
|
#
|
||||||
|
# NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be
|
||||||
|
# consulted for `server`/`prefix` information to use in locating a
|
||||||
|
# graph server for our dashboard. `graphs:` is recommended over
|
||||||
|
# `graphite:`
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Available configuration parameters:
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# - server
|
||||||
|
# The FQDN hostname of the statsd daemon.
|
||||||
|
# (required)
|
||||||
|
#
|
||||||
|
# - prefix
|
||||||
|
# The prefix to use while storing statsd data.
|
||||||
|
# (optional; default: 'vmpooler')
|
||||||
|
|
||||||
|
# :statsd:
|
||||||
|
#
|
||||||
|
# This section contains the connection information required to store
|
||||||
|
# historical data via statsd. This is mutually exclusive with graphite
|
||||||
|
# and takes precedence.
|
||||||
|
#
|
||||||
|
# Available configuration parameters:
|
||||||
|
#
|
||||||
|
# - server
|
||||||
|
# The FQDN hostname of the statsd daemon.
|
||||||
|
# (required)
|
||||||
|
#
|
||||||
|
# - prefix
|
||||||
|
# The prefix to use while storing statsd data.
|
||||||
|
# (optional; default: 'vmpooler')
|
||||||
|
#
|
||||||
|
# - port
|
||||||
|
# The UDP port to communicate with the statsd daemon.
|
||||||
|
# (optional; default: 8125)
|
||||||
|
|
||||||
|
# Example:
|
||||||
|
|
||||||
|
:statsd:
|
||||||
|
server: 'statsd.company.com'
|
||||||
|
prefix: 'vmpooler'
|
||||||
|
port: 8125
|
||||||
|
|
||||||
# :graphite:
|
# :graphite:
|
||||||
#
|
#
|
||||||
# This section contains the connection information required to store
|
# This section contains the connection information required to store
|
||||||
# historical data in an external Graphite database.
|
# historical data in an external Graphite database. This is mutually exclusive
|
||||||
|
# with statsd.
|
||||||
#
|
#
|
||||||
# Available configuration parameters:
|
# Available configuration parameters:
|
||||||
#
|
#
|
||||||
# - server
|
# - server
|
||||||
# The FQDN hostname of the Graphite server.
|
# The FQDN hostname of the Graphite server.
|
||||||
# (optional)
|
# (required)
|
||||||
#
|
#
|
||||||
# - prefix
|
# - prefix
|
||||||
# The prefix to use while storing Graphite data.
|
# The prefix to use while storing Graphite data.
|
||||||
# (optional; default: 'vmpooler')
|
# (optional; default: 'vmpooler')
|
||||||
|
#
|
||||||
|
# - port
|
||||||
|
# The TCP port to communicate with the graphite server.
|
||||||
|
# (optional; default: 2003)
|
||||||
|
|
||||||
# Example:
|
# Example:
|
||||||
|
|
||||||
|
|
@ -246,4 +305,3 @@
|
||||||
size: 5
|
size: 5
|
||||||
timeout: 15
|
timeout: 15
|
||||||
ready_ttl: 1440
|
ready_ttl: 1440
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue