puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-27 02:18:41 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #159 from puppetlabs/dont-call-statsd-a-comeback

Browse source 
[QENG-4075] Land statsd support after earlier ci.next changes
This commit is contained in:
Josh Cooper 2016-07-15 12:37:12 -07:00 • committed by GitHub
commit 063d45c60b
15 changed files with 320 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Gemfile
View file

@ -12,10 +12,12 @@ gem 'rbvmomi', '>= 1.8'
gem 'redis', '>= 3.2'
gem 'sinatra', '>= 1.4'
gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3
gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd'
# Test deps
group :test do
gem 'rack-test', '>= 0.6'
gem 'rspec', '>= 3.2'
gem 'simplecov', '>= 0.11.2'
gem 'yarjuf', '>= 2.0'
end

17
lib/vmpooler.rb
View file

@ -12,7 +12,7 @@ module Vmpooler
require 'yaml'
require 'set'
%w( api graphite logger pool_manager vsphere_helper ).each do |lib|
%w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib|
begin
require "vmpooler/#{lib}"
rescue LoadError
@ -53,10 +53,6 @@ module Vmpooler
end
end
if parsed_config[:graphite]['server']
parsed_config[:graphite]['prefix'] ||= 'vmpooler'
end
if parsed_config[:tagfilter]
parsed_config[:tagfilter].keys.each do |tag|
parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
@ -64,7 +60,6 @@ module Vmpooler
end
parsed_config[:uptime] = Time.now
parsed_config
end
@ -76,11 +71,13 @@ module Vmpooler
Vmpooler::Logger.new logfile
end
def self.new_graphite(server)
if server.nil? or server.empty? or server.length == 0
nil
def self.new_metrics(params)
if params[:statsd]
Vmpooler::Statsd.new(params[:statsd])
elsif params[:graphite]
Vmpooler::Graphite.new(params[:graphite])
else
Vmpooler::Graphite.new server
Vmpooler::DummyStatsd.new
end
end

3
lib/vmpooler/api.rb
View file

@ -42,9 +42,10 @@ module Vmpooler
use Vmpooler::API::Reroute
use Vmpooler::API::V1
def configure(config, redis, environment = :production)
def configure(config, redis, metrics, environment = :production)
self.settings.set :config, config
self.settings.set :redis, redis
self.settings.set :metrics, metrics
self.settings.set :environment, environment
end

72
lib/vmpooler/api/dashboard.rb
View file

@ -1,9 +1,57 @@
module Vmpooler
class API
class Dashboard < Sinatra::Base
# handle to the App's configuration information
def config
@config ||= Vmpooler::API.settings.config
end
# configuration setting for server hosting graph URLs to view
def graph_server
return @graph_server if @graph_server
if config[:graphs]
return false unless config[:graphs]['server']
@graph_server = config[:graphs]['server']
elsif config[:graphite]
return false unless config[:graphite]['server']
@graph_server = config[:graphite]['server']
else
false
end
end
# configuration setting for URL prefix for graphs to view
def graph_prefix
return @graph_prefix if @graph_prefix
if config[:graphs]
return "vmpooler" unless config[:graphs]['prefix']
@graph_prefix = config[:graphs]['prefix']
elsif config[:graphite]
return false unless config[:graphite]['prefix']
@graph_prefix = config[:graphite]['prefix']
else
false
end
end
# what is the base URL for viewable graphs?
def graph_url
return false unless graph_server && graph_prefix
@graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
end
# return a full URL to a viewable graph for a given metrics target (graphite syntax)
def graph_link(target = "")
return "" unless graph_url
graph_url + target
end
get '/dashboard/stats/vmpooler/pool/?' do
content_type :json
result = {}
Vmpooler::API.settings.config[:pools].each do |pool|
@ -13,13 +61,11 @@ module Vmpooler
end
if params[:history]
if Vmpooler::API.settings.config[:graphite]['server']
if graph_url
history ||= {}
begin
buffer = open(
'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json'
).read
buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
history = JSON.parse(buffer)
history.each do |pool|
@ -52,59 +98,47 @@ module Vmpooler
end
end
end
JSON.pretty_generate(result)
end
get '/dashboard/stats/vmpooler/running/?' do
content_type :json
result = {}
Vmpooler::API.settings.config[:pools].each do |pool|
running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
result[pool['major']] ||= {}
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
end
if params[:history]
if Vmpooler::API.settings.config[:graphite]['server']
if graph_url
begin
buffer = open(
'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json'
).read
buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
JSON.parse(buffer).each do |pool|
if pool['target'] =~ /.*\.(.*)$/
pool['name'] = Regexp.last_match[1]
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
result[pool['major']]['history'] ||= Array.new
for i in 0..pool['datapoints'].length
if
pool['datapoints'][i] &&
pool['datapoints'][i][0]
pool['last'] = pool['datapoints'][i][0]
result[pool['major']]['history'][i] ||= 0
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
else
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
end
end
end
end
rescue
end
end
end
JSON.pretty_generate(result)
end
end

44
lib/vmpooler/api/v1.rb
View file

@ -12,6 +12,10 @@ module Vmpooler
Vmpooler::API.settings.redis
end
def metrics
Vmpooler::API.settings.metrics
end
def config
Vmpooler::API.settings.config[:config]
end
@ -34,13 +38,11 @@ module Vmpooler
def fetch_single_vm(template)
vm = backend.spop('vmpooler__ready__' + template)
return [vm, template] if vm
aliases = Vmpooler::API.settings.config[:alias]
if aliases && aliased_template = aliases[template]
vm = backend.spop('vmpooler__ready__' + aliased_template)
return [vm, aliased_template] if vm
end
@ -85,14 +87,16 @@ module Vmpooler
failed = false
vms = []
payload.each do |template, count|
payload.each do |requested, count|
count.to_i.times do |_i|
vm, name = fetch_single_vm(template)
vm, name = fetch_single_vm(requested)
if !vm
failed = true
metrics.increment('checkout.empty.' + requested)
break
else
vms << [ name, vm ]
metrics.increment('checkout.success.' + name)
end
end
end
@ -372,9 +376,18 @@ module Vmpooler
payload = JSON.parse(request.body.read)
if all_templates_valid?(payload)
if payload
invalid = invalid_templates(payload)
if invalid.empty?
result = atomically_allocate_vms(payload)
else
invalid.each do |bad_template|
metrics.increment('checkout.invalid.' + bad_template)
end
status 404
end
else
metrics.increment('checkout.invalid.unknown')
status 404
end
@ -392,12 +405,12 @@ module Vmpooler
payload
end
def all_templates_valid?(payload)
return false unless payload
payload.keys.all? do |templates|
pool_exists?(templates)
def invalid_templates(payload)
invalid = []
payload.keys.each do |template|
invalid << template unless pool_exists?(template)
end
invalid
end
post "#{api_prefix}/vm/:template/?" do
@ -406,9 +419,18 @@ module Vmpooler
payload = extract_templates_from_query_params(params[:template])
if all_templates_valid?(payload)
if payload
invalid = invalid_templates(payload)
if invalid.empty?
result = atomically_allocate_vms(payload)
else
invalid.each do |bad_template|
metrics.increment('checkout.invalid.' + bad_template)
end
status 404
end
else
metrics.increment('checkout.invalid.unknown')
status 404
end

20
lib/vmpooler/dummy_statsd.rb Normal file
View file

@ -0,0 +1,20 @@
module Vmpooler
class DummyStatsd
attr_reader :server, :port, :prefix
def initialize(params = {})
end
def increment(label)
true
end
def gauge(label, value)
true
end
def timing(label, duration)
true
end
end
end

35
lib/vmpooler/graphite.rb
View file

@ -2,18 +2,41 @@ require 'rubygems' unless defined?(Gem)
module Vmpooler
class Graphite
def initialize(
s = 'graphite'
)
@server = s
attr_reader :server, :port, :prefix
def initialize(params = {})
if params["server"].nil? || params["server"].empty?
raise ArgumentError, "Graphite server is required. Config: #{params.inspect}"
end
@server = params["server"]
@port = params["port"] || 2003
@prefix = params["prefix"] || "vmpooler"
end
def increment(label)
log label, 1
end
def gauge(label, value)
log label, value
end
def timing(label, duration)
log label, duration
end
def log(path, value)
Thread.new do
socket = TCPSocket.new(@server, 2003)
socket.puts "#{path} #{value} #{Time.now.to_i}"
socket = TCPSocket.new(server, port)
begin
socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}"
ensure
socket.close
end
end
rescue => err
$stderr.puts "Failure logging #{path} to graphite server [#{server}:#{port}]: #{err}"
end
end
end

26
lib/vmpooler/pool_manager.rb
View file

@ -1,14 +1,13 @@
module Vmpooler
class PoolManager
def initialize(config, logger, redis, graphite=nil)
def initialize(config, logger, redis, metrics)
$config = config
# Load logger library
$logger = logger
unless graphite.nil?
$graphite = graphite
end
# metrics logging handle
$metrics = metrics
# Connect to Redis
$redis = redis
@ -63,7 +62,7 @@ module Vmpooler
(host.summary.guest.hostName == vm)
begin
Socket.getaddrinfo(vm, nil)
Socket.getaddrinfo(vm, nil) # WTF?
rescue
end
@ -257,10 +256,7 @@ module Vmpooler
$redis.decr('vmpooler__tasks__clone')
begin
$graphite.log($config[:graphite]['prefix'] + ".clone.#{vm['template']}", finish) if defined? $graphite
rescue
end
$metrics.timing("clone.#{vm['template']}", finish)
end
end
@ -293,8 +289,7 @@ module Vmpooler
finish = '%.2f' % (Time.now - start)
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
$graphite.log($config[:graphite]['prefix'] + ".destroy.#{pool}", finish) if defined? $graphite
$metrics.timing("destroy.#{pool}", finish)
end
end
end
@ -564,13 +559,8 @@ module Vmpooler
ready = $redis.scard('vmpooler__ready__' + pool['name'])
total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
begin
if defined? $graphite
$graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
$graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
end
rescue
end
$metrics.gauge('ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
$metrics.gauge('running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
if $redis.get('vmpooler__empty__' + pool['name'])
unless ready == 0

37
lib/vmpooler/statsd.rb Normal file
View file

@ -0,0 +1,37 @@
require 'rubygems' unless defined?(Gem)
require 'statsd'
module Vmpooler
class Statsd
attr_reader :server, :port, :prefix
def initialize(params = {})
if params["server"].nil? || params["server"].empty?
raise ArgumentError, "Statsd server is required. Config: #{params.inspect}"
end
host = params["server"]
@port = params["port"] || 8125
@prefix = params["prefix"] || 'vmpooler'
@server = ::Statsd.new(host, @port)
end
def increment(label)
server.increment(prefix + "." + label)
rescue => err
$stderr.puts "Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
end
def gauge(label, value)
server.gauge(prefix + "." + label, value)
rescue => err
$stderr.puts "Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
end
def timing(label, duration)
server.timing(prefix + "." + label, duration)
rescue => err
$stderr.puts "Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}"
end
end
end

5
spec/spec_helper.rb
View file

@ -1,5 +1,10 @@
require 'simplecov'
SimpleCov.start do
add_filter '/spec/'
end
require 'helpers'
require 'rbvmomi'
require 'rspec'
require 'vmpooler'
require 'redis'
require 'vmpooler/statsd'

4
spec/vmpooler/api/v1/vm_spec.rb
View file

@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do
describe '/vm' do
let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new }
let(:config) {
{
config: {
@ -31,6 +31,7 @@ describe Vmpooler::API::V1 do
{'name' => 'pool1', 'size' => 5},
{'name' => 'pool2', 'size' => 10}
],
statsd: { 'prefix' => 'stats_prefix'},
alias: { 'poolone' => 'pool1' },
pool_names: [ 'pool1', 'pool2', 'poolone' ]
}
@ -43,6 +44,7 @@ describe Vmpooler::API::V1 do
app.settings.set :config, config
app.settings.set :redis, redis
app.settings.set :metrics, metrics
app.settings.set :config, auth: false
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
end

4
spec/vmpooler/api/v1/vm_template_spec.rb
View file

@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do
describe '/vm/:template' do
let(:prefix) { '/api/v1' }
let(:metrics) { Vmpooler::DummyStatsd.new }
let(:config) {
{
config: {
@ -31,6 +31,7 @@ describe Vmpooler::API::V1 do
{'name' => 'pool1', 'size' => 5},
{'name' => 'pool2', 'size' => 10}
],
statsd: { 'prefix' => 'stats_prefix'},
alias: { 'poolone' => 'pool1' },
pool_names: [ 'pool1', 'pool2', 'poolone' ]
}
@ -43,6 +44,7 @@ describe Vmpooler::API::V1 do
app.settings.set :config, config
app.settings.set :redis, redis
app.settings.set :metrics, metrics
app.settings.set :config, auth: false
create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
end

60
spec/vmpooler/pool_manager_spec.rb
View file

@ -4,14 +4,14 @@ require 'time'
describe 'Pool Manager' do
let(:logger) { double('logger') }
let(:redis) { double('redis') }
let(:metrics) { Vmpooler::DummyStatsd.new }
let(:config) { {} }
let(:graphite) { nil }
let(:pool) { 'pool1' }
let(:vm) { 'vm1' }
let(:timeout) { 5 }
let(:host) { double('host') }
subject { Vmpooler::PoolManager.new(config, logger, redis, graphite) }
subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
describe '#_check_pending_vm' do
let(:pool_helper) { double('pool') }
@ -23,14 +23,12 @@ describe 'Pool Manager' do
end
context 'host not in pool' do
it 'calls fail_pending_vm' do
allow(pool_helper).to receive(:find_vm).and_return(nil)
allow(redis).to receive(:hget)
expect(redis).to receive(:hget).with(String, 'clone').once
subject._check_pending_vm(vm, pool, timeout)
end
end
context 'host is in pool' do
@ -58,7 +56,6 @@ describe 'Pool Manager' do
end
context 'a host without correct summary' do
it 'does nothing when summary is nil' do
allow(host).to receive(:summary).and_return nil
subject.move_pending_vm_to_ready(vm, pool, host)
@ -114,7 +111,6 @@ describe 'Pool Manager' do
subject.move_pending_vm_to_ready(vm, pool, host)
end
end
end
@ -191,9 +187,7 @@ describe 'Pool Manager' do
subject._check_running_vm(vm, pool, timeout)
end
end
end
describe '#move_running_to_completed' do
@ -240,15 +234,62 @@ describe 'Pool Manager' do
end
context 'logging' do
it 'logs empty pool' do
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0)
expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
subject._check_pool(config[:pools][0])
end
end
end
describe '#_stats_running_ready' do
let(:pool_helper) { double('pool') }
let(:vsphere) { {pool => pool_helper} }
let(:metrics) { Vmpooler::DummyStatsd.new }
let(:config) { {
config: { task_limit: 10 },
pools: [ {'name' => 'pool1', 'size' => 5} ],
graphite: { 'prefix' => 'vmpooler' }
} }
before do
expect(subject).not_to be_nil
$vsphere = vsphere
allow(logger).to receive(:log)
allow(pool_helper).to receive(:find_folder)
allow(redis).to receive(:smembers).and_return([])
allow(redis).to receive(:set)
allow(redis).to receive(:get).with('vmpooler__tasks__clone').and_return(0)
allow(redis).to receive(:get).with('vmpooler__empty__pool1').and_return(nil)
end
context 'metrics' do
subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
it 'increments metrics' do
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(1)
allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5)
expect(metrics).to receive(:gauge).with('ready.pool1', 1)
expect(metrics).to receive(:gauge).with('running.pool1', 5)
subject._check_pool(config[:pools][0])
end
it 'increments metrics when ready with 0 when pool empty' do
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5)
expect(metrics).to receive(:gauge).with('ready.pool1', 0)
expect(metrics).to receive(:gauge).with('running.pool1', 5)
subject._check_pool(config[:pools][0])
end
end
end
@ -319,5 +360,4 @@ describe 'Pool Manager' do
subject._check_snapshot_queue
end
end
end

8
vmpooler
View file

@ -8,11 +8,12 @@ require 'lib/vmpooler'
config = Vmpooler.config
redis_host = config[:redis]['server']
logger_file = config[:config]['logfile']
graphite = config[:graphite]['server'] ? config[:graphite]['server'] : nil
metrics = Vmpooler.new_metrics(config)
api = Thread.new {
thr = Vmpooler::API.new
thr.helpers.configure(config, Vmpooler.new_redis(redis_host))
thr.helpers.configure(config, Vmpooler.new_redis(redis_host), metrics)
thr.helpers.execute!
}
@ -21,9 +22,8 @@ manager = Thread.new {
config,
Vmpooler.new_logger(logger_file),
Vmpooler.new_redis(redis_host),
Vmpooler.new_graphite(graphite)
metrics
).execute!
}
[api, manager].each { |t| t.join }

64
vmpooler.yaml.example
View file

@ -53,20 +53,79 @@
:redis:
server: 'redis.company.com'
# :graphs:
#
# This section contains the server and prefix information for a graphite-
# compatible web front-end where graphs may be viewed. This is used by the
# vmpooler dashboard to retrieve statistics and graphs for a given instance.
#
# NOTE: This is not the endpoint for publishing metrics data. See `graphite:`
# and `statsd:` below.
#
# NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be
# consulted for `server`/`prefix` information to use in locating a
# graph server for our dashboard. `graphs:` is recommended over
# `graphite:`
#
#
# Available configuration parameters:
#
#
# - server
# The FQDN hostname of the statsd daemon.
# (required)
#
# - prefix
# The prefix to use while storing statsd data.
# (optional; default: 'vmpooler')
# :statsd:
#
# This section contains the connection information required to store
# historical data via statsd. This is mutually exclusive with graphite
# and takes precedence.
#
# Available configuration parameters:
#
# - server
# The FQDN hostname of the statsd daemon.
# (required)
#
# - prefix
# The prefix to use while storing statsd data.
# (optional; default: 'vmpooler')
#
# - port
# The UDP port to communicate with the statsd daemon.
# (optional; default: 8125)
# Example:
:statsd:
server: 'statsd.company.com'
prefix: 'vmpooler'
port: 8125
# :graphite:
#
# This section contains the connection information required to store
# historical data in an external Graphite database.
# historical data in an external Graphite database. This is mutually exclusive
# with statsd.
#
# Available configuration parameters:
#
# - server
# The FQDN hostname of the Graphite server.
# (optional)
# (required)
#
# - prefix
# The prefix to use while storing Graphite data.
# (optional; default: 'vmpooler')
#
# - port
# The TCP port to communicate with the graphite server.
# (optional; default: 2003)
# Example:
@ -246,4 +305,3 @@
size: 5
timeout: 15
ready_ttl: 1440