diff --git a/API.md b/API.md index ebec9d2..bbab501 100644 --- a/API.md +++ b/API.md @@ -117,6 +117,8 @@ $ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.company.c } ``` +**NOTE: Returns either all requested VMs or no VMs.** + ##### POST /vm/<pool> Check-out a VM or VMs. @@ -156,6 +158,8 @@ $ curl -d --url vmpooler.company.com/api/v1/vm/debian-7-i386+debian-7-i386+debia } ``` +**NOTE: Returns either all requested VMs or no VMs.** + ##### GET /vm/<hostname> Query a checked-out VM. diff --git a/Gemfile b/Gemfile index 724b058..9c9a05c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,15 +1,23 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' -gem 'json', '>= 1.8' +if RUBY_VERSION =~ /^1\.9\./ + gem 'json', '~> 1.8' +else + gem 'json', '>= 1.8' +end + gem 'rack', '>= 1.6' gem 'rake', '>= 10.4' gem 'rbvmomi', '>= 1.8' gem 'redis', '>= 3.2' gem 'sinatra', '>= 1.4' +gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3 +gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd' # Test deps group :test do gem 'rack-test', '>= 0.6' gem 'rspec', '>= 3.2' + gem 'simplecov', '>= 0.11.2' gem 'yarjuf', '>= 2.0' end diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 70bb819..0e5707c 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -10,8 +10,9 @@ module Vmpooler require 'time' require 'timeout' require 'yaml' + require 'set' - %w( api graphite logger pool_manager vsphere_helper ).each do |lib| + %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib| begin require "vmpooler/#{lib}" rescue LoadError @@ -35,23 +36,23 @@ module Vmpooler parsed_config[:config]['prefix'] ||= '' # Create an index of pool aliases + parsed_config[:pool_names] = Set.new parsed_config[:pools].each do |pool| + parsed_config[:pool_names] << pool['name'] if pool['alias'] if pool['alias'].kind_of?(Array) pool['alias'].each do |a| parsed_config[:alias] ||= {} parsed_config[:alias][a] = pool['name'] + parsed_config[:pool_names] << a end elsif pool['alias'].kind_of?(String) parsed_config[:alias][pool['alias']] = pool['name'] + parsed_config[:pool_names] << pool['alias'] end end end - if parsed_config[:graphite]['server'] - parsed_config[:graphite]['prefix'] ||= 'vmpooler' - end - if parsed_config[:tagfilter] parsed_config[:tagfilter].keys.each do |tag| parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag]) @@ -59,7 +60,6 @@ module Vmpooler end parsed_config[:uptime] = Time.now - parsed_config end @@ -71,11 +71,13 @@ module Vmpooler Vmpooler::Logger.new logfile end - def self.new_graphite(server) - if server.nil? or server.empty? or server.length == 0 - nil + def self.new_metrics(params) + if params[:statsd] + Vmpooler::Statsd.new(params[:statsd]) + elsif params[:graphite] + Vmpooler::Graphite.new(params[:graphite]) else - Vmpooler::Graphite.new server + Vmpooler::DummyStatsd.new end end diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index ee51cbc..68649e9 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -42,9 +42,10 @@ module Vmpooler use Vmpooler::API::Reroute use Vmpooler::API::V1 - def configure(config, redis, environment = :production) + def configure(config, redis, metrics, environment = :production) self.settings.set :config, config self.settings.set :redis, redis + self.settings.set :metrics, metrics self.settings.set :environment, environment end diff --git a/lib/vmpooler/api/dashboard.rb b/lib/vmpooler/api/dashboard.rb index b188dc5..297ade3 100644 --- a/lib/vmpooler/api/dashboard.rb +++ b/lib/vmpooler/api/dashboard.rb @@ -1,9 +1,57 @@ module Vmpooler class API class Dashboard < Sinatra::Base + + # handle to the App's configuration information + def config + @config ||= Vmpooler::API.settings.config + end + + # configuration setting for server hosting graph URLs to view + def graph_server + return @graph_server if @graph_server + + if config[:graphs] + return false unless config[:graphs]['server'] + @graph_server = config[:graphs]['server'] + elsif config[:graphite] + return false unless config[:graphite]['server'] + @graph_server = config[:graphite]['server'] + else + false + end + end + + # configuration setting for URL prefix for graphs to view + def graph_prefix + return @graph_prefix if @graph_prefix + + if config[:graphs] + return "vmpooler" unless config[:graphs]['prefix'] + @graph_prefix = config[:graphs]['prefix'] + elsif config[:graphite] + return false unless config[:graphite]['prefix'] + @graph_prefix = config[:graphite]['prefix'] + else + false + end + end + + # what is the base URL for viewable graphs? + def graph_url + return false unless graph_server && graph_prefix + @graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}" + end + + # return a full URL to a viewable graph for a given metrics target (graphite syntax) + def graph_link(target = "") + return "" unless graph_url + graph_url + target + end + + get '/dashboard/stats/vmpooler/pool/?' do content_type :json - result = {} Vmpooler::API.settings.config[:pools].each do |pool| @@ -13,13 +61,11 @@ module Vmpooler end if params[:history] - if Vmpooler::API.settings.config[:graphite]['server'] + if graph_url history ||= {} begin - buffer = open( - 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json' - ).read + buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read history = JSON.parse(buffer) history.each do |pool| @@ -52,59 +98,47 @@ module Vmpooler end end end - JSON.pretty_generate(result) end get '/dashboard/stats/vmpooler/running/?' do content_type :json - result = {} Vmpooler::API.settings.config[:pools].each do |pool| running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name']) pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/ - result[pool['major']] ||= {} - result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i end if params[:history] - if Vmpooler::API.settings.config[:graphite]['server'] + if graph_url begin - buffer = open( - 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json' - ).read + buffer = open(graph_link('.running.*&from=-1hour&format=json')).read JSON.parse(buffer).each do |pool| if pool['target'] =~ /.*\.(.*)$/ pool['name'] = Regexp.last_match[1] - pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/ - result[pool['major']]['history'] ||= Array.new for i in 0..pool['datapoints'].length if pool['datapoints'][i] && pool['datapoints'][i][0] - pool['last'] = pool['datapoints'][i][0] - result[pool['major']]['history'][i] ||= 0 result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i else result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i end end - end end rescue end end end - JSON.pretty_generate(result) end end diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index ccd8c9a..4bd9852 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -12,6 +12,10 @@ module Vmpooler Vmpooler::API.settings.redis end + def metrics + Vmpooler::API.settings.metrics + end + def config Vmpooler::API.settings.config[:config] end @@ -20,6 +24,10 @@ module Vmpooler Vmpooler::API.settings.config[:pools] end + def pool_exists?(template) + Vmpooler::API.settings.config[:pool_names].include?(template) + end + def need_auth! validate_auth(backend) end @@ -28,55 +36,84 @@ module Vmpooler validate_token(backend) end - def alias_deref(hash) - newhash = {} + def fetch_single_vm(template) + vm = backend.spop('vmpooler__ready__' + template) + return [vm, template] if vm - hash.each do |key, val| - if backend.exists('vmpooler__ready__' + key) - newhash[key] = val - else - if Vmpooler::API.settings.config[:alias][key] - newkey = Vmpooler::API.settings.config[:alias][key] - newhash[newkey] = val + aliases = Vmpooler::API.settings.config[:alias] + if aliases && aliased_template = aliases[template] + vm = backend.spop('vmpooler__ready__' + aliased_template) + return [vm, aliased_template] if vm + end + + [nil, nil] + end + + def return_vm_to_ready_state(template, vm) + backend.sadd('vmpooler__ready__' + template, vm) + end + + def account_for_starting_vm(template, vm) + backend.sadd('vmpooler__running__' + template, vm) + backend.hset('vmpooler__active__' + template, vm, Time.now) + backend.hset('vmpooler__vm__' + vm, 'checkout', Time.now) + + if Vmpooler::API.settings.config[:auth] and has_token? + validate_token(backend) + + backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN']) + backend.hset('vmpooler__vm__' + vm, 'token:user', + backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user') + ) + + if config['vm_lifetime_auth'].to_i > 0 + backend.hset('vmpooler__vm__' + vm, 'lifetime', config['vm_lifetime_auth'].to_i) + end + end + end + + def update_result_hosts(result, template, vm) + result[template] ||= {} + if result[template]['hostname'] + result[template]['hostname'] = Array(result[template]['hostname']) + result[template]['hostname'].push(vm) + else + result[template]['hostname'] = vm + end + end + + def atomically_allocate_vms(payload) + result = { 'ok' => false } + failed = false + vms = [] + + payload.each do |requested, count| + count.to_i.times do |_i| + vm, name = fetch_single_vm(requested) + if !vm + failed = true + metrics.increment('checkout.empty.' + requested) + break + else + vms << [ name, vm ] + metrics.increment('checkout.success.' + name) end end end - newhash - end - - def checkout_vm(template, result) - vm = backend.spop('vmpooler__ready__' + template) - - unless vm.nil? - backend.sadd('vmpooler__running__' + template, vm) - backend.hset('vmpooler__active__' + template, vm, Time.now) - backend.hset('vmpooler__vm__' + vm, 'checkout', Time.now) - - if Vmpooler::API.settings.config[:auth] and has_token? - validate_token(backend) - - backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN']) - backend.hset('vmpooler__vm__' + vm, 'token:user', - backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user') - ) - - if config['vm_lifetime_auth'].to_i > 0 - backend.hset('vmpooler__vm__' + vm, 'lifetime', config['vm_lifetime_auth'].to_i) - end + if failed + vms.each do |(name, vm)| + return_vm_to_ready_state(name, vm) end - - result[template] ||= {} - - if result[template]['hostname'] - result[template]['hostname'] = [result[template]['hostname']] unless result[template]['hostname'].is_a?(Array) - result[template]['hostname'].push(vm) - else - result[template]['hostname'] = vm - end - else status 503 - result['ok'] = false + else + vms.each do |(name, vm)| + account_for_starting_vm(name, vm) + update_result_hosts(result, name, vm) + end + + result['ok'] = true + result['domain'] = config['domain'] if config['domain'] end result @@ -335,77 +372,66 @@ module Vmpooler post "#{api_prefix}/vm/?" do content_type :json - result = { 'ok' => false } - jdata = alias_deref(JSON.parse(request.body.read)) + payload = JSON.parse(request.body.read) - if not jdata.nil? and not jdata.empty? - available = 1 - else - status 404 - end - - jdata.each do |key, val| - if backend.scard('vmpooler__ready__' + key).to_i < val.to_i - available = 0 - end - end - - if (available == 1) - result['ok'] = true - - jdata.each do |key, val| - val.to_i.times do |_i| - result = checkout_vm(key, result) + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment('checkout.invalid.' + bad_template) end + status 404 end - end - - if result['ok'] && config['domain'] - result['domain'] = config['domain'] + else + metrics.increment('checkout.invalid.unknown') + status 404 end JSON.pretty_generate(result) end - post "#{api_prefix}/vm/:template/?" do - content_type :json - - result = { 'ok' => false } + def extract_templates_from_query_params(params) payload = {} - params[:template].split('+').each do |template| + params.split('+').each do |template| payload[template] ||= 0 - payload[template] = payload[template] + 1 + payload[template] += 1 end - payload = alias_deref(payload) + payload + end - if not payload.nil? and not payload.empty? - available = 1 - else - status 404 + def invalid_templates(payload) + invalid = [] + payload.keys.each do |template| + invalid << template unless pool_exists?(template) end + invalid + end - payload.each do |key, val| - if backend.scard('vmpooler__ready__' + key).to_i < val.to_i - available = 0 - end - end + post "#{api_prefix}/vm/:template/?" do + content_type :json + result = { 'ok' => false } - if (available == 1) - result['ok'] = true + payload = extract_templates_from_query_params(params[:template]) - payload.each do |key, val| - val.to_i.times do |_i| - result = checkout_vm(key, result) + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = atomically_allocate_vms(payload) + else + invalid.each do |bad_template| + metrics.increment('checkout.invalid.' + bad_template) end + status 404 end - end - - if result['ok'] && config['domain'] - result['domain'] = config['domain'] + else + metrics.increment('checkout.invalid.unknown') + status 404 end JSON.pretty_generate(result) diff --git a/lib/vmpooler/dummy_statsd.rb b/lib/vmpooler/dummy_statsd.rb new file mode 100644 index 0000000..2b1b621 --- /dev/null +++ b/lib/vmpooler/dummy_statsd.rb @@ -0,0 +1,20 @@ +module Vmpooler + class DummyStatsd + attr_reader :server, :port, :prefix + + def initialize(params = {}) + end + + def increment(label) + true + end + + def gauge(label, value) + true + end + + def timing(label, duration) + true + end + end +end diff --git a/lib/vmpooler/graphite.rb b/lib/vmpooler/graphite.rb index 128d07b..d7d1a0d 100644 --- a/lib/vmpooler/graphite.rb +++ b/lib/vmpooler/graphite.rb @@ -2,18 +2,41 @@ require 'rubygems' unless defined?(Gem) module Vmpooler class Graphite - def initialize( - s = 'graphite' - ) - @server = s + attr_reader :server, :port, :prefix + + def initialize(params = {}) + if params["server"].nil? || params["server"].empty? + raise ArgumentError, "Graphite server is required. Config: #{params.inspect}" + end + + @server = params["server"] + @port = params["port"] || 2003 + @prefix = params["prefix"] || "vmpooler" + end + + def increment(label) + log label, 1 + end + + def gauge(label, value) + log label, value + end + + def timing(label, duration) + log label, duration end def log(path, value) Thread.new do - socket = TCPSocket.new(@server, 2003) - socket.puts "#{path} #{value} #{Time.now.to_i}" - socket.close + socket = TCPSocket.new(server, port) + begin + socket.puts "#{prefix}.#{path} #{value} #{Time.now.to_i}" + ensure + socket.close + end end + rescue => err + $stderr.puts "Failure logging #{path} to graphite server [#{server}:#{port}]: #{err}" end end end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 1a0454c..53895bb 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -1,14 +1,13 @@ module Vmpooler class PoolManager - def initialize(config, logger, redis, graphite=nil) + def initialize(config, logger, redis, metrics) $config = config # Load logger library $logger = logger - unless graphite.nil? - $graphite = graphite - end + # metrics logging handle + $metrics = metrics # Connect to Redis $redis = redis @@ -63,7 +62,7 @@ module Vmpooler (host.summary.guest.hostName == vm) begin - Socket.getaddrinfo(vm, nil) + Socket.getaddrinfo(vm, nil) # WTF? rescue end @@ -257,10 +256,7 @@ module Vmpooler $redis.decr('vmpooler__tasks__clone') - begin - $graphite.log($config[:graphite]['prefix'] + ".clone.#{vm['template']}", finish) if defined? $graphite - rescue - end + $metrics.timing("clone.#{vm['template']}", finish) end end @@ -293,8 +289,7 @@ module Vmpooler finish = '%.2f' % (Time.now - start) $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") - - $graphite.log($config[:graphite]['prefix'] + ".destroy.#{pool}", finish) if defined? $graphite + $metrics.timing("destroy.#{pool}", finish) end end end @@ -564,13 +559,8 @@ module Vmpooler ready = $redis.scard('vmpooler__ready__' + pool['name']) total = $redis.scard('vmpooler__pending__' + pool['name']) + ready - begin - if defined? $graphite - $graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name'])) - $graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name'])) - end - rescue - end + $metrics.gauge('ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name'])) + $metrics.gauge('running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name'])) if $redis.get('vmpooler__empty__' + pool['name']) unless ready == 0 diff --git a/lib/vmpooler/statsd.rb b/lib/vmpooler/statsd.rb new file mode 100644 index 0000000..94a4065 --- /dev/null +++ b/lib/vmpooler/statsd.rb @@ -0,0 +1,37 @@ +require 'rubygems' unless defined?(Gem) +require 'statsd' + +module Vmpooler + class Statsd + attr_reader :server, :port, :prefix + + def initialize(params = {}) + if params["server"].nil? || params["server"].empty? + raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" + end + + host = params["server"] + @port = params["port"] || 8125 + @prefix = params["prefix"] || 'vmpooler' + @server = ::Statsd.new(host, @port) + end + + def increment(label) + server.increment(prefix + "." + label) + rescue => err + $stderr.puts "Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}" + end + + def gauge(label, value) + server.gauge(prefix + "." + label, value) + rescue => err + $stderr.puts "Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}" + end + + def timing(label, duration) + server.timing(prefix + "." + label, duration) + rescue => err + $stderr.puts "Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{err}" + end + end +end diff --git a/spec/helpers.rb b/spec/helpers.rb index d28b761..292b9f4 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -35,6 +35,7 @@ end def create_ready_vm(template, name, token = nil) create_vm(name, token) redis.sadd("vmpooler__ready__#{template}", name) + # REMIND: should be __vm__? redis.hset("vmpooler_vm_#{name}", "template", template) end @@ -73,3 +74,7 @@ def vm_reverted_to_snapshot?(vm, snapshot = nil) instance == vm and (snapshot ? (sha == snapshot) : true) end end + +def pool_has_ready_vm?(pool, vm) + !!redis.sismember('vmpooler__ready__' + pool, vm) +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7589276..83a5cdf 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,10 @@ +require 'simplecov' +SimpleCov.start do + add_filter '/spec/' +end require 'helpers' require 'rbvmomi' require 'rspec' require 'vmpooler' require 'redis' +require 'vmpooler/statsd' diff --git a/spec/vmpooler/api/v1/vm_spec.rb b/spec/vmpooler/api/v1/vm_spec.rb index 6a4a068..0c14561 100644 --- a/spec/vmpooler/api/v1/vm_spec.rb +++ b/spec/vmpooler/api/v1/vm_spec.rb @@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do describe '/vm' do let(:prefix) { '/api/v1' } - + let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { @@ -31,7 +31,9 @@ describe Vmpooler::API::V1 do {'name' => 'pool1', 'size' => 5}, {'name' => 'pool2', 'size' => 10} ], + statsd: { 'prefix' => 'stats_prefix'}, alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] } } @@ -42,6 +44,7 @@ describe Vmpooler::API::V1 do app.settings.set :config, config app.settings.set :redis, redis + app.settings.set :metrics, metrics app.settings.set :config, auth: false create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) end @@ -84,6 +87,31 @@ describe Vmpooler::API::V1 do expect_json(ok = false, http = 404) end + it 'returns 503 for empty pool when aliases are not defined' do + Vmpooler::API.settings.config.delete(:alias) + Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2'] + + create_ready_vm 'pool1', 'abcdefghijklmnop' + post "#{prefix}/vm/pool1" + post "#{prefix}/vm/pool1" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns 503 for empty pool referenced by alias' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + post "#{prefix}/vm/poolone" + post "#{prefix}/vm/poolone" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + it 'returns multiple VMs' do create_ready_vm 'pool1', 'abcdefghijklmnop' create_ready_vm 'pool2', 'qrstuvwxyz012345' @@ -104,6 +132,132 @@ describe Vmpooler::API::V1 do expect(last_response.body).to eq(JSON.pretty_generate(expected)) end + it 'returns multiple VMs even when multiple instances from the same pool are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + create_ready_vm 'pool1', '2abcdefghijklmnop' + create_ready_vm 'pool2', 'qrstuvwxyz012345' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: 'qrstuvwxyz012345' + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop') + expect(result['pool2']['hostname']).to eq('qrstuvwxyz012345') + + expect_json(ok = true, http = 200) + end + + it 'returns multiple VMs even when multiple instances from multiple pools are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + create_ready_vm 'pool1', '2abcdefghijklmnop' + create_ready_vm 'pool2', '1qrstuvwxyz012345' + create_ready_vm 'pool2', '2qrstuvwxyz012345' + create_ready_vm 'pool2', '3qrstuvwxyz012345' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ] + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop') + expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345') + + expect_json(ok = true, http = 200) + end + + it 'fails when not all requested vms can be allocated' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"1","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"1"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + create_ready_vm 'pool1', '2abcdefghijklmnop' + + post "#{prefix}/vm", '{"pool1":"2","pool2":"3"}' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', '1abcdefghijklmnop')).to eq(true) + expect(pool_has_ready_vm?('pool1', '2abcdefghijklmnop')).to eq(true) + end + context '(auth not configured)' do it 'does not extend VM lifetime if auth token is provided' do app.settings.set :config, auth: false diff --git a/spec/vmpooler/api/v1/vm_template_spec.rb b/spec/vmpooler/api/v1/vm_template_spec.rb index 28b85c3..aaabf4a 100644 --- a/spec/vmpooler/api/v1/vm_template_spec.rb +++ b/spec/vmpooler/api/v1/vm_template_spec.rb @@ -20,7 +20,7 @@ describe Vmpooler::API::V1 do describe '/vm/:template' do let(:prefix) { '/api/v1' } - + let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { @@ -31,7 +31,9 @@ describe Vmpooler::API::V1 do {'name' => 'pool1', 'size' => 5}, {'name' => 'pool2', 'size' => 10} ], + statsd: { 'prefix' => 'stats_prefix'}, alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] } } @@ -42,6 +44,7 @@ describe Vmpooler::API::V1 do app.settings.set :config, config app.settings.set :redis, redis + app.settings.set :metrics, metrics app.settings.set :config, auth: false create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) end @@ -84,6 +87,31 @@ describe Vmpooler::API::V1 do expect_json(ok = false, http = 404) end + it 'returns 503 for empty pool when aliases are not defined' do + Vmpooler::API.settings.config.delete(:alias) + Vmpooler::API.settings.config[:pool_names] = ['pool1', 'pool2'] + + create_ready_vm 'pool1', 'abcdefghijklmnop' + post "#{prefix}/vm/pool1" + post "#{prefix}/vm/pool1" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns 503 for empty pool referenced by alias' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + post "#{prefix}/vm/poolone" + post "#{prefix}/vm/poolone" + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + it 'returns multiple VMs' do create_ready_vm 'pool1', 'abcdefghijklmnop' create_ready_vm 'pool2', 'qrstuvwxyz012345' @@ -104,6 +132,111 @@ describe Vmpooler::API::V1 do expect(last_response.body).to eq(JSON.pretty_generate(expected)) end + it 'returns multiple VMs even when multiple instances from multiple pools are requested' do + create_ready_vm 'pool1', '1abcdefghijklmnop' + create_ready_vm 'pool1', '2abcdefghijklmnop' + + create_ready_vm 'pool2', '1qrstuvwxyz012345' + create_ready_vm 'pool2', '2qrstuvwxyz012345' + create_ready_vm 'pool2', '3qrstuvwxyz012345' + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { + ok: true, + pool1: { + hostname: [ '1abcdefghijklmnop', '2abcdefghijklmnop' ] + }, + pool2: { + hostname: [ '1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345' ] + } + } + + result = JSON.parse(last_response.body) + expect(result['ok']).to eq(true) + expect(result['pool1']['hostname']).to include('1abcdefghijklmnop', '2abcdefghijklmnop') + expect(result['pool2']['hostname']).to include('1qrstuvwxyz012345', '2qrstuvwxyz012345', '3qrstuvwxyz012345') + expect_json(ok = true, http = 200) + end + + it 'fails when not all requested vms can be allocated' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + + post "#{prefix}/vm/pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + + post "#{prefix}/vm/pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + create_ready_vm 'pool1', '0123456789012345' + + post "#{prefix}/vm/pool1+pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from a pool' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + create_ready_vm 'pool1', '0123456789012345' + + post "#{prefix}/vm/pool1+pool1+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true) + expect(pool_has_ready_vm?('pool1', '0123456789012345')).to eq(true) + end + + it 'fails when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + create_ready_vm 'pool2', '0123456789012345' + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + end + + it 'returns any checked out vms to their pools when not all requested vms can be allocated, when requesting multiple instances from multiple pools' do + create_ready_vm 'pool1', 'abcdefghijklmnop' + create_ready_vm 'pool2', '0123456789012345' + + post "#{prefix}/vm/pool1+pool1+pool2+pool2+pool2", '' + + expected = { ok: false } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + expect_json(ok = false, http = 503) + + expect(pool_has_ready_vm?('pool1', 'abcdefghijklmnop')).to eq(true) + expect(pool_has_ready_vm?('pool2', '0123456789012345')).to eq(true) + end + context '(auth not configured)' do it 'does not extend VM lifetime if auth token is provided' do app.settings.set :config, auth: false diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index a402bf4..c569578 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -4,14 +4,14 @@ require 'time' describe 'Pool Manager' do let(:logger) { double('logger') } let(:redis) { double('redis') } + let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { {} } - let(:graphite) { nil } let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } let(:host) { double('host') } - subject { Vmpooler::PoolManager.new(config, logger, redis, graphite) } + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } describe '#_check_pending_vm' do let(:pool_helper) { double('pool') } @@ -23,14 +23,12 @@ describe 'Pool Manager' do end context 'host not in pool' do - it 'calls fail_pending_vm' do allow(pool_helper).to receive(:find_vm).and_return(nil) allow(redis).to receive(:hget) expect(redis).to receive(:hget).with(String, 'clone').once subject._check_pending_vm(vm, pool, timeout) end - end context 'host is in pool' do @@ -58,7 +56,6 @@ describe 'Pool Manager' do end context 'a host without correct summary' do - it 'does nothing when summary is nil' do allow(host).to receive(:summary).and_return nil subject.move_pending_vm_to_ready(vm, pool, host) @@ -114,7 +111,6 @@ describe 'Pool Manager' do subject.move_pending_vm_to_ready(vm, pool, host) end - end end @@ -191,9 +187,7 @@ describe 'Pool Manager' do subject._check_running_vm(vm, pool, timeout) end - end - end describe '#move_running_to_completed' do @@ -240,15 +234,62 @@ describe 'Pool Manager' do end context 'logging' do - it 'logs empty pool' do allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0) allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0) expect(logger).to receive(:log).with('s', "[!] [pool1] is empty") subject._check_pool(config[:pools][0]) end + end + end + describe '#_stats_running_ready' do + let(:pool_helper) { double('pool') } + let(:vsphere) { {pool => pool_helper} } + let(:metrics) { Vmpooler::DummyStatsd.new } + let(:config) { { + config: { task_limit: 10 }, + pools: [ {'name' => 'pool1', 'size' => 5} ], + graphite: { 'prefix' => 'vmpooler' } + } } + + before do + expect(subject).not_to be_nil + $vsphere = vsphere + allow(logger).to receive(:log) + allow(pool_helper).to receive(:find_folder) + allow(redis).to receive(:smembers).and_return([]) + allow(redis).to receive(:set) + allow(redis).to receive(:get).with('vmpooler__tasks__clone').and_return(0) + allow(redis).to receive(:get).with('vmpooler__empty__pool1').and_return(nil) + end + + context 'metrics' do + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + + it 'increments metrics' do + allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(1) + allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5) + + expect(metrics).to receive(:gauge).with('ready.pool1', 1) + expect(metrics).to receive(:gauge).with('running.pool1', 5) + subject._check_pool(config[:pools][0]) + end + + it 'increments metrics when ready with 0 when pool empty' do + allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__cloning__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0) + allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(5) + + expect(metrics).to receive(:gauge).with('ready.pool1', 0) + expect(metrics).to receive(:gauge).with('running.pool1', 5) + subject._check_pool(config[:pools][0]) + end end end @@ -319,5 +360,4 @@ describe 'Pool Manager' do subject._check_snapshot_queue end end - end diff --git a/vmpooler b/vmpooler index 5d0dd51..20eba53 100755 --- a/vmpooler +++ b/vmpooler @@ -8,11 +8,12 @@ require 'lib/vmpooler' config = Vmpooler.config redis_host = config[:redis]['server'] logger_file = config[:config]['logfile'] -graphite = config[:graphite]['server'] ? config[:graphite]['server'] : nil + +metrics = Vmpooler.new_metrics(config) api = Thread.new { thr = Vmpooler::API.new - thr.helpers.configure(config, Vmpooler.new_redis(redis_host)) + thr.helpers.configure(config, Vmpooler.new_redis(redis_host), metrics) thr.helpers.execute! } @@ -21,9 +22,8 @@ manager = Thread.new { config, Vmpooler.new_logger(logger_file), Vmpooler.new_redis(redis_host), - Vmpooler.new_graphite(graphite) + metrics ).execute! } [api, manager].each { |t| t.join } - diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 26f2c51..4e54891 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -53,20 +53,79 @@ :redis: server: 'redis.company.com' + + # :graphs: + # + # This section contains the server and prefix information for a graphite- + # compatible web front-end where graphs may be viewed. This is used by the + # vmpooler dashboard to retrieve statistics and graphs for a given instance. + # + # NOTE: This is not the endpoint for publishing metrics data. See `graphite:` + # and `statsd:` below. + # + # NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be + # consulted for `server`/`prefix` information to use in locating a + # graph server for our dashboard. `graphs:` is recommended over + # `graphite:` + # + # + # Available configuration parameters: + # + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + + # :statsd: + # + # This section contains the connection information required to store + # historical data via statsd. This is mutually exclusive with graphite + # and takes precedence. + # + # Available configuration parameters: + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + # + # - port + # The UDP port to communicate with the statsd daemon. + # (optional; default: 8125) + + # Example: + + :statsd: + server: 'statsd.company.com' + prefix: 'vmpooler' + port: 8125 + # :graphite: # # This section contains the connection information required to store -# historical data in an external Graphite database. +# historical data in an external Graphite database. This is mutually exclusive +# with statsd. # # Available configuration parameters: # # - server # The FQDN hostname of the Graphite server. -# (optional) +# (required) # # - prefix # The prefix to use while storing Graphite data. # (optional; default: 'vmpooler') +# +# - port +# The TCP port to communicate with the graphite server. +# (optional; default: 2003) # Example: @@ -246,4 +305,3 @@ size: 5 timeout: 15 ready_ttl: 1440 -