puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 01:58:41 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

(QENG-1906) Refactor initialize to allow config passing

Browse source 
Prior to this commit, several pieces of vmpooler performed configuration
and initialization steps within 'initialize'. This made it difficult to
pass in mock objects for testing purposes.

This commit performs a single configuration and passes the results to
the various pieces of vmpooler.
This commit is contained in:
Colin 2015-03-27 13:37:52 -07:00
parent 34fd054a48
commit 1408f35867
7 changed files with 141 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

43
lib/vmpooler.rb
View file

@ -18,4 +18,47 @@ module Vmpooler
require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib)) require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
end end
end end
def self.config(filepath='vmpooler.yaml')
# Load the configuration file
config_file = File.expand_path(filepath)
parsed_config = YAML.load_file(config_file)
# Set some defaults
parsed_config[:redis] ||= {}
parsed_config[:redis]['server'] ||= 'localhost'
parsed_config[:redis]['data_ttl'] ||= 168
parsed_config[:config]['task_limit'] ||= 10
parsed_config[:config]['vm_checktime'] ||= 15
parsed_config[:config]['vm_lifetime'] ||= 24
if parsed_config[:graphite]['server']
parsed_config[:graphite]['prefix'] ||= 'vmpooler'
end
parsed_config[:uptime] = Time.now
parsed_config
end
def self.new_redis(host='localhost')
Redis.new(host: host)
end
def self.new_logger(logfile)
Vmpooler::Logger.new logfile
end
def self.new_graphite(server)
if server.nil? or server.empty? or server.length == 0
nil
else
Vmpooler::Graphite.new server
end
end
def self.pools(conf)
conf[:pools]
end
end end

29
lib/vmpooler/api.rb
View file

@ -1,24 +1,7 @@
module Vmpooler module Vmpooler
class API < Sinatra::Base class API < Sinatra::Base
def initialize def initialize
# Load the configuration file super
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
$config[:uptime] = Time.now
# Set some defaults
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
if $config[:graphite]['server']
$config[:graphite]['prefix'] ||= 'vmpooler'
end
# Connect to Redis
$redis = Redis.new(host: $config[:redis]['server'])
super()
end end
set :environment, :production set :environment, :production
@ -51,8 +34,14 @@ module Vmpooler
use Vmpooler::API::Reroute use Vmpooler::API::Reroute
use Vmpooler::API::V1 use Vmpooler::API::V1
Thread.new do def configure(config, redis, environment = :production)
run! self.settings.set :config, config
self.settings.set :redis, redis
self.settings.set :environment, environment
end
def execute!
self.settings.run!
end end
end end
end end

18
lib/vmpooler/api/dashboard.rb
View file

@ -6,19 +6,19 @@ module Vmpooler
result = {} result = {}
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result[pool['name']] ||= {} result[pool['name']] ||= {}
result[pool['name']]['size'] = pool['size'] result[pool['name']]['size'] = pool['size']
result[pool['name']]['ready'] = $redis.scard('vmpooler__ready__' + pool['name']) result[pool['name']]['ready'] = $redis.scard('vmpooler__ready__' + pool['name'])
end end
if params[:history] if params[:history]
if $config[:graphite]['server'] if Vmpooler::API.settings.config[:graphite]['server']
history ||= {} history ||= {}
begin begin
buffer = open( buffer = open(
'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json' 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json'
).read ).read
history = JSON.parse(buffer) history = JSON.parse(buffer)
@ -46,9 +46,9 @@ module Vmpooler
rescue rescue
end end
else else
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result[pool['name']] ||= {} result[pool['name']] ||= {}
result[pool['name']]['history'] = [$redis.scard('vmpooler__ready__' + pool['name'])] result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])]
end end
end end
end end
@ -61,8 +61,8 @@ module Vmpooler
result = {} result = {}
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
running = $redis.scard('vmpooler__running__' + pool['name']) running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/ pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
result[pool['major']] ||= {} result[pool['major']] ||= {}
@ -71,10 +71,10 @@ module Vmpooler
end end
if params[:history] if params[:history]
if $config[:graphite]['server'] if Vmpooler::API.settings.config[:graphite]['server']
begin begin
buffer = open( buffer = open(
'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json' 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json'
).read ).read
JSON.parse(buffer).each do |pool| JSON.parse(buffer).each do |pool|
if pool['target'] =~ /.*\.(.*)$/ if pool['target'] =~ /.*\.(.*)$/

84
lib/vmpooler/api/v1.rb
View file

@ -12,8 +12,8 @@ module Vmpooler
percent: 0 percent: 0
} }
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i pool['capacity'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i
capacity[:current] += pool['capacity'] capacity[:current] += pool['capacity']
capacity[:total] += pool['size'].to_i capacity[:total] += pool['size'].to_i
@ -37,14 +37,14 @@ module Vmpooler
total: 0 total: 0
} }
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
queue[:pending] += $redis.scard('vmpooler__pending__' + pool['name']).to_i queue[:pending] += Vmpooler::API.settings.redis.scard('vmpooler__pending__' + pool['name']).to_i
queue[:ready] += $redis.scard('vmpooler__ready__' + pool['name']).to_i queue[:ready] += Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i
queue[:running] += $redis.scard('vmpooler__running__' + pool['name']).to_i queue[:running] += Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name']).to_i
queue[:completed] += $redis.scard('vmpooler__completed__' + pool['name']).to_i queue[:completed] += Vmpooler::API.settings.redis.scard('vmpooler__completed__' + pool['name']).to_i
end end
queue[:cloning] = $redis.get('vmpooler__tasks__clone').to_i queue[:cloning] = Vmpooler::API.settings.redis.get('vmpooler__tasks__clone').to_i
queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
queue[:booting] = 0 if queue[:booting] < 0 queue[:booting] = 0 if queue[:booting] < 0
queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
@ -67,7 +67,7 @@ module Vmpooler
} }
} }
task[:count][:total] = $redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i task[:count][:total] = Vmpooler::API.settings.redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i
if task[:count][:total] > 0 if task[:count][:total] > 0
if opts[:bypool] == true if opts[:bypool] == true
@ -76,7 +76,7 @@ module Vmpooler
task[:count][:pool] = {} task[:count][:pool] = {}
task[:duration][:pool] = {} task[:duration][:pool] = {}
$redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value| Vmpooler::API.settings.redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
pool = 'unknown' pool = 'unknown'
hostname = 'unknown' hostname = 'unknown'
@ -113,11 +113,11 @@ module Vmpooler
end end
def get_task_times(task, date_str) def get_task_times(task, date_str)
$redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) Vmpooler::API.settings.redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
end end
def hostname_shorten(hostname) def hostname_shorten(hostname)
if $config[:config]['domain'] && hostname =~ /^\w+\.#{$config[:config]['domain']}$/ if Vmpooler::API.settings.config[:config]['domain'] && hostname =~ /^\w+\.#{Vmpooler::API.settings.config[:config]['domain']}$/
hostname = hostname[/[^\.]+/] hostname = hostname[/[^\.]+/]
end end
@ -150,8 +150,8 @@ module Vmpooler
result[:boot] = get_task_metrics('boot', Date.today.to_s) result[:boot] = get_task_metrics('boot', Date.today.to_s)
# Check for empty pools # Check for empty pools
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
if $redis.scard('vmpooler__ready__' + pool['name']).to_i == 0 if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i == 0
result[:status][:empty] ||= [] result[:status][:empty] ||= []
result[:status][:empty].push(pool['name']) result[:status][:empty].push(pool['name'])
@ -160,7 +160,7 @@ module Vmpooler
end end
end end
result[:status][:uptime] = (Time.now - $config[:uptime]).round(1) if $config[:uptime] result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
end end
@ -304,7 +304,7 @@ module Vmpooler
result = [] result = []
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result.push(pool['name']) result.push(pool['name'])
end end
@ -321,7 +321,7 @@ module Vmpooler
jdata = JSON.parse(request.body.read) jdata = JSON.parse(request.body.read)
jdata.each do |key, val| jdata.each do |key, val|
if $redis.scard('vmpooler__ready__' + key) < val.to_i if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + key) < val.to_i
available = 0 available = 0
end end
end end
@ -335,12 +335,12 @@ module Vmpooler
result[key]['ok'] = true ## result[key]['ok'] = true ##
val.to_i.times do |_i| val.to_i.times do |_i|
vm = $redis.spop('vmpooler__ready__' + key) vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + key)
unless vm.nil? unless vm.nil?
$redis.sadd('vmpooler__running__' + key, vm) Vmpooler::API.settings.redis.sadd('vmpooler__running__' + key, vm)
$redis.hset('vmpooler__active__' + key, vm, Time.now) Vmpooler::API.settings.redis.hset('vmpooler__active__' + key, vm, Time.now)
$redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now)
result[key] ||= {} result[key] ||= {}
@ -365,8 +365,8 @@ module Vmpooler
result['ok'] = false result['ok'] = false
end end
if result['ok'] && $config[:config]['domain'] if result['ok'] && Vmpooler::API.settings.config[:config]['domain']
result['domain'] = $config[:config]['domain'] result['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
JSON.pretty_generate(result) JSON.pretty_generate(result)
@ -386,7 +386,7 @@ module Vmpooler
available = 1 available = 1
request.keys.each do |template| request.keys.each do |template|
if $redis.scard('vmpooler__ready__' + template) < request[template] if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + template) < request[template]
available = 0 available = 0
end end
end end
@ -399,12 +399,12 @@ module Vmpooler
result[template]['ok'] = true ## result[template]['ok'] = true ##
vm = $redis.spop('vmpooler__ready__' + template) vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + template)
unless vm.nil? unless vm.nil?
$redis.sadd('vmpooler__running__' + template, vm) Vmpooler::API.settings.redis.sadd('vmpooler__running__' + template, vm)
$redis.hset('vmpooler__active__' + template, vm, Time.now) Vmpooler::API.settings.redis.hset('vmpooler__active__' + template, vm, Time.now)
$redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now)
result[template] ||= {} result[template] ||= {}
@ -426,8 +426,8 @@ module Vmpooler
result['ok'] = false result['ok'] = false
end end
if result['ok'] && $config[:config]['domain'] if result['ok'] && Vmpooler::API.settings.config[:config]['domain']
result['domain'] = $config[:config]['domain'] result['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
JSON.pretty_generate(result) JSON.pretty_generate(result)
@ -443,16 +443,16 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
if $redis.exists('vmpooler__vm__' + params[:hostname]) if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname])
status 200 status 200
result['ok'] = true result['ok'] = true
rdata = $redis.hgetall('vmpooler__vm__' + params[:hostname]) rdata = Vmpooler::API.settings.redis.hgetall('vmpooler__vm__' + params[:hostname])
result[params[:hostname]] = {} result[params[:hostname]] = {}
result[params[:hostname]]['template'] = rdata['template'] result[params[:hostname]]['template'] = rdata['template']
result[params[:hostname]]['lifetime'] = rdata['lifetime'] || $config[:config]['vm_lifetime'] result[params[:hostname]]['lifetime'] = rdata['lifetime'] || Vmpooler::API.settings.config[:config]['vm_lifetime']
if rdata['destroy'] if rdata['destroy']
result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
@ -467,8 +467,8 @@ module Vmpooler
end end
end end
if $config[:config]['domain'] if Vmpooler::API.settings.config[:config]['domain']
result[params[:hostname]]['domain'] = $config[:config]['domain'] result[params[:hostname]]['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
end end
@ -485,10 +485,10 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
if $redis.sismember('vmpooler__running__' + pool['name'], params[:hostname]) if Vmpooler::API.settings.redis.sismember('vmpooler__running__' + pool['name'], params[:hostname])
$redis.srem('vmpooler__running__' + pool['name'], params[:hostname]) Vmpooler::API.settings.redis.srem('vmpooler__running__' + pool['name'], params[:hostname])
$redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname]) Vmpooler::API.settings.redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname])
status 200 status 200
result['ok'] = true result['ok'] = true
@ -510,7 +510,7 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
if $redis.exists('vmpooler__vm__' + params[:hostname]) if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname])
begin begin
jdata = JSON.parse(request.body.read) jdata = JSON.parse(request.body.read)
rescue rescue
@ -542,10 +542,10 @@ module Vmpooler
when 'lifetime' when 'lifetime'
arg = arg.to_i arg = arg.to_i
$redis.hset('vmpooler__vm__' + params[:hostname], param, arg) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], param, arg)
when 'tags' when 'tags'
arg.keys.each do |tag| arg.keys.each do |tag|
$redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag]) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag])
end end
end end
end end

22
lib/vmpooler/janitor.rb
View file

@ -1,32 +1,26 @@
module Vmpooler module Vmpooler
class Janitor class Janitor
def initialize def initialize(logger, redis, data_ttl)
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
# Set some defaults
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
$config[:redis]['data_ttl'] ||= 168
# Load logger library # Load logger library
$logger = Vmpooler::Logger.new $config[:config]['logfile'] $logger = logger
# Connect to Redis # Connect to Redis
$redis = Redis.new(host: $config[:redis]['server']) $redis = redis
# TTL
$data_ttl = data_ttl
end end
def execute! def execute!
loop do loop do
$redis.keys('vmpooler__vm__*').each do |key| $redis.keys('vmpooler__vm__*').each do |key|
data = $redis.hgetall(key); data = $redis.hgetall(key)
if data['destroy'] if data['destroy']
lifetime = (Time.now - Time.parse(data['destroy'])) / 60 / 60 lifetime = (Time.now - Time.parse(data['destroy'])) / 60 / 60
if lifetime > $config[:redis]['data_ttl'] if lifetime > $data_ttl
$redis.del(key) $redis.del(key)
end end
end end

25
lib/vmpooler/pool_manager.rb
View file

@ -1,30 +1,19 @@
module Vmpooler module Vmpooler
class PoolManager class PoolManager
def initialize def initialize(config, pools, logger, redis, graphite=nil)
# Load the configuration file $config = config
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
$pools = $config[:pools] $pools = pools
# Set some defaults
$config[:config]['task_limit'] ||= 10
$config[:config]['vm_checktime'] ||= 15
$config[:config]['vm_lifetime'] ||= 24
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
# Load logger library # Load logger library
$logger = Vmpooler::Logger.new $config[:config]['logfile'] $logger = logger
# Load Graphite helper library (if configured) unless graphite.nil?
if defined? $config[:graphite]['server'] $graphite = graphite
$config[:graphite]['prefix'] ||= 'vmpooler'
$graphite = Vmpooler::Graphite.new $config[:graphite]['server']
end end
# Connect to Redis # Connect to Redis
$redis = Redis.new(host: $config[:redis]['server']) $redis = redis
# vSphere object # vSphere object
$vsphere = {} $vsphere = {}

29
vmpooler
View file

@ -5,10 +5,27 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
require 'rubygems' unless defined?(Gem) require 'rubygems' unless defined?(Gem)
require 'lib/vmpooler' require 'lib/vmpooler'
Thread.new { Vmpooler::API.new.execute! } config = Vmpooler.config
Thread.new { Vmpooler::Janitor.new.execute! } redis_host = config[:redis]['server']
Thread.new { Vmpooler::PoolManager.new.execute! } redis_ttl = config[:redis]['data_ttl']
logger_file = config[:config]['logfile']
graphite = defined? config[:graphite]['server'] ? config[:graphite]['server'] : nil
api = Thread.new {
thr = Vmpooler::API.new
thr.helpers.configure(config, Vmpooler.new_redis(redis_host))
thr.helpers.execute!
}
janitor = Thread.new {
Vmpooler::Janitor.new(Vmpooler.new_logger(logger_file), Vmpooler.new_redis(redis_host), redis_ttl).execute!
}
manager = Thread.new {
Vmpooler::PoolManager.new(config,
config[:pools],
Vmpooler.new_logger(logger_file),
Vmpooler.new_redis(redis_host),
Vmpooler.new_graphite(graphite)).execute!
}
[api, janitor, manager].each { |t| t.join }
loop do
sleep(10)
end