puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 10:08:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #77 from colinPL/qeng_1906

Browse source 
(QENG-1906) Refactor initialize to allow config passing
This commit is contained in:
Scott Schneider 2015-03-31 10:10:42 -07:00
commit fd90756b83
7 changed files with 141 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

43
lib/vmpooler.rb
View file

@ -18,4 +18,47 @@ module Vmpooler
require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib)) require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
end end
end end
def self.config(filepath='vmpooler.yaml')
# Load the configuration file
config_file = File.expand_path(filepath)
parsed_config = YAML.load_file(config_file)
# Set some defaults
parsed_config[:redis] ||= {}
parsed_config[:redis]['server'] ||= 'localhost'
parsed_config[:redis]['data_ttl'] ||= 168
parsed_config[:config]['task_limit'] ||= 10
parsed_config[:config]['vm_checktime'] ||= 15
parsed_config[:config]['vm_lifetime'] ||= 24
if parsed_config[:graphite]['server']
parsed_config[:graphite]['prefix'] ||= 'vmpooler'
end
parsed_config[:uptime] = Time.now
parsed_config
end
def self.new_redis(host='localhost')
Redis.new(host: host)
end
def self.new_logger(logfile)
Vmpooler::Logger.new logfile
end
def self.new_graphite(server)
if server.nil? or server.empty? or server.length == 0
nil
else
Vmpooler::Graphite.new server
end
end
def self.pools(conf)
conf[:pools]
end
end end

29
lib/vmpooler/api.rb
View file

@ -1,24 +1,7 @@
module Vmpooler module Vmpooler
class API < Sinatra::Base class API < Sinatra::Base
def initialize def initialize
# Load the configuration file super
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
$config[:uptime] = Time.now
# Set some defaults
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
if $config[:graphite]['server']
$config[:graphite]['prefix'] ||= 'vmpooler'
end
# Connect to Redis
$redis = Redis.new(host: $config[:redis]['server'])
super()
end end
set :environment, :production set :environment, :production
@ -51,8 +34,14 @@ module Vmpooler
use Vmpooler::API::Reroute use Vmpooler::API::Reroute
use Vmpooler::API::V1 use Vmpooler::API::V1
Thread.new do def configure(config, redis, environment = :production)
run! self.settings.set :config, config
self.settings.set :redis, redis
self.settings.set :environment, environment
end
def execute!
self.settings.run!
end end
end end
end end

18
lib/vmpooler/api/dashboard.rb
View file

@ -6,19 +6,19 @@ module Vmpooler
result = {} result = {}
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result[pool['name']] ||= {} result[pool['name']] ||= {}
result[pool['name']]['size'] = pool['size'] result[pool['name']]['size'] = pool['size']
result[pool['name']]['ready'] = $redis.scard('vmpooler__ready__' + pool['name']) result[pool['name']]['ready'] = $redis.scard('vmpooler__ready__' + pool['name'])
end end
if params[:history] if params[:history]
if $config[:graphite]['server'] if Vmpooler::API.settings.config[:graphite]['server']
history ||= {} history ||= {}
begin begin
buffer = open( buffer = open(
'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json' 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json'
).read ).read
history = JSON.parse(buffer) history = JSON.parse(buffer)
@ -46,9 +46,9 @@ module Vmpooler
rescue rescue
end end
else else
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result[pool['name']] ||= {} result[pool['name']] ||= {}
result[pool['name']]['history'] = [$redis.scard('vmpooler__ready__' + pool['name'])] result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])]
end end
end end
end end
@ -61,8 +61,8 @@ module Vmpooler
result = {} result = {}
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
running = $redis.scard('vmpooler__running__' + pool['name']) running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/ pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
result[pool['major']] ||= {} result[pool['major']] ||= {}
@ -71,10 +71,10 @@ module Vmpooler
end end
if params[:history] if params[:history]
if $config[:graphite]['server'] if Vmpooler::API.settings.config[:graphite]['server']
begin begin
buffer = open( buffer = open(
'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json' 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json'
).read ).read
JSON.parse(buffer).each do |pool| JSON.parse(buffer).each do |pool|
if pool['target'] =~ /.*\.(.*)$/ if pool['target'] =~ /.*\.(.*)$/

84
lib/vmpooler/api/v1.rb
View file

@ -12,8 +12,8 @@ module Vmpooler
percent: 0 percent: 0
} }
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i pool['capacity'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i
capacity[:current] += pool['capacity'] capacity[:current] += pool['capacity']
capacity[:total] += pool['size'].to_i capacity[:total] += pool['size'].to_i
@ -37,14 +37,14 @@ module Vmpooler
total: 0 total: 0
} }
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
queue[:pending] += $redis.scard('vmpooler__pending__' + pool['name']).to_i queue[:pending] += Vmpooler::API.settings.redis.scard('vmpooler__pending__' + pool['name']).to_i
queue[:ready] += $redis.scard('vmpooler__ready__' + pool['name']).to_i queue[:ready] += Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i
queue[:running] += $redis.scard('vmpooler__running__' + pool['name']).to_i queue[:running] += Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name']).to_i
queue[:completed] += $redis.scard('vmpooler__completed__' + pool['name']).to_i queue[:completed] += Vmpooler::API.settings.redis.scard('vmpooler__completed__' + pool['name']).to_i
end end
queue[:cloning] = $redis.get('vmpooler__tasks__clone').to_i queue[:cloning] = Vmpooler::API.settings.redis.get('vmpooler__tasks__clone').to_i
queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
queue[:booting] = 0 if queue[:booting] < 0 queue[:booting] = 0 if queue[:booting] < 0
queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
@ -67,7 +67,7 @@ module Vmpooler
} }
} }
task[:count][:total] = $redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i task[:count][:total] = Vmpooler::API.settings.redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i
if task[:count][:total] > 0 if task[:count][:total] > 0
if opts[:bypool] == true if opts[:bypool] == true
@ -76,7 +76,7 @@ module Vmpooler
task[:count][:pool] = {} task[:count][:pool] = {}
task[:duration][:pool] = {} task[:duration][:pool] = {}
$redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value| Vmpooler::API.settings.redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
pool = 'unknown' pool = 'unknown'
hostname = 'unknown' hostname = 'unknown'
@ -113,11 +113,11 @@ module Vmpooler
end end
def get_task_times(task, date_str) def get_task_times(task, date_str)
$redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) Vmpooler::API.settings.redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
end end
def hostname_shorten(hostname) def hostname_shorten(hostname)
if $config[:config]['domain'] && hostname =~ /^\w+\.#{$config[:config]['domain']}$/ if Vmpooler::API.settings.config[:config]['domain'] && hostname =~ /^\w+\.#{Vmpooler::API.settings.config[:config]['domain']}$/
hostname = hostname[/[^\.]+/] hostname = hostname[/[^\.]+/]
end end
@ -150,8 +150,8 @@ module Vmpooler
result[:boot] = get_task_metrics('boot', Date.today.to_s) result[:boot] = get_task_metrics('boot', Date.today.to_s)
# Check for empty pools # Check for empty pools
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
if $redis.scard('vmpooler__ready__' + pool['name']).to_i == 0 if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i == 0
result[:status][:empty] ||= [] result[:status][:empty] ||= []
result[:status][:empty].push(pool['name']) result[:status][:empty].push(pool['name'])
@ -160,7 +160,7 @@ module Vmpooler
end end
end end
result[:status][:uptime] = (Time.now - $config[:uptime]).round(1) if $config[:uptime] result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
end end
@ -304,7 +304,7 @@ module Vmpooler
result = [] result = []
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
result.push(pool['name']) result.push(pool['name'])
end end
@ -321,7 +321,7 @@ module Vmpooler
jdata = JSON.parse(request.body.read) jdata = JSON.parse(request.body.read)
jdata.each do |key, val| jdata.each do |key, val|
if $redis.scard('vmpooler__ready__' + key) < val.to_i if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + key) < val.to_i
available = 0 available = 0
end end
end end
@ -335,12 +335,12 @@ module Vmpooler
result[key]['ok'] = true ## result[key]['ok'] = true ##
val.to_i.times do |_i| val.to_i.times do |_i|
vm = $redis.spop('vmpooler__ready__' + key) vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + key)
unless vm.nil? unless vm.nil?
$redis.sadd('vmpooler__running__' + key, vm) Vmpooler::API.settings.redis.sadd('vmpooler__running__' + key, vm)
$redis.hset('vmpooler__active__' + key, vm, Time.now) Vmpooler::API.settings.redis.hset('vmpooler__active__' + key, vm, Time.now)
$redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now)
result[key] ||= {} result[key] ||= {}
@ -365,8 +365,8 @@ module Vmpooler
result['ok'] = false result['ok'] = false
end end
if result['ok'] && $config[:config]['domain'] if result['ok'] && Vmpooler::API.settings.config[:config]['domain']
result['domain'] = $config[:config]['domain'] result['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
JSON.pretty_generate(result) JSON.pretty_generate(result)
@ -386,7 +386,7 @@ module Vmpooler
available = 1 available = 1
request.keys.each do |template| request.keys.each do |template|
if $redis.scard('vmpooler__ready__' + template) < request[template] if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + template) < request[template]
available = 0 available = 0
end end
end end
@ -399,12 +399,12 @@ module Vmpooler
result[template]['ok'] = true ## result[template]['ok'] = true ##
vm = $redis.spop('vmpooler__ready__' + template) vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + template)
unless vm.nil? unless vm.nil?
$redis.sadd('vmpooler__running__' + template, vm) Vmpooler::API.settings.redis.sadd('vmpooler__running__' + template, vm)
$redis.hset('vmpooler__active__' + template, vm, Time.now) Vmpooler::API.settings.redis.hset('vmpooler__active__' + template, vm, Time.now)
$redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now)
result[template] ||= {} result[template] ||= {}
@ -426,8 +426,8 @@ module Vmpooler
result['ok'] = false result['ok'] = false
end end
if result['ok'] && $config[:config]['domain'] if result['ok'] && Vmpooler::API.settings.config[:config]['domain']
result['domain'] = $config[:config]['domain'] result['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
JSON.pretty_generate(result) JSON.pretty_generate(result)
@ -443,16 +443,16 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
if $redis.exists('vmpooler__vm__' + params[:hostname]) if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname])
status 200 status 200
result['ok'] = true result['ok'] = true
rdata = $redis.hgetall('vmpooler__vm__' + params[:hostname]) rdata = Vmpooler::API.settings.redis.hgetall('vmpooler__vm__' + params[:hostname])
result[params[:hostname]] = {} result[params[:hostname]] = {}
result[params[:hostname]]['template'] = rdata['template'] result[params[:hostname]]['template'] = rdata['template']
result[params[:hostname]]['lifetime'] = rdata['lifetime'] || $config[:config]['vm_lifetime'] result[params[:hostname]]['lifetime'] = rdata['lifetime'] || Vmpooler::API.settings.config[:config]['vm_lifetime']
if rdata['destroy'] if rdata['destroy']
result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
@ -467,8 +467,8 @@ module Vmpooler
end end
end end
if $config[:config]['domain'] if Vmpooler::API.settings.config[:config]['domain']
result[params[:hostname]]['domain'] = $config[:config]['domain'] result[params[:hostname]]['domain'] = Vmpooler::API.settings.config[:config]['domain']
end end
end end
@ -485,10 +485,10 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
$config[:pools].each do |pool| Vmpooler::API.settings.config[:pools].each do |pool|
if $redis.sismember('vmpooler__running__' + pool['name'], params[:hostname]) if Vmpooler::API.settings.redis.sismember('vmpooler__running__' + pool['name'], params[:hostname])
$redis.srem('vmpooler__running__' + pool['name'], params[:hostname]) Vmpooler::API.settings.redis.srem('vmpooler__running__' + pool['name'], params[:hostname])
$redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname]) Vmpooler::API.settings.redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname])
status 200 status 200
result['ok'] = true result['ok'] = true
@ -510,7 +510,7 @@ module Vmpooler
params[:hostname] = hostname_shorten(params[:hostname]) params[:hostname] = hostname_shorten(params[:hostname])
if $redis.exists('vmpooler__vm__' + params[:hostname]) if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname])
begin begin
jdata = JSON.parse(request.body.read) jdata = JSON.parse(request.body.read)
rescue rescue
@ -542,10 +542,10 @@ module Vmpooler
when 'lifetime' when 'lifetime'
arg = arg.to_i arg = arg.to_i
$redis.hset('vmpooler__vm__' + params[:hostname], param, arg) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], param, arg)
when 'tags' when 'tags'
arg.keys.each do |tag| arg.keys.each do |tag|
$redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag]) Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag])
end end
end end
end end

22
lib/vmpooler/janitor.rb
View file

@ -1,32 +1,26 @@
module Vmpooler module Vmpooler
class Janitor class Janitor
def initialize def initialize(logger, redis, data_ttl)
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
# Set some defaults
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
$config[:redis]['data_ttl'] ||= 168
# Load logger library # Load logger library
$logger = Vmpooler::Logger.new $config[:config]['logfile'] $logger = logger
# Connect to Redis # Connect to Redis
$redis = Redis.new(host: $config[:redis]['server']) $redis = redis
# TTL
$data_ttl = data_ttl
end end
def execute! def execute!
loop do loop do
$redis.keys('vmpooler__vm__*').each do |key| $redis.keys('vmpooler__vm__*').each do |key|
data = $redis.hgetall(key); data = $redis.hgetall(key)
if data['destroy'] if data['destroy']
lifetime = (Time.now - Time.parse(data['destroy'])) / 60 / 60 lifetime = (Time.now - Time.parse(data['destroy'])) / 60 / 60
if lifetime > $config[:redis]['data_ttl'] if lifetime > $data_ttl
$redis.del(key) $redis.del(key)
end end
end end

25
lib/vmpooler/pool_manager.rb
View file

@ -1,30 +1,19 @@
module Vmpooler module Vmpooler
class PoolManager class PoolManager
def initialize def initialize(config, pools, logger, redis, graphite=nil)
# Load the configuration file $config = config
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
$pools = $config[:pools] $pools = pools
# Set some defaults
$config[:config]['task_limit'] ||= 10
$config[:config]['vm_checktime'] ||= 15
$config[:config]['vm_lifetime'] ||= 24
$config[:redis] ||= {}
$config[:redis]['server'] ||= 'localhost'
# Load logger library # Load logger library
$logger = Vmpooler::Logger.new $config[:config]['logfile'] $logger = logger
# Load Graphite helper library (if configured) unless graphite.nil?
if defined? $config[:graphite]['server'] $graphite = graphite
$config[:graphite]['prefix'] ||= 'vmpooler'
$graphite = Vmpooler::Graphite.new $config[:graphite]['server']
end end
# Connect to Redis # Connect to Redis
$redis = Redis.new(host: $config[:redis]['server']) $redis = redis
# vSphere object # vSphere object
$vsphere = {} $vsphere = {}

29
vmpooler
View file

@ -5,10 +5,27 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
require 'rubygems' unless defined?(Gem) require 'rubygems' unless defined?(Gem)
require 'lib/vmpooler' require 'lib/vmpooler'
Thread.new { Vmpooler::API.new.execute! } config = Vmpooler.config
Thread.new { Vmpooler::Janitor.new.execute! } redis_host = config[:redis]['server']
Thread.new { Vmpooler::PoolManager.new.execute! } redis_ttl = config[:redis]['data_ttl']
logger_file = config[:config]['logfile']
graphite = defined? config[:graphite]['server'] ? config[:graphite]['server'] : nil
api = Thread.new {
thr = Vmpooler::API.new
thr.helpers.configure(config, Vmpooler.new_redis(redis_host))
thr.helpers.execute!
}
janitor = Thread.new {
Vmpooler::Janitor.new(Vmpooler.new_logger(logger_file), Vmpooler.new_redis(redis_host), redis_ttl).execute!
}
manager = Thread.new {
Vmpooler::PoolManager.new(config,
config[:pools],
Vmpooler.new_logger(logger_file),
Vmpooler.new_redis(redis_host),
Vmpooler.new_graphite(graphite)).execute!
}
[api, janitor, manager].each { |t| t.join }
loop do
sleep(10)
end