diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index dbb1bd0..bb84d21 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -18,4 +18,47 @@ module Vmpooler require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib)) end end + + def self.config(filepath='vmpooler.yaml') + # Load the configuration file + config_file = File.expand_path(filepath) + parsed_config = YAML.load_file(config_file) + + # Set some defaults + parsed_config[:redis] ||= {} + parsed_config[:redis]['server'] ||= 'localhost' + parsed_config[:redis]['data_ttl'] ||= 168 + + parsed_config[:config]['task_limit'] ||= 10 + parsed_config[:config]['vm_checktime'] ||= 15 + parsed_config[:config]['vm_lifetime'] ||= 24 + + if parsed_config[:graphite]['server'] + parsed_config[:graphite]['prefix'] ||= 'vmpooler' + end + + parsed_config[:uptime] = Time.now + + parsed_config + end + + def self.new_redis(host='localhost') + Redis.new(host: host) + end + + def self.new_logger(logfile) + Vmpooler::Logger.new logfile + end + + def self.new_graphite(server) + if server.nil? or server.empty? or server.length == 0 + nil + else + Vmpooler::Graphite.new server + end + end + + def self.pools(conf) + conf[:pools] + end end diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 6b8419e..2219614 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -1,24 +1,7 @@ module Vmpooler class API < Sinatra::Base def initialize - # Load the configuration file - config_file = File.expand_path('vmpooler.yaml') - $config = YAML.load_file(config_file) - - $config[:uptime] = Time.now - - # Set some defaults - $config[:redis] ||= {} - $config[:redis]['server'] ||= 'localhost' - - if $config[:graphite]['server'] - $config[:graphite]['prefix'] ||= 'vmpooler' - end - - # Connect to Redis - $redis = Redis.new(host: $config[:redis]['server']) - - super() + super end set :environment, :production @@ -51,8 +34,14 @@ module Vmpooler use Vmpooler::API::Reroute use Vmpooler::API::V1 - Thread.new do - run! + def configure(config, redis, environment = :production) + self.settings.set :config, config + self.settings.set :redis, redis + self.settings.set :environment, environment + end + + def execute! + self.settings.run! end end end diff --git a/lib/vmpooler/api/dashboard.rb b/lib/vmpooler/api/dashboard.rb index 4fe4514..c36fa4b 100644 --- a/lib/vmpooler/api/dashboard.rb +++ b/lib/vmpooler/api/dashboard.rb @@ -6,19 +6,19 @@ module Vmpooler result = {} - $config[:pools].each do |pool| + Vmpooler::API.settings.config[:pools].each do |pool| result[pool['name']] ||= {} result[pool['name']]['size'] = pool['size'] result[pool['name']]['ready'] = $redis.scard('vmpooler__ready__' + pool['name']) end if params[:history] - if $config[:graphite]['server'] + if Vmpooler::API.settings.config[:graphite]['server'] history ||= {} begin buffer = open( - 'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json' + 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.ready.*&from=-1hour&format=json' ).read history = JSON.parse(buffer) @@ -46,9 +46,9 @@ module Vmpooler rescue end else - $config[:pools].each do |pool| + Vmpooler::API.settings.config[:pools].each do |pool| result[pool['name']] ||= {} - result[pool['name']]['history'] = [$redis.scard('vmpooler__ready__' + pool['name'])] + result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])] end end end @@ -61,8 +61,8 @@ module Vmpooler result = {} - $config[:pools].each do |pool| - running = $redis.scard('vmpooler__running__' + pool['name']) + Vmpooler::API.settings.config[:pools].each do |pool| + running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name']) pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/ result[pool['major']] ||= {} @@ -71,10 +71,10 @@ module Vmpooler end if params[:history] - if $config[:graphite]['server'] + if Vmpooler::API.settings.config[:graphite]['server'] begin buffer = open( - 'http://' + $config[:graphite]['server'] + '/render?target=' + $config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json' + 'http://' + Vmpooler::API.settings.config[:graphite]['server'] + '/render?target=' + Vmpooler::API.settings.config[:graphite]['prefix'] + '.running.*&from=-1hour&format=json' ).read JSON.parse(buffer).each do |pool| if pool['target'] =~ /.*\.(.*)$/ diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 1e3f9e9..1656219 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -12,8 +12,8 @@ module Vmpooler percent: 0 } - $config[:pools].each do |pool| - pool['capacity'] = $redis.scard('vmpooler__ready__' + pool['name']).to_i + Vmpooler::API.settings.config[:pools].each do |pool| + pool['capacity'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i capacity[:current] += pool['capacity'] capacity[:total] += pool['size'].to_i @@ -37,14 +37,14 @@ module Vmpooler total: 0 } - $config[:pools].each do |pool| - queue[:pending] += $redis.scard('vmpooler__pending__' + pool['name']).to_i - queue[:ready] += $redis.scard('vmpooler__ready__' + pool['name']).to_i - queue[:running] += $redis.scard('vmpooler__running__' + pool['name']).to_i - queue[:completed] += $redis.scard('vmpooler__completed__' + pool['name']).to_i + Vmpooler::API.settings.config[:pools].each do |pool| + queue[:pending] += Vmpooler::API.settings.redis.scard('vmpooler__pending__' + pool['name']).to_i + queue[:ready] += Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i + queue[:running] += Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name']).to_i + queue[:completed] += Vmpooler::API.settings.redis.scard('vmpooler__completed__' + pool['name']).to_i end - queue[:cloning] = $redis.get('vmpooler__tasks__clone').to_i + queue[:cloning] = Vmpooler::API.settings.redis.get('vmpooler__tasks__clone').to_i queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i queue[:booting] = 0 if queue[:booting] < 0 queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i @@ -67,7 +67,7 @@ module Vmpooler } } - task[:count][:total] = $redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i + task[:count][:total] = Vmpooler::API.settings.redis.hlen('vmpooler__' + task_str + '__' + date_str).to_i if task[:count][:total] > 0 if opts[:bypool] == true @@ -76,7 +76,7 @@ module Vmpooler task[:count][:pool] = {} task[:duration][:pool] = {} - $redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value| + Vmpooler::API.settings.redis.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value| pool = 'unknown' hostname = 'unknown' @@ -113,11 +113,11 @@ module Vmpooler end def get_task_times(task, date_str) - $redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) + Vmpooler::API.settings.redis.hvals("vmpooler__#{task}__" + date_str).map(&:to_f) end def hostname_shorten(hostname) - if $config[:config]['domain'] && hostname =~ /^\w+\.#{$config[:config]['domain']}$/ + if Vmpooler::API.settings.config[:config]['domain'] && hostname =~ /^\w+\.#{Vmpooler::API.settings.config[:config]['domain']}$/ hostname = hostname[/[^\.]+/] end @@ -150,8 +150,8 @@ module Vmpooler result[:boot] = get_task_metrics('boot', Date.today.to_s) # Check for empty pools - $config[:pools].each do |pool| - if $redis.scard('vmpooler__ready__' + pool['name']).to_i == 0 + Vmpooler::API.settings.config[:pools].each do |pool| + if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name']).to_i == 0 result[:status][:empty] ||= [] result[:status][:empty].push(pool['name']) @@ -160,7 +160,7 @@ module Vmpooler end end - result[:status][:uptime] = (Time.now - $config[:uptime]).round(1) if $config[:uptime] + result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime] JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) end @@ -304,7 +304,7 @@ module Vmpooler result = [] - $config[:pools].each do |pool| + Vmpooler::API.settings.config[:pools].each do |pool| result.push(pool['name']) end @@ -321,7 +321,7 @@ module Vmpooler jdata = JSON.parse(request.body.read) jdata.each do |key, val| - if $redis.scard('vmpooler__ready__' + key) < val.to_i + if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + key) < val.to_i available = 0 end end @@ -335,12 +335,12 @@ module Vmpooler result[key]['ok'] = true ## val.to_i.times do |_i| - vm = $redis.spop('vmpooler__ready__' + key) + vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + key) unless vm.nil? - $redis.sadd('vmpooler__running__' + key, vm) - $redis.hset('vmpooler__active__' + key, vm, Time.now) - $redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) + Vmpooler::API.settings.redis.sadd('vmpooler__running__' + key, vm) + Vmpooler::API.settings.redis.hset('vmpooler__active__' + key, vm, Time.now) + Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) result[key] ||= {} @@ -365,8 +365,8 @@ module Vmpooler result['ok'] = false end - if result['ok'] && $config[:config]['domain'] - result['domain'] = $config[:config]['domain'] + if result['ok'] && Vmpooler::API.settings.config[:config]['domain'] + result['domain'] = Vmpooler::API.settings.config[:config]['domain'] end JSON.pretty_generate(result) @@ -386,7 +386,7 @@ module Vmpooler available = 1 request.keys.each do |template| - if $redis.scard('vmpooler__ready__' + template) < request[template] + if Vmpooler::API.settings.redis.scard('vmpooler__ready__' + template) < request[template] available = 0 end end @@ -399,12 +399,12 @@ module Vmpooler result[template]['ok'] = true ## - vm = $redis.spop('vmpooler__ready__' + template) + vm = Vmpooler::API.settings.redis.spop('vmpooler__ready__' + template) unless vm.nil? - $redis.sadd('vmpooler__running__' + template, vm) - $redis.hset('vmpooler__active__' + template, vm, Time.now) - $redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) + Vmpooler::API.settings.redis.sadd('vmpooler__running__' + template, vm) + Vmpooler::API.settings.redis.hset('vmpooler__active__' + template, vm, Time.now) + Vmpooler::API.settings.redis.hset('vmpooler__vm__' + vm, 'checkout', Time.now) result[template] ||= {} @@ -426,8 +426,8 @@ module Vmpooler result['ok'] = false end - if result['ok'] && $config[:config]['domain'] - result['domain'] = $config[:config]['domain'] + if result['ok'] && Vmpooler::API.settings.config[:config]['domain'] + result['domain'] = Vmpooler::API.settings.config[:config]['domain'] end JSON.pretty_generate(result) @@ -443,16 +443,16 @@ module Vmpooler params[:hostname] = hostname_shorten(params[:hostname]) - if $redis.exists('vmpooler__vm__' + params[:hostname]) + if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname]) status 200 result['ok'] = true - rdata = $redis.hgetall('vmpooler__vm__' + params[:hostname]) + rdata = Vmpooler::API.settings.redis.hgetall('vmpooler__vm__' + params[:hostname]) result[params[:hostname]] = {} result[params[:hostname]]['template'] = rdata['template'] - result[params[:hostname]]['lifetime'] = rdata['lifetime'] || $config[:config]['vm_lifetime'] + result[params[:hostname]]['lifetime'] = rdata['lifetime'] || Vmpooler::API.settings.config[:config]['vm_lifetime'] if rdata['destroy'] result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) @@ -467,8 +467,8 @@ module Vmpooler end end - if $config[:config]['domain'] - result[params[:hostname]]['domain'] = $config[:config]['domain'] + if Vmpooler::API.settings.config[:config]['domain'] + result[params[:hostname]]['domain'] = Vmpooler::API.settings.config[:config]['domain'] end end @@ -485,10 +485,10 @@ module Vmpooler params[:hostname] = hostname_shorten(params[:hostname]) - $config[:pools].each do |pool| - if $redis.sismember('vmpooler__running__' + pool['name'], params[:hostname]) - $redis.srem('vmpooler__running__' + pool['name'], params[:hostname]) - $redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname]) + Vmpooler::API.settings.config[:pools].each do |pool| + if Vmpooler::API.settings.redis.sismember('vmpooler__running__' + pool['name'], params[:hostname]) + Vmpooler::API.settings.redis.srem('vmpooler__running__' + pool['name'], params[:hostname]) + Vmpooler::API.settings.redis.sadd('vmpooler__completed__' + pool['name'], params[:hostname]) status 200 result['ok'] = true @@ -510,7 +510,7 @@ module Vmpooler params[:hostname] = hostname_shorten(params[:hostname]) - if $redis.exists('vmpooler__vm__' + params[:hostname]) + if Vmpooler::API.settings.redis.exists('vmpooler__vm__' + params[:hostname]) begin jdata = JSON.parse(request.body.read) rescue @@ -542,10 +542,10 @@ module Vmpooler when 'lifetime' arg = arg.to_i - $redis.hset('vmpooler__vm__' + params[:hostname], param, arg) + Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], param, arg) when 'tags' arg.keys.each do |tag| - $redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag]) + Vmpooler::API.settings.redis.hset('vmpooler__vm__' + params[:hostname], 'tag:' + tag, arg[tag]) end end end diff --git a/lib/vmpooler/janitor.rb b/lib/vmpooler/janitor.rb index 91e0f3f..075c477 100644 --- a/lib/vmpooler/janitor.rb +++ b/lib/vmpooler/janitor.rb @@ -1,32 +1,26 @@ module Vmpooler class Janitor - def initialize - # Load the configuration file - config_file = File.expand_path('vmpooler.yaml') - $config = YAML.load_file(config_file) - - # Set some defaults - $config[:redis] ||= {} - $config[:redis]['server'] ||= 'localhost' - $config[:redis]['data_ttl'] ||= 168 - + def initialize(logger, redis, data_ttl) # Load logger library - $logger = Vmpooler::Logger.new $config[:config]['logfile'] + $logger = logger # Connect to Redis - $redis = Redis.new(host: $config[:redis]['server']) + $redis = redis + + # TTL + $data_ttl = data_ttl end def execute! loop do $redis.keys('vmpooler__vm__*').each do |key| - data = $redis.hgetall(key); + data = $redis.hgetall(key) if data['destroy'] lifetime = (Time.now - Time.parse(data['destroy'])) / 60 / 60 - if lifetime > $config[:redis]['data_ttl'] + if lifetime > $data_ttl $redis.del(key) end end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index b6f7ee8..07dd7cd 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -1,30 +1,19 @@ module Vmpooler class PoolManager - def initialize - # Load the configuration file - config_file = File.expand_path('vmpooler.yaml') - $config = YAML.load_file(config_file) + def initialize(config, pools, logger, redis, graphite=nil) + $config = config - $pools = $config[:pools] - - # Set some defaults - $config[:config]['task_limit'] ||= 10 - $config[:config]['vm_checktime'] ||= 15 - $config[:config]['vm_lifetime'] ||= 24 - $config[:redis] ||= {} - $config[:redis]['server'] ||= 'localhost' + $pools = pools # Load logger library - $logger = Vmpooler::Logger.new $config[:config]['logfile'] + $logger = logger - # Load Graphite helper library (if configured) - if defined? $config[:graphite]['server'] - $config[:graphite]['prefix'] ||= 'vmpooler' - $graphite = Vmpooler::Graphite.new $config[:graphite]['server'] + unless graphite.nil? + $graphite = graphite end # Connect to Redis - $redis = Redis.new(host: $config[:redis]['server']) + $redis = redis # vSphere object $vsphere = {} diff --git a/vmpooler b/vmpooler index 51dc258..7193426 100755 --- a/vmpooler +++ b/vmpooler @@ -5,10 +5,27 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) require 'rubygems' unless defined?(Gem) require 'lib/vmpooler' -Thread.new { Vmpooler::API.new.execute! } -Thread.new { Vmpooler::Janitor.new.execute! } -Thread.new { Vmpooler::PoolManager.new.execute! } +config = Vmpooler.config +redis_host = config[:redis]['server'] +redis_ttl = config[:redis]['data_ttl'] +logger_file = config[:config]['logfile'] +graphite = defined? config[:graphite]['server'] ? config[:graphite]['server'] : nil + +api = Thread.new { + thr = Vmpooler::API.new + thr.helpers.configure(config, Vmpooler.new_redis(redis_host)) + thr.helpers.execute! + } +janitor = Thread.new { + Vmpooler::Janitor.new(Vmpooler.new_logger(logger_file), Vmpooler.new_redis(redis_host), redis_ttl).execute! + } +manager = Thread.new { + Vmpooler::PoolManager.new(config, + config[:pools], + Vmpooler.new_logger(logger_file), + Vmpooler.new_redis(redis_host), + Vmpooler.new_graphite(graphite)).execute! + } + +[api, janitor, manager].each { |t| t.join } -loop do - sleep(10) -end