diff --git a/README.md b/README.md index 5d68d17..0525aa4 100644 --- a/README.md +++ b/README.md @@ -84,21 +84,63 @@ $ curl --url vmpooler.company.com/vm ] ``` +#### POST /vm +Useful for batch operations; post JSON (see format below), get back VMs. + +``` +$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.company.com/vm +``` +```json +{ + "ok": true, + "debian-7-i386": { + "hostname": [ + "o41xtodlvnvu5cw", + "khirruvwfjlmx3y" + ] + }, + "debian-7-x86_64": { + "hostname": "y91qbrpbfj6d13q" + } +} +``` + #### POST /vm/ -Check-out a VM. +Check-out a VM or VMs. ``` $ curl -d --url vmpooler.company.com/vm/debian-7-i386 ``` ```json { + "ok": true, "debian-7-i386": { - "ok": true, "hostname": "fq6qlpjlsskycq6" } } ``` +Multiple VMs can be requested by using multiple query parameters in the URL: + +``` +$ curl -d --url vmpooler.company.com/vm/debian-7-i386+debian-7-i386+debian-7-x86_64 +``` + +```json +{ + "ok": true, + "debian-7-i386": { + "hostname": [ + "sc0o4xqtodlul5w", + "4m4dkhqiufnjmxy" + ] + }, + "debian-7-x86_64": { + "hostname": "zb91y9qbrbf6d3q" + } +} +``` + #### DELETE /vm/ Schedule a checked-out VM for deletion. @@ -121,9 +163,9 @@ A dashboard is provided to offer real-time statistics and historical graphs. It [Graphite](http://graphite.wikidot.com/) is required for historical data retrieval. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for details. -## Author +## Authors and Contributors -Scott Schneider (sschneid@gmail.com) +- Scott Schneider (sschneid@gmail.com) ## License diff --git a/lib/graphite.rb b/lib/graphite.rb deleted file mode 100755 index b52e011..0000000 --- a/lib/graphite.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'rubygems' unless defined?(Gem) - -class Graphite - def initialize( - s = 'graphite' - ) - @server = s - end - - def log path, value - Thread.new { - socket = TCPSocket.new(@server, 2003) - socket.puts "#{path} #{value} #{Time.now.to_i}" - socket.close - } - end -end - diff --git a/lib/logger.rb b/lib/logger.rb deleted file mode 100755 index 2ec8918..0000000 --- a/lib/logger.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'rubygems' unless defined?(Gem) - -class Logger - def initialize( - f = '/var/log/vmpooler.log' - ) - @file = f - end - - def log level, string - time = Time.new - stamp = time.strftime('%Y-%m-%d %H:%M:%S') - puts "[#{stamp}] #{string}" - - open(@file, 'a') do |f| - f.puts "[#{stamp}] #{string}" - end - end -end - diff --git a/lib/require_relative.rb b/lib/require_relative.rb deleted file mode 100755 index ab9114c..0000000 --- a/lib/require_relative.rb +++ /dev/null @@ -1,21 +0,0 @@ -# require_relative was introduced in 1.9.2. This makes it -# available to younger rubies. It trys hard to not re-require -# files. - -unless Kernel.respond_to?(:require_relative) - module Kernel - def require_relative(path) - desired_path = File.expand_path('../'+path.to_str, caller[0]) - shortest = desired_path - $:.each do |path| - path += '/' - if desired_path.index(path) == 0 - candidate = desired_path.sub(path, '') - shortest = candidate if candidate.size < shortest.size - end - end - require shortest - end - end -end - diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb new file mode 100644 index 0000000..ef820da --- /dev/null +++ b/lib/vmpooler.rb @@ -0,0 +1,21 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + require 'json' + require 'open-uri' + require 'rbvmomi' + require 'redis' + require 'sinatra/base' + require 'time' + require 'timeout' + require 'yaml' + + %w( api graphite logger pool_manager vsphere_helper ).each do |lib| + begin + require "vmpooler/#{lib}" + rescue LoadError + require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib)) + end + end +end + diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb new file mode 100644 index 0000000..d137f86 --- /dev/null +++ b/lib/vmpooler/api.rb @@ -0,0 +1,304 @@ +module Vmpooler + class API + def initialize + # Load the configuration file + config_file = File.expand_path('vmpooler.yaml') + $config = YAML.load_file(config_file) + + # Set some defaults + $config[:redis] ||= Hash.new + $config[:redis]['server'] ||= 'localhost' + + if ($config[:graphite]['server']) + $config[:graphite]['prefix'] ||= 'vmpooler' + end + + # Connect to Redis + $redis = Redis.new(:host => $config[:redis]['server']) + end + + def execute! + my_app = Sinatra.new { + + set :environment, 'production' + + get '/' do + erb :dashboard, locals: { + site_name: $config[:config]['site_name'] || 'vmpooler', + } + end + + get '/dashboard/stats/vmpooler/numbers/?' do + result = Hash.new + result['pending'] = 0 + result['cloning'] = 0 + result['booting'] = 0 + result['ready'] = 0 + result['running'] = 0 + result['completed'] = 0 + + $config[:pools].each do |pool| + result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] ) + result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] ) + result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] ) + result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] ) + end + + result['cloning'] = $redis.get( 'vmpooler__tasks__clone' ) + result['booting'] = result['pending'].to_i - result['cloning'].to_i + result['booting'] = 0 if result['booting'] < 0 + result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i + + content_type :json + JSON.pretty_generate(result) + end + + get '/dashboard/stats/vmpooler/pool/?' do + result = Hash.new + + $config[:pools].each do |pool| + result[pool['name']] ||= Hash.new + result[pool['name']]['size'] = pool['size'] + result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] ) + end + + if ( params[:history] ) + if ( $config[:graphite]['server'] ) + history ||= Hash.new + + begin + buffer = open( + 'http://'+$config[:graphite]['server']+'/render?target='+$config[:graphite]['prefix']+'.ready.*&from=-1hour&format=json' + ).read + history = JSON.parse( buffer ) + + history.each do |pool| + if pool['target'] =~ /.*\.(.*)$/ + pool['name'] = $1 + + if ( result[pool['name']] ) + pool['last'] = result[pool['name']]['size'] + result[pool['name']]['history'] ||= Array.new + + pool['datapoints'].each do |metric| + 8.times do |n| + if ( metric[0] ) + pool['last'] = metric[0].to_i + result[pool['name']]['history'].push( metric[0].to_i ) + else + result[pool['name']]['history'].push( pool['last'] ) + end + end + end + end + end + end + rescue + end + else + $config[:pools].each do |pool| + result[pool['name']] ||= Hash.new + result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ] + end + end + end + + content_type :json + JSON.pretty_generate(result) + end + + get '/dashboard/stats/vmpooler/running/?' do + result = Hash.new + + $config[:pools].each do |pool| + running = $redis.scard( 'vmpooler__running__' + pool['name'] ) + pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/ + + result[pool['major']] ||= Hash.new + + result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i + end + + if ( params[:history] ) + if ( $config[:graphite]['server'] ) + begin + buffer = open( + 'http://'+$config[:graphite]['server']+'/render?target='+$config[:graphite]['prefix']+'.running.*&from=-1hour&format=json' + ).read + JSON.parse( buffer ).each do |pool| + if pool['target'] =~ /.*\.(.*)$/ + pool['name'] = $1 + + pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/ + + result[pool['major']]['history'] ||= Array.new + + for i in 0..pool['datapoints'].length + if ( + pool['datapoints'][i] and + pool['datapoints'][i][0] + ) + pool['last'] = pool['datapoints'][i][0] + + result[pool['major']]['history'][i] ||= 0 + result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i + else + result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i + end + end + + end + end + rescue + end + end + end + + content_type :json + JSON.pretty_generate(result) + end + + get '/vm/?' do + content_type :json + + result = [] + + $config[:pools].each do |pool| + result.push(pool['name']) + end + + JSON.pretty_generate(result) + end + + post '/vm/?' do + content_type :json + + result = {} + + available = 1 + + jdata = JSON.parse(request.body.read) + + jdata.each do |template, count| + if ( $redis.scard('vmpooler__ready__'+template) < count.to_i ) + available = 0 + end + end + + if ( available == 1 ) + result['ok'] = true + + jdata.each do |template, count| + result[template] ||= {} + + count.to_i.times do |i| + vm = $redis.spop('vmpooler__ready__'+template) + + unless (vm.nil?) + $redis.sadd('vmpooler__running__'+template, vm) + $redis.hset('vmpooler__active__'+template, vm, Time.now) + + result[template] ||= {} + + if ( result[template]['hostname'] ) + result[template]['hostname'] = [result[template]['hostname']] if ! result[template]['hostname'].is_a?(Array) + result[template]['hostname'].push(vm) + else + result[template]['hostname'] = vm + end + else + result['ok'] = false + end + end + end + else + result['ok'] = false + end + + JSON.pretty_generate(result) + end + + get '/vm/:template/?' do + content_type :json + + result = {} + result[params[:template]] = {} + result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template]) + + JSON.pretty_generate(result) + end + + post '/vm/:template/?' do + content_type :json + + result = {} + request = {} + + params[:template].split('+').each do |template| + request[template] ||= 0 + request[template] = request[template] + 1 + end + + available = 1 + + request.keys.each do |template| + if ( $redis.scard('vmpooler__ready__'+template) < request[template] ) + available = 0 + end + end + + if ( available == 1 ) + result['ok'] = true + + params[:template].split('+').each do |template| + result[template] ||= {} + + vm = $redis.spop('vmpooler__ready__'+template) + + unless (vm.nil?) + $redis.sadd('vmpooler__running__'+template, vm) + $redis.hset('vmpooler__active__'+template, vm, Time.now) + + result[template] ||= {} + + if ( result[template]['hostname'] ) + result[template]['hostname'] = [result[template]['hostname']] if ! result[template]['hostname'].is_a?(Array) + result[template]['hostname'].push(vm) + else + result[template]['hostname'] = vm + end + else + result['ok'] = false + end + end + else + result['ok'] = false + end + + JSON.pretty_generate(result) + end + + delete '/vm/:hostname/?' do + content_type :json + + result = {} + + result['ok'] = false + + $config[:pools].each do |pool| + if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname]) + $redis.srem('vmpooler__running__'+pool['name'], params[:hostname]) + $redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname]) + result['ok'] = true + end + end + + JSON.pretty_generate(result) + end + } + + my_app.run! + end + end +end + diff --git a/lib/vmpooler/graphite.rb b/lib/vmpooler/graphite.rb new file mode 100644 index 0000000..ecdb8ea --- /dev/null +++ b/lib/vmpooler/graphite.rb @@ -0,0 +1,20 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + class Graphite + def initialize( + s = 'graphite' + ) + @server = s + end + + def log path, value + Thread.new { + socket = TCPSocket.new(@server, 2003) + socket.puts "#{path} #{value} #{Time.now.to_i}" + socket.close + } + end + end +end + diff --git a/lib/vmpooler/logger.rb b/lib/vmpooler/logger.rb new file mode 100644 index 0000000..de97481 --- /dev/null +++ b/lib/vmpooler/logger.rb @@ -0,0 +1,21 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + class Logger + def initialize( + f = '/var/log/vmpooler.log' + ) + @file = f + end + + def log level, string + time = Time.new + stamp = time.strftime('%Y-%m-%d %H:%M:%S') + + open(@file, 'a') do |f| + f.puts "[#{stamp}] #{string}" + end + end + end +end + diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb new file mode 100644 index 0000000..a990d70 --- /dev/null +++ b/lib/vmpooler/pool_manager.rb @@ -0,0 +1,451 @@ +module Vmpooler + class PoolManager + def initialize + # Load the configuration file + config_file = File.expand_path('vmpooler.yaml') + $config = YAML.load_file(config_file) + + $pools = $config[:pools] + + # Set some defaults + $config[:config]['task_limit'] ||= 10 + $config[:config]['vm_checktime'] ||= 15 + $config[:config]['vm_lifetime'] ||= 24 + $config[:redis] ||= Hash.new + $config[:redis]['server'] ||= 'localhost' + + # Load logger library + $logger = Vmpooler::Logger.new $config[:config]['logfile'] + + # Load Graphite helper library (if configured) + if (defined? $config[:graphite]['server']) + $config[:graphite]['prefix'] ||= 'vmpooler' + $graphite = Vmpooler::Graphite.new $config[:graphite]['server'] + end + + # Connect to Redis + $redis = Redis.new(:host => $config[:redis]['server']) + + # vSphere object + $vsphere = {} + + # Our thread-tracker object + $threads = {} + end + + + # Check the state of a VM + def check_pending_vm vm, pool, timeout + Thread.new { + host = $vsphere[pool].find_vm(vm) + + if (host) + if ( + (host.summary) and + (host.summary.guest) and + (host.summary.guest.hostName) and + (host.summary.guest.hostName == vm) + ) + begin + Socket.getaddrinfo(vm, nil) + rescue + end + + $redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm) + + $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") + end + else + clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone') + + if ( + (clone_stamp) and + (((Time.now - Time.parse(clone_stamp))/60) > timeout) + ) + $redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") + end + end + } + end + + def check_ready_vm vm, pool, ttl + Thread.new { + if (ttl > 0) + if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) + $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + end + end + + check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check') + + if ( + (! check_stamp) or + (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime']) + ) + $redis.hset('vmpooler__vm__'+vm, 'check', Time.now) + + host = $vsphere[pool].find_vm(vm) || + $vsphere[pool].find_vm_heavy(vm)[vm] + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState != 'poweredOn') + ) + $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end + + if ( + (host.summary.guest) and + (host.summary.guest.hostName) and + (host.summary.guest.hostName != vm) + ) + $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") + end + else + $redis.srem('vmpooler__ready__'+pool, vm) + + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") + end + + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + end + end + end + } + end + + def check_running_vm vm, pool, ttl + Thread.new { + host = $vsphere[pool].find_vm(vm) + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState != 'poweredOn') + ) + $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") + else + if ( + (host.runtime) and + (host.runtime.bootTime) + ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) + ) + $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes") + end + end + end + } + end + + # Clone a VM + def clone_vm template, pool, folder, datastore + Thread.new { + vm = {} + + if template =~ /\// + templatefolders = template.split('/') + vm['template'] = templatefolders.pop + end + + if templatefolders + vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template']) + else + raise "Please provide a full path to the template" + end + + if vm['template'].length == 0 + raise "Unable to find template '#{vm['template']}'!" + end + + # Generate a randomized hostname + o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten + vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join + + # Add VM to Redis inventory ('pending' pool) + $redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname']) + $redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now) + + # Annotate with creation time, origin template, etc. + configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( + :annotation => JSON.pretty_generate({ + name: vm['hostname'], + created_by: $config[:vsphere]['username'], + base_template: vm['template'], + creation_timestamp: Time.now.utc + }) + ) + + # Put the VM in the specified folder and resource pool + relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( + :datastore => $vsphere[vm['template']].find_datastore(datastore), + :pool => $vsphere[vm['template']].find_pool(pool), + :diskMoveType => :moveChildMostDiskBacking + ) + + # Create a clone spec + spec = RbVmomi::VIM.VirtualMachineCloneSpec( + :location => relocateSpec, + :config => configSpec, + :powerOn => true, + :template => false + ) + + # Clone the VM + $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + + begin + start = Time.now + vm[vm['template']].CloneVM_Task( + :folder => $vsphere[vm['template']].find_folder(folder), + :name => vm['hostname'], + :spec => spec + ).wait_for_completion + finish = '%.2f' % (Time.now-start) + + $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + rescue + $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed") + $redis.srem('vmpooler__pending__'+vm['template'], vm['hostname']) + end + + $redis.decr('vmpooler__tasks__clone') + + begin + $graphite.log($config[:graphite]['prefix']+".clone.#{vm['template']}", finish) if defined? $graphite + rescue + end + } + end + + # Destroy a VM + def destroy_vm vm, pool + Thread.new { + $redis.srem('vmpooler__completed__'+pool, vm) + $redis.hdel('vmpooler__active__'+pool, vm) + $redis.del('vmpooler__vm__'+vm) + + host = $vsphere[pool].find_vm(vm) || + $vsphere[pool].find_vm_heavy(vm)[vm] + + if (host) + start = Time.now + + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState == 'poweredOn') + ) + $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") + host.PowerOffVM_Task.wait_for_completion + end + + host.Destroy_Task.wait_for_completion + finish = '%.2f' % (Time.now-start) + + $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") + + $graphite.log($config[:graphite]['prefix']+".destroy.#{pool}", finish) if defined? $graphite + end + } + end + + def check_pool pool + $logger.log('d', "[*] [#{pool['name']}] starting worker thread") + + $threads[pool['name']] = Thread.new { + $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new + + loop do + # INVENTORY + inventory = {} + begin + base = $vsphere[pool['name']].find_pool(pool['pool']) + + base.vm.each do |vm| + if ( + (! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and + (! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and + (! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and + (! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and + (! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name'])) + ) + $redis.sadd('vmpooler__discovered__'+pool['name'], vm['name']) + + $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue") + end + + inventory[vm['name']] = 1 + end + rescue + end + + # RUNNING + $redis.smembers('vmpooler__running__'+pool['name']).each do |vm| + if (inventory[vm]) + if (pool['running_ttl']) + begin + check_running_vm(vm, pool['name'], pool['running_ttl']) + rescue + end + else + begin + check_running_vm(vm, pool['name'], '720') + rescue + end + end + end + end + + # READY + $redis.smembers('vmpooler__ready__'+pool['name']).each do |vm| + if (inventory[vm]) + begin + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) + rescue + end + end + end + + # PENDING + $redis.smembers('vmpooler__pending__'+pool['name']).each do |vm| + pool['timeout'] ||= 15 + + if (inventory[vm]) + begin + check_pending_vm(vm, pool['name'], pool['timeout']) + rescue + end + end + end + + # COMPLETED + $redis.smembers('vmpooler__completed__'+pool['name']).each do |vm| + if (inventory[vm]) + begin + destroy_vm(vm, pool['name']) + rescue + $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") + $redis.srem('vmpooler__completed__'+pool['name'], vm) + $redis.hdel('vmpooler__active__'+pool['name'], vm) + $redis.del('vmpooler__vm__'+vm) + end + else + $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue") + $redis.srem('vmpooler__completed__'+pool['name'], vm) + $redis.hdel('vmpooler__active__'+pool['name'], vm) + $redis.del('vmpooler__vm__'+vm) + end + end + + # DISCOVERED + $redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm| + ['pending', 'ready', 'running', 'completed'].each do |queue| + if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm)) + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue") + $redis.srem('vmpooler__discovered__'+pool['name'], vm) + end + end + + if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm)) + $redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm) + end + end + + # LONG-RUNNING + $redis.smembers('vmpooler__running__'+pool['name']).each do |vm| + if ($redis.hget('vmpooler__active__'+pool['name'], vm)) + running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60 + if ( + ($config[:config]['vm_lifetime'] > 0) and + (running > $config[:config]['vm_lifetime']) + ) + $redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm) + + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours") + end + end + end + + # REPOPULATE + total = $redis.scard('vmpooler__ready__'+pool['name']) + + $redis.scard('vmpooler__pending__'+pool['name']) + + begin + if (defined? $graphite) + $graphite.log($config[:graphite]['prefix']+'.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name'])) + $graphite.log($config[:graphite]['prefix']+'.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name'])) + end + rescue + end + + if (total < pool['size']) + (1..(pool['size'] - total)).each { |i| + + if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit']) + begin + $redis.incr('vmpooler__tasks__clone') + + clone_vm( + pool['template'], + pool['pool'], + pool['folder'], + pool['datastore'] + ) + rescue + $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed") + $redis.decr('vmpooler__tasks__clone') + end + end + } + end + + sleep(1) + end + } + end + + def execute! + $logger.log('d', "starting vmpooler") + + # Clear out the tasks manager, as we don't know about any tasks at this point + $redis.set('vmpooler__tasks__clone', 0) + + loop do + $pools.each do |pool| + if (! $threads[pool['name']]) + check_pool(pool) + else + if (! $threads[pool['name']].alive?) + $logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting") + check_pool(pool) + end + end + end + + sleep(1) + end + end + + end +end + diff --git a/public/dashboard.css b/lib/vmpooler/public/dashboard.css similarity index 99% rename from public/dashboard.css rename to lib/vmpooler/public/dashboard.css index 3134e8d..86680a9 100644 --- a/public/dashboard.css +++ b/lib/vmpooler/public/dashboard.css @@ -140,5 +140,5 @@ body, .tick { fill: none; - stroke: #eee; + stroke: #ddd; } diff --git a/lib/vmpooler/public/img/bg.png b/lib/vmpooler/public/img/bg.png new file mode 100644 index 0000000..f07c531 Binary files /dev/null and b/lib/vmpooler/public/img/bg.png differ diff --git a/public/img/logo.jpg b/lib/vmpooler/public/img/logo.jpg similarity index 100% rename from public/img/logo.jpg rename to lib/vmpooler/public/img/logo.jpg diff --git a/public/img/spinner.svg b/lib/vmpooler/public/img/spinner.svg similarity index 100% rename from public/img/spinner.svg rename to lib/vmpooler/public/img/spinner.svg diff --git a/public/lib/stats-vmpooler-numbers.js b/lib/vmpooler/public/lib/stats-vmpooler-numbers.js similarity index 100% rename from public/lib/stats-vmpooler-numbers.js rename to lib/vmpooler/public/lib/stats-vmpooler-numbers.js diff --git a/public/lib/stats-vmpooler-pool.js b/lib/vmpooler/public/lib/stats-vmpooler-pool.js similarity index 86% rename from public/lib/stats-vmpooler-pool.js rename to lib/vmpooler/public/lib/stats-vmpooler-pool.js index 83d343f..689f144 100644 --- a/public/lib/stats-vmpooler-pool.js +++ b/lib/vmpooler/public/lib/stats-vmpooler-pool.js @@ -74,6 +74,27 @@ d3.json( pool_url+'?history=1', .attr( 'width', pool_width ) .attr( 'height', pool_height ); + defs = stats_vmpooler_pool__svg[ pool ].append( 'svg:defs' ); + + defs.append( 'svg:pattern' ) + .attr( 'id', 'background' ) + .attr( 'patternUnits', 'userSpaceOnUse' ) + .attr( 'width', '600px' ) + .attr( 'height', '100px' ) + .append( 'svg:image' ) + .attr( 'xlink:href', '/img/bg.png' ) + .attr( 'x', 0 ) + .attr( 'y', 0 ) + .attr( 'width', '600px' ) + .attr( 'height', '100px' ); + + stats_vmpooler_pool__svg[ pool ] + .append( 'path' ) + .attr( 'class', 'area' ) + .attr( 'fill', 'url( #background )' ) + .attr( 'opacity', '0.50' ) + .attr( 'd', area( stats_vmpooler_pool__data[ pool ][ 'r' ] ) ); + stats_vmpooler_pool__svg[ pool ] .append( 'g' ) .attr( 'class', 'x tick' ) @@ -140,7 +161,7 @@ d3.json( pool_url+'?history=1', .append( 'path' ) .attr( 'class', 'area' ) .attr( 'fill', statuscolor ) - .attr( 'opacity', '0.25' ) + .attr( 'opacity', '0.35' ) .attr( 'd', area( stats_vmpooler_pool__data[ pool ][ 'r' ] ) ); stats_vmpooler_pool__svg[ pool ] diff --git a/public/lib/stats-vmpooler-running.js b/lib/vmpooler/public/lib/stats-vmpooler-running.js similarity index 87% rename from public/lib/stats-vmpooler-running.js rename to lib/vmpooler/public/lib/stats-vmpooler-running.js index 2e0bced..146a566 100644 --- a/public/lib/stats-vmpooler-running.js +++ b/lib/vmpooler/public/lib/stats-vmpooler-running.js @@ -118,9 +118,9 @@ d3.json( running_url+'?history=1', var svg = d3.select( '#stats-vmpooler-running' ) .append( 'svg' ) .attr( 'height', running_height ) - .attr( 'width', '100%' ) - .style( 'margin-top', '15px' ) - .style( 'margin-bottom', '10px' ) + .attr( 'width', ( document.getElementById( 'stats-vmpooler-running' ).offsetWidth - 35 ) ) + .style( 'margin', '15px 0px 0px 0px' ) + .style( 'padding', '0px 10px 10px 10px' ) .append( 'g' ); var mysvg = svg.selectAll( '#stats-vmpooler-running' ) @@ -128,6 +128,27 @@ d3.json( running_url+'?history=1', .enter() .append( 'g' ); + defs = mysvg.append( 'svg:defs' ); + + defs.append( 'svg:pattern' ) + .attr( 'id', 'background' ) + .attr( 'patternUnits', 'userSpaceOnUse' ) + .attr( 'width', '600px' ) + .attr( 'height', '100px' ) + .append( 'svg:image' ) + .attr( 'xlink:href', '/img/bg.png' ) + .attr( 'x', 0 ) + .attr( 'y', 0 ) + .attr( 'width', '600px' ) + .attr( 'height', '100px' ); + + mysvg + .append( 'path' ) + .attr( 'class', 'area' ) + .attr( 'fill', 'url( #background )' ) + .attr( 'opacity', '0.50' ) + .attr( 'd', function( d ) { return area( d.values ); } ); + mysvg.append( 'path' ) .attr( 'd', function( d ) { return area( d.values ); } ) .attr( 'clas', 'area' ) diff --git a/views/dashboard.erb b/lib/vmpooler/views/dashboard.erb similarity index 100% rename from views/dashboard.erb rename to lib/vmpooler/views/dashboard.erb diff --git a/views/layout.erb b/lib/vmpooler/views/layout.erb similarity index 100% rename from views/layout.erb rename to lib/vmpooler/views/layout.erb diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb new file mode 100644 index 0000000..402530b --- /dev/null +++ b/lib/vmpooler/vsphere_helper.rb @@ -0,0 +1,174 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + class VsphereHelper + def initialize vInfo = {} + config_file = File.expand_path('vmpooler.yaml') + vsphere = YAML.load_file(config_file)[:vsphere] + + @connection = RbVmomi::VIM.connect :host => vsphere['server'], + :user => vsphere['username'], + :password => vsphere['password'], + :insecure => true + end + + + + def find_datastore datastorename + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + datacenter = @connection.serviceInstance.find_datacenter + datacenter.find_datastore(datastorename) + end + + + + def find_folder foldername + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.vmFolder + folders = foldername.split('/') + folders.each do |folder| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == folder } + else + abort "Unexpected object type encountered (#{base.class}) while finding folder" + end + end + + base + end + + + + def find_pool poolname + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.hostFolder + pools = poolname.split('/') + pools.each do |pool| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == pool } + when RbVmomi::VIM::ClusterComputeResource + base = base.resourcePool.resourcePool.find { |f| f.name == pool } + when RbVmomi::VIM::ResourcePool + base = base.resourcePool.find { |f| f.name == pool } + else + abort "Unexpected object type encountered (#{base.class}) while finding resource pool" + end + end + + base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool) + base + end + + + + def find_vm vmname + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname) + end + + + + def find_vm_heavy vmname + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + vmname = vmname.is_a?(Array) ? vmname : [ vmname ] + containerView = get_base_vm_container_from @connection + propertyCollector = @connection.propertyCollector + + objectSet = [{ + :obj => containerView, + :skip => true, + :selectSet => [ RbVmomi::VIM::TraversalSpec.new({ + :name => 'gettingTheVMs', + :path => 'view', + :skip => false, + :type => 'ContainerView' + }) ] + }] + + propSet = [{ + :pathSet => [ 'name' ], + :type => 'VirtualMachine' + }] + + results = propertyCollector.RetrievePropertiesEx({ + :specSet => [{ + :objectSet => objectSet, + :propSet => propSet + }], + :options => { :maxObjects => nil } + }) + + vms = {} + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + + while results.token do + results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token}) + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + end + + vms + end + + + + def get_base_vm_container_from connection + begin + connection.serviceInstance.CurrentTime + rescue + initialize() + end + + viewManager = connection.serviceContent.viewManager + viewManager.CreateContainerView({ + :container => connection.serviceContent.rootFolder, + :recursive => true, + :type => [ 'VirtualMachine' ] + }) + end + + + + def close + @connection.close + end + + end +end + diff --git a/lib/vsphere_helper.rb b/lib/vsphere_helper.rb deleted file mode 100755 index 1dfa018..0000000 --- a/lib/vsphere_helper.rb +++ /dev/null @@ -1,180 +0,0 @@ -require 'yaml' unless defined?(YAML) -require 'rubygems' unless defined?(Gem) - -class VsphereHelper - def initialize vInfo = {} - begin - require 'rbvmomi' - rescue LoadError - raise "Unable to load RbVmomi, please ensure its installed" - end - - Dir.chdir(File.dirname(__FILE__)) - - config_file = File.expand_path('../vmpooler.yaml') - vsphere = YAML.load_file(config_file)[:vsphere] - - @connection = RbVmomi::VIM.connect :host => vsphere['server'], - :user => vsphere['username'], - :password => vsphere['password'], - :insecure => true - end - - - - def find_datastore datastorename - begin - @connection.serviceInstance.CurrentTime - rescue - initialize() - end - - datacenter = @connection.serviceInstance.find_datacenter - datacenter.find_datastore(datastorename) - end - - - - def find_folder foldername - begin - @connection.serviceInstance.CurrentTime - rescue - initialize() - end - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.vmFolder - folders = foldername.split('/') - folders.each do |folder| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == folder } - else - abort "Unexpected object type encountered (#{base.class}) while finding folder" - end - end - - base - end - - - - def find_pool poolname - begin - @connection.serviceInstance.CurrentTime - rescue - initialize() - end - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.hostFolder - pools = poolname.split('/') - pools.each do |pool| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == pool } - when RbVmomi::VIM::ClusterComputeResource - base = base.resourcePool.resourcePool.find { |f| f.name == pool } - when RbVmomi::VIM::ResourcePool - base = base.resourcePool.find { |f| f.name == pool } - else - abort "Unexpected object type encountered (#{base.class}) while finding resource pool" - end - end - - base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool) - base - end - - - - def find_vm vmname - begin - @connection.serviceInstance.CurrentTime - rescue - initialize() - end - - @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname) - end - - - - def find_vm_heavy vmname - begin - @connection.serviceInstance.CurrentTime - rescue - initialize() - end - - vmname = vmname.is_a?(Array) ? vmname : [ vmname ] - containerView = get_base_vm_container_from @connection - propertyCollector = @connection.propertyCollector - - objectSet = [{ - :obj => containerView, - :skip => true, - :selectSet => [ RbVmomi::VIM::TraversalSpec.new({ - :name => 'gettingTheVMs', - :path => 'view', - :skip => false, - :type => 'ContainerView' - }) ] - }] - - propSet = [{ - :pathSet => [ 'name' ], - :type => 'VirtualMachine' - }] - - results = propertyCollector.RetrievePropertiesEx({ - :specSet => [{ - :objectSet => objectSet, - :propSet => propSet - }], - :options => { :maxObjects => nil } - }) - - vms = {} - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - - while results.token do - results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token}) - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - end - - vms - end - - - - def get_base_vm_container_from connection - begin - connection.serviceInstance.CurrentTime - rescue - initialize() - end - - viewManager = connection.serviceContent.viewManager - viewManager.CreateContainerView({ - :container => connection.serviceContent.rootFolder, - :recursive => true, - :type => [ 'VirtualMachine' ] - }) - end - - - - def close - @connection.close - end -end - diff --git a/vmpooler b/vmpooler index a304eca..0ffc1d7 100755 --- a/vmpooler +++ b/vmpooler @@ -1,463 +1,14 @@ -#!/usr/bin/ruby - -require 'json' -require 'rbvmomi' -require 'redis' -require 'time' -require 'timeout' -require 'yaml' +#!/usr/bin/env ruby $:.unshift(File.dirname(__FILE__)) -require 'lib/logger' -require 'lib/require_relative' -require 'lib/vsphere_helper' -Dir.chdir(File.dirname(__FILE__)) +require 'rubygems' unless defined?(Gem) +require 'lib/vmpooler' -# Load the configuration file -config_file = File.expand_path('vmpooler.yaml') -$config = YAML.load_file(config_file) - -pools = $config[:pools] -vsphere = $config[:vsphere] -redis = $config[:redis] - -# Load logger library -$logger = Logger.new $config[:config]['logfile'] - -# Load Graphite helper library (if configured) -if (defined? $config[:config]['graphite']) - require 'lib/graphite' - $graphite = Graphite.new $config[:config]['graphite'] -end - -# Set some defaults -$config[:config]['task_limit'] ||= 10 -$config[:config]['vm_checktime'] ||= 15 -$config[:config]['vm_lifetime'] ||= 24 -$config[:redis] ||= Hash.new -$config[:redis]['server'] ||= 'localhost' - -# Connect to Redis -$redis = Redis.new(:host => $config[:redis]['server']) - -# vSphere object -$vsphere = {} - -# Our thread-tracker object -$threads = {} - - - -# Check the state of a VM -def check_pending_vm vm, pool, timeout - Thread.new { - host = $vsphere[pool].find_vm(vm) - - if (host) - if ( - (host.summary) and - (host.summary.guest) and - (host.summary.guest.hostName) and - (host.summary.guest.hostName == vm) - ) - begin - Socket.getaddrinfo(vm, nil) - rescue - end - - $redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm) - - $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") - end - else - clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone') - - if ( - (clone_stamp) and - (((Time.now - Time.parse(clone_stamp))/60) > timeout) - ) - $redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") - end - end - } -end - -def check_ready_vm vm, pool, ttl - Thread.new { - if (ttl > 0) - if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) - $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - end - end - - check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check') - - if ( - (! check_stamp) or - (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime']) - ) - $redis.hset('vmpooler__vm__'+vm, 'check', Time.now) - - host = $vsphere[pool].find_vm(vm) || - $vsphere[pool].find_vm_heavy(vm)[vm] - - if (host) - if ( - (host.runtime) and - (host.runtime.powerState) and - (host.runtime.powerState != 'poweredOn') - ) - $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") - end - - if ( - (host.summary.guest) and - (host.summary.guest.hostName) and - (host.summary.guest.hostName != vm) - ) - $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") - end - else - $redis.srem('vmpooler__ready__'+pool, vm) - - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - Timeout::timeout(5) { - TCPSocket.new vm, 22 - } - rescue - if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") - end - end - end - } -end - -def check_running_vm vm, pool, ttl - Thread.new { - host = $vsphere[pool].find_vm(vm) - - if (host) - if ( - (host.runtime) and - (host.runtime.powerState != 'poweredOn') - ) - $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - else - if ( - (host.runtime) and - (host.runtime.bootTime) - ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) - ) - $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes") - end - end - end - } -end - -# Clone a VM -def clone_vm template, pool, folder, datastore - Thread.new { - vm = {} - - if template =~ /\// - templatefolders = template.split('/') - vm['template'] = templatefolders.pop - end - - if templatefolders - vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template']) - else - raise "Please provide a full path to the template" - end - - if vm['template'].length == 0 - raise "Unable to find template '#{vm['template']}'!" - end - - # Generate a randomized hostname - o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten - vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join - - # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname']) - $redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now) - - # Annotate with creation time, origin template, etc. - configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - :annotation => JSON.pretty_generate({ - name: vm['hostname'], - created_by: $config[:vsphere]['username'], - base_template: vm['template'], - creation_timestamp: Time.now.utc - }) - ) - - # Put the VM in the specified folder and resource pool - relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - :datastore => $vsphere[vm['template']].find_datastore(datastore), - :pool => $vsphere[vm['template']].find_pool(pool), - :diskMoveType => :moveChildMostDiskBacking - ) - - # Create a clone spec - spec = RbVmomi::VIM.VirtualMachineCloneSpec( - :location => relocateSpec, - :config => configSpec, - :powerOn => true, - :template => false - ) - - # Clone the VM - $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") - - begin - start = Time.now - vm[vm['template']].CloneVM_Task( - :folder => $vsphere[vm['template']].find_folder(folder), - :name => vm['hostname'], - :spec => spec - ).wait_for_completion - finish = '%.2f' % (Time.now-start) - - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") - rescue - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed") - $redis.srem('vmpooler__pending__'+vm['template'], vm['hostname']) - end - - $redis.decr('vmpooler__tasks__clone') - - begin - $graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite - rescue - end - } -end - -# Destroy a VM -def destroy_vm vm, pool - Thread.new { - $redis.srem('vmpooler__completed__'+pool, vm) - $redis.hdel('vmpooler__active__'+pool, vm) - $redis.del('vmpooler__vm__'+vm) - - host = $vsphere[pool].find_vm(vm) || - $vsphere[pool].find_vm_heavy(vm)[vm] - - if (host) - start = Time.now - - if ( - (host.runtime) and - (host.runtime.powerState) and - (host.runtime.powerState == 'poweredOn') - ) - $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") - host.PowerOffVM_Task.wait_for_completion - end - - host.Destroy_Task.wait_for_completion - finish = '%.2f' % (Time.now-start) - - $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") - - $graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite - end - } -end - -def check_pool pool - $logger.log('d', "[*] [#{pool['name']}] starting worker thread") - - $threads[pool['name']] = Thread.new { - $vsphere[pool['name']] ||= VsphereHelper.new - - loop do - # INVENTORY - inventory = {} - begin - base = $vsphere[pool['name']].find_pool(pool['pool']) - - base.vm.each do |vm| - if ( - (! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and - (! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and - (! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and - (! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and - (! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name'])) - ) - $redis.sadd('vmpooler__discovered__'+pool['name'], vm['name']) - - $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue") - end - - inventory[vm['name']] = 1 - end - rescue - end - - # RUNNING - $redis.smembers('vmpooler__running__'+pool['name']).each do |vm| - if (inventory[vm]) - if (pool['running_ttl']) - begin - check_running_vm(vm, pool['name'], pool['running_ttl']) - rescue - end - else - begin - check_running_vm(vm, pool['name'], '720') - rescue - end - end - end - end - - # READY - $redis.smembers('vmpooler__ready__'+pool['name']).each do |vm| - if (inventory[vm]) - begin - check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) - rescue - end - end - end - - # PENDING - $redis.smembers('vmpooler__pending__'+pool['name']).each do |vm| - pool['timeout'] ||= 15 - - if (inventory[vm]) - begin - check_pending_vm(vm, pool['name'], pool['timeout']) - rescue - end - end - end - - # COMPLETED - $redis.smembers('vmpooler__completed__'+pool['name']).each do |vm| - if (inventory[vm]) - begin - destroy_vm(vm, pool['name']) - rescue - $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") - $redis.srem('vmpooler__completed__'+pool['name'], vm) - $redis.hdel('vmpooler__active__'+pool['name'], vm) - $redis.del('vmpooler__vm__'+vm) - end - else - $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue") - $redis.srem('vmpooler__completed__'+pool['name'], vm) - $redis.hdel('vmpooler__active__'+pool['name'], vm) - $redis.del('vmpooler__vm__'+vm) - end - end - - # DISCOVERED - $redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm| - ['pending', 'ready', 'running', 'completed'].each do |queue| - if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm)) - $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue") - $redis.srem('vmpooler__discovered__'+pool['name'], vm) - end - end - - if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm)) - $redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm) - end - end - - # LONG-RUNNING - $redis.smembers('vmpooler__running__'+pool['name']).each do |vm| - if ($redis.hget('vmpooler__active__'+pool['name'], vm)) - running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60 - if ( - ($config[:config]['vm_lifetime'] > 0) and - (running > $config[:config]['vm_lifetime']) - ) - $redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm) - - $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours") - end - end - end - - # REPOPULATE - total = $redis.scard('vmpooler__ready__'+pool['name']) + - $redis.scard('vmpooler__pending__'+pool['name']) - - begin - if (defined? $graphite) - $graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name'])) - $graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name'])) - end - rescue - end - - if (total < pool['size']) - (1..(pool['size'] - total)).each { |i| - - if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit']) - begin - $redis.incr('vmpooler__tasks__clone') - - clone_vm( - pool['template'], - pool['pool'], - pool['folder'], - pool['datastore'] - ) - rescue - $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed") - $redis.decr('vmpooler__tasks__clone') - end - end - } - end - - sleep(1) - end - } -end - - - -$logger.log('d', "starting vmpooler") - -# Clear out the tasks manager, as we don't know about any tasks at this point -$redis.set('vmpooler__tasks__clone', 0) +Thread.new { Vmpooler::API.new.execute! } +Thread.new { Vmpooler::PoolManager.new.execute! } loop do - pools.each do |pool| - if (! $threads[pool['name']]) - check_pool(pool) - else - if (! $threads[pool['name']].alive?) - $logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting") - check_pool(pool) - end - end - end - - sleep(1) + sleep(10) end diff --git a/vmpooler-api b/vmpooler-api deleted file mode 100755 index 6ecf611..0000000 --- a/vmpooler-api +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/ruby - -require 'rubygems' - -require 'json' -require 'open-uri' -require 'redis' -require 'sinatra' -require 'yaml' - -$:.unshift(File.dirname(__FILE__)) -require 'lib/logger' -require 'lib/require_relative' - -Dir.chdir(File.dirname(__FILE__)) - -# Load the configuration file -config_file = File.expand_path('vmpooler.yaml') -$config = YAML.load_file(config_file) - -pools = $config[:pools] -redis = $config[:redis] - -# Load logger library -$logger = Logger.new $config[:config]['logfile'] - -# Set some defaults -$config[:redis] ||= Hash.new -$config[:redis]['server'] ||= 'localhost' - -# Connect to Redis -$redis = Redis.new(:host => $config[:redis]['server']) - -# Sinatra! -get '/' do - erb :dashboard, locals: { - site_name: $config[:config]['site_name'] || 'vmpooler', - } -end - -get '/dashboard/stats/vmpooler/numbers/?' do - result = Hash.new - result['pending'] = 0 - result['cloning'] = 0 - result['booting'] = 0 - result['ready'] = 0 - result['running'] = 0 - result['completed'] = 0 - - $config[:pools].each do |pool| - result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] ) - result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] ) - result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] ) - result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] ) - end - - result['cloning'] = $redis.get( 'vmpooler__tasks__clone' ) - result['booting'] = result['pending'].to_i - result['cloning'].to_i - result['booting'] = 0 if result['booting'] < 0 - result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i - - content_type :json - JSON.pretty_generate(result) -end - -get '/dashboard/stats/vmpooler/pool/?' do - result = Hash.new - - $config[:pools].each do |pool| - result[pool['name']] ||= Hash.new - result[pool['name']]['size'] = pool['size'] - result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] ) - end - - if ( params[:history] ) - if ( $config[:config]['graphite'] ) - history ||= Hash.new - - begin - buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read - history = JSON.parse( buffer ) - - history.each do |pool| - if pool['target'] =~ /.*\.(.*)$/ - pool['name'] = $1 - - if ( result[pool['name']] ) - pool['last'] = result[pool['name']]['size'] - result[pool['name']]['history'] ||= Array.new - - pool['datapoints'].each do |metric| - 8.times do |n| - if ( metric[0] ) - pool['last'] = metric[0].to_i - result[pool['name']]['history'].push( metric[0].to_i ) - else - result[pool['name']]['history'].push( pool['last'] ) - end - end - end - end - end - end - rescue - end - else - $config[:pools].each do |pool| - result[pool['name']] ||= Hash.new - result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ] - end - end - end - - content_type :json - JSON.pretty_generate(result) -end - -get '/dashboard/stats/vmpooler/running/?' do - result = Hash.new - - $config[:pools].each do |pool| - running = $redis.scard( 'vmpooler__running__' + pool['name'] ) - pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/ - - result[pool['major']] ||= Hash.new - - result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i - end - - if ( params[:history] ) - if ( $config[:config]['graphite'] ) - begin - buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read - JSON.parse( buffer ).each do |pool| - if pool['target'] =~ /.*\.(.*)$/ - pool['name'] = $1 - - pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/ - - result[pool['major']]['history'] ||= Array.new - - for i in 0..pool['datapoints'].length - if ( - pool['datapoints'][i] and - pool['datapoints'][i][0] - ) - pool['last'] = pool['datapoints'][i][0] - - result[pool['major']]['history'][i] ||= 0 - result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i - else - result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i - end - end - - end - end - rescue - end - end - end - - content_type :json - JSON.pretty_generate(result) -end - -get '/vm/?' do - content_type :json - - result = [] - - pools.each do |pool| - result.push(pool['name']) - end - - JSON.pretty_generate(result) -end - -get '/vm/:template/?' do - content_type :json - - result = {} - result[params[:template]] = {} - result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template]) - - JSON.pretty_generate(result) -end - -post '/vm/:template/?' do - content_type :json - - result = {} - result[params[:template]] = {} - - if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 ) - vm = $redis.spop('vmpooler__ready__'+params[:template]) - - unless (vm.nil?) - $redis.sadd('vmpooler__running__'+params[:template], vm) - $redis.hset('vmpooler__active__'+params[:template], vm, Time.now) - - result[params[:template]]['ok'] = true - result[params[:template]]['hostname'] = vm - else - result[params[:template]]['ok'] = false - end - else - result[params[:template]]['ok'] = false - end - - JSON.pretty_generate(result) -end - -delete '/vm/:hostname/?' do - content_type :json - - result = {} - - result['ok'] = false - - pools.each do |pool| - if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname]) - $redis.srem('vmpooler__running__'+pool['name'], params[:hostname]) - $redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname]) - result['ok'] = true - end - end - - JSON.pretty_generate(result) -end - diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 4746db2..c0d4b44 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -49,6 +49,26 @@ :redis: server: 'redis.company.com' +# :graphite: +# +# This section contains the connection information required to store +# historical data in an external Graphite database. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the Graphite server. +# (optional) +# +# - prefix +# The prefix to use while storing Graphite data. +# (optional; default: 'vmpooler') + +# Example: + +:graphite: + server: 'graphite.company.com' + # :config: # # This section contains global configuration information. @@ -63,10 +83,6 @@ # The path to vmpooler's log file. # (optional; default: '/var/log/vmpooler.log') # -# - graphite -# The FQDN hostname of the Graphite server. -# (optional) -# # - task_limit # The number of concurrent VMware vSphere tasks to perform. # (optional; default: '10') @@ -84,7 +100,6 @@ :config: site_name: 'vmpooler' logfile: '/var/log/vmpooler.log' - graphite: 'graphite.company.com' task_limit: 10 vm_checktime: 15 vm_lifetime: 12