mirror of
https://github.com/puppetlabs/vmpooler.git
synced 2026-01-26 01:58:41 -05:00
Reworked into a single namespaced Ruby application
This commit is contained in:
Scott Schneider
parent
e0ac8c735f
commit
60cc1ef178
20 changed files with 930 additions and 924 deletions
|
|
@ -1,18 +0,0 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
class Graphite
|
||||
def initialize(
|
||||
s = 'graphite'
|
||||
)
|
||||
@server = s
|
||||
end
|
||||
|
||||
def log path, value
|
||||
Thread.new {
|
||||
socket = TCPSocket.new(@server, 2003)
|
||||
socket.puts "#{path} #{value} #{Time.now.to_i}"
|
||||
socket.close
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
class Logger
|
||||
def initialize(
|
||||
f = '/var/log/vmpooler.log'
|
||||
)
|
||||
@file = f
|
||||
end
|
||||
|
||||
def log level, string
|
||||
time = Time.new
|
||||
stamp = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
puts "[#{stamp}] #{string}"
|
||||
|
||||
open(@file, 'a') do |f|
|
||||
f.puts "[#{stamp}] #{string}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
# require_relative was introduced in 1.9.2. This makes it
|
||||
# available to younger rubies. It trys hard to not re-require
|
||||
# files.
|
||||
|
||||
unless Kernel.respond_to?(:require_relative)
|
||||
module Kernel
|
||||
def require_relative(path)
|
||||
desired_path = File.expand_path('../'+path.to_str, caller[0])
|
||||
shortest = desired_path
|
||||
$:.each do |path|
|
||||
path += '/'
|
||||
if desired_path.index(path) == 0
|
||||
candidate = desired_path.sub(path, '')
|
||||
shortest = candidate if candidate.size < shortest.size
|
||||
end
|
||||
end
|
||||
require shortest
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
12
lib/vmpooler.rb
Normal file
12
lib/vmpooler.rb
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
module Vmpooler
|
||||
%w( api graphite logger pool_manager vsphere_helper ).each do |lib|
|
||||
begin
|
||||
require "vmpooler/#{lib}"
|
||||
rescue LoadError
|
||||
require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
233
lib/vmpooler/api.rb
Normal file
233
lib/vmpooler/api.rb
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
module Vmpooler
|
||||
class API
|
||||
def initialize
|
||||
require 'sinatra/base'
|
||||
|
||||
require 'json'
|
||||
require 'open-uri'
|
||||
require 'redis'
|
||||
require 'yaml'
|
||||
|
||||
# Load the configuration file
|
||||
config_file = File.expand_path('vmpooler.yaml')
|
||||
$config = YAML.load_file(config_file)
|
||||
|
||||
pools = $config[:pools]
|
||||
redis = $config[:redis]
|
||||
|
||||
# Set some defaults
|
||||
$config[:redis] ||= Hash.new
|
||||
$config[:redis]['server'] ||= 'localhost'
|
||||
|
||||
# Connect to Redis
|
||||
$redis = Redis.new(:host => $config[:redis]['server'])
|
||||
end
|
||||
|
||||
def execute!
|
||||
my_app = Sinatra.new {
|
||||
|
||||
set :environment, 'production'
|
||||
|
||||
get '/' do
|
||||
erb :dashboard, locals: {
|
||||
site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
|
||||
}
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/numbers/?' do
|
||||
result = Hash.new
|
||||
result['pending'] = 0
|
||||
result['cloning'] = 0
|
||||
result['booting'] = 0
|
||||
result['ready'] = 0
|
||||
result['running'] = 0
|
||||
result['completed'] = 0
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
|
||||
result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
|
||||
result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
|
||||
result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
|
||||
end
|
||||
|
||||
result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
|
||||
result['booting'] = result['pending'].to_i - result['cloning'].to_i
|
||||
result['booting'] = 0 if result['booting'] < 0
|
||||
result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/pool/?' do
|
||||
result = Hash.new
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
result[pool['name']] ||= Hash.new
|
||||
result[pool['name']]['size'] = pool['size']
|
||||
result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
|
||||
end
|
||||
|
||||
if ( params[:history] )
|
||||
if ( $config[:config]['graphite'] )
|
||||
history ||= Hash.new
|
||||
|
||||
begin
|
||||
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
|
||||
history = JSON.parse( buffer )
|
||||
|
||||
history.each do |pool|
|
||||
if pool['target'] =~ /.*\.(.*)$/
|
||||
pool['name'] = $1
|
||||
|
||||
if ( result[pool['name']] )
|
||||
pool['last'] = result[pool['name']]['size']
|
||||
result[pool['name']]['history'] ||= Array.new
|
||||
|
||||
pool['datapoints'].each do |metric|
|
||||
8.times do |n|
|
||||
if ( metric[0] )
|
||||
pool['last'] = metric[0].to_i
|
||||
result[pool['name']]['history'].push( metric[0].to_i )
|
||||
else
|
||||
result[pool['name']]['history'].push( pool['last'] )
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue
|
||||
end
|
||||
else
|
||||
$config[:pools].each do |pool|
|
||||
result[pool['name']] ||= Hash.new
|
||||
result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/running/?' do
|
||||
result = Hash.new
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
running = $redis.scard( 'vmpooler__running__' + pool['name'] )
|
||||
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
|
||||
|
||||
result[pool['major']] ||= Hash.new
|
||||
|
||||
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
|
||||
end
|
||||
|
||||
if ( params[:history] )
|
||||
if ( $config[:config]['graphite'] )
|
||||
begin
|
||||
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
|
||||
JSON.parse( buffer ).each do |pool|
|
||||
if pool['target'] =~ /.*\.(.*)$/
|
||||
pool['name'] = $1
|
||||
|
||||
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
|
||||
|
||||
result[pool['major']]['history'] ||= Array.new
|
||||
|
||||
for i in 0..pool['datapoints'].length
|
||||
if (
|
||||
pool['datapoints'][i] and
|
||||
pool['datapoints'][i][0]
|
||||
)
|
||||
pool['last'] = pool['datapoints'][i][0]
|
||||
|
||||
result[pool['major']]['history'][i] ||= 0
|
||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
|
||||
else
|
||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/vm/?' do
|
||||
content_type :json
|
||||
|
||||
result = []
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
result.push(pool['name'])
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/vm/:template/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
result[params[:template]] = {}
|
||||
result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
post '/vm/:template/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
result[params[:template]] = {}
|
||||
|
||||
if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
|
||||
vm = $redis.spop('vmpooler__ready__'+params[:template])
|
||||
|
||||
unless (vm.nil?)
|
||||
$redis.sadd('vmpooler__running__'+params[:template], vm)
|
||||
$redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
|
||||
|
||||
result[params[:template]]['ok'] = true
|
||||
result[params[:template]]['hostname'] = vm
|
||||
else
|
||||
result[params[:template]]['ok'] = false
|
||||
end
|
||||
else
|
||||
result[params[:template]]['ok'] = false
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
delete '/vm/:hostname/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
|
||||
result['ok'] = false
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
|
||||
$redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
|
||||
$redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
|
||||
result['ok'] = true
|
||||
end
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
}
|
||||
|
||||
my_app.run!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
20
lib/vmpooler/graphite.rb
Normal file
20
lib/vmpooler/graphite.rb
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
module Vmpooler
|
||||
class Graphite
|
||||
def initialize(
|
||||
s = 'graphite'
|
||||
)
|
||||
@server = s
|
||||
end
|
||||
|
||||
def log path, value
|
||||
Thread.new {
|
||||
socket = TCPSocket.new(@server, 2003)
|
||||
socket.puts "#{path} #{value} #{Time.now.to_i}"
|
||||
socket.close
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
21
lib/vmpooler/logger.rb
Normal file
21
lib/vmpooler/logger.rb
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
module Vmpooler
|
||||
class Logger
|
||||
def initialize(
|
||||
f = '/var/log/vmpooler.log'
|
||||
)
|
||||
@file = f
|
||||
end
|
||||
|
||||
def log level, string
|
||||
time = Time.new
|
||||
stamp = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
open(@file, 'a') do |f|
|
||||
f.puts "[#{stamp}] #{string}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
459
lib/vmpooler/pool_manager.rb
Normal file
459
lib/vmpooler/pool_manager.rb
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
module Vmpooler
|
||||
class PoolManager
|
||||
def initialize
|
||||
require 'json'
|
||||
require 'rbvmomi'
|
||||
require 'redis'
|
||||
require 'time'
|
||||
require 'timeout'
|
||||
require 'yaml'
|
||||
|
||||
# Load the configuration file
|
||||
config_file = File.expand_path('vmpooler.yaml')
|
||||
$config = YAML.load_file(config_file)
|
||||
|
||||
$pools = $config[:pools]
|
||||
vsphere = $config[:vsphere]
|
||||
redis = $config[:redis]
|
||||
|
||||
# Load logger library
|
||||
$logger = Vmpooler::Logger.new $config[:config]['logfile']
|
||||
|
||||
# Load Graphite helper library (if configured)
|
||||
if (defined? $config[:config]['graphite'])
|
||||
$graphite = Vmpooler::Graphite.new $config[:config]['graphite']
|
||||
end
|
||||
|
||||
# Set some defaults
|
||||
$config[:config]['task_limit'] ||= 10
|
||||
$config[:config]['vm_checktime'] ||= 15
|
||||
$config[:config]['vm_lifetime'] ||= 24
|
||||
$config[:redis] ||= Hash.new
|
||||
$config[:redis]['server'] ||= 'localhost'
|
||||
|
||||
# Connect to Redis
|
||||
$redis = Redis.new(:host => $config[:redis]['server'])
|
||||
|
||||
# vSphere object
|
||||
$vsphere = {}
|
||||
|
||||
# Our thread-tracker object
|
||||
$threads = {}
|
||||
end
|
||||
|
||||
|
||||
# Check the state of a VM
|
||||
def check_pending_vm vm, pool, timeout
|
||||
Thread.new {
|
||||
host = $vsphere[pool].find_vm(vm)
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.summary) and
|
||||
(host.summary.guest) and
|
||||
(host.summary.guest.hostName) and
|
||||
(host.summary.guest.hostName == vm)
|
||||
)
|
||||
begin
|
||||
Socket.getaddrinfo(vm, nil)
|
||||
rescue
|
||||
end
|
||||
|
||||
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
|
||||
|
||||
$logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
|
||||
end
|
||||
else
|
||||
clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
|
||||
|
||||
if (
|
||||
(clone_stamp) and
|
||||
(((Time.now - Time.parse(clone_stamp))/60) > timeout)
|
||||
)
|
||||
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_ready_vm vm, pool, ttl
|
||||
Thread.new {
|
||||
if (ttl > 0)
|
||||
if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
|
||||
end
|
||||
end
|
||||
|
||||
check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
|
||||
|
||||
if (
|
||||
(! check_stamp) or
|
||||
(((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
|
||||
)
|
||||
$redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
|
||||
|
||||
host = $vsphere[pool].find_vm(vm) ||
|
||||
$vsphere[pool].find_vm_heavy(vm)[vm]
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState) and
|
||||
(host.runtime.powerState != 'poweredOn')
|
||||
)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
|
||||
end
|
||||
|
||||
if (
|
||||
(host.summary.guest) and
|
||||
(host.summary.guest.hostName) and
|
||||
(host.summary.guest.hostName != vm)
|
||||
)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
|
||||
end
|
||||
else
|
||||
$redis.srem('vmpooler__ready__'+pool, vm)
|
||||
|
||||
$logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
|
||||
end
|
||||
|
||||
begin
|
||||
Timeout::timeout(5) {
|
||||
TCPSocket.new vm, 22
|
||||
}
|
||||
rescue
|
||||
if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_running_vm vm, pool, ttl
|
||||
Thread.new {
|
||||
host = $vsphere[pool].find_vm(vm)
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState != 'poweredOn')
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
|
||||
else
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.bootTime)
|
||||
((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
# Clone a VM
|
||||
def clone_vm template, pool, folder, datastore
|
||||
Thread.new {
|
||||
vm = {}
|
||||
|
||||
if template =~ /\//
|
||||
templatefolders = template.split('/')
|
||||
vm['template'] = templatefolders.pop
|
||||
end
|
||||
|
||||
if templatefolders
|
||||
vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
|
||||
else
|
||||
raise "Please provide a full path to the template"
|
||||
end
|
||||
|
||||
if vm['template'].length == 0
|
||||
raise "Unable to find template '#{vm['template']}'!"
|
||||
end
|
||||
|
||||
# Generate a randomized hostname
|
||||
o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
|
||||
vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
|
||||
|
||||
# Add VM to Redis inventory ('pending' pool)
|
||||
$redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
|
||||
$redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
|
||||
|
||||
# Annotate with creation time, origin template, etc.
|
||||
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
|
||||
:annotation => JSON.pretty_generate({
|
||||
name: vm['hostname'],
|
||||
created_by: $config[:vsphere]['username'],
|
||||
base_template: vm['template'],
|
||||
creation_timestamp: Time.now.utc
|
||||
})
|
||||
)
|
||||
|
||||
# Put the VM in the specified folder and resource pool
|
||||
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
|
||||
:datastore => $vsphere[vm['template']].find_datastore(datastore),
|
||||
:pool => $vsphere[vm['template']].find_pool(pool),
|
||||
:diskMoveType => :moveChildMostDiskBacking
|
||||
)
|
||||
|
||||
# Create a clone spec
|
||||
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
|
||||
:location => relocateSpec,
|
||||
:config => configSpec,
|
||||
:powerOn => true,
|
||||
:template => false
|
||||
)
|
||||
|
||||
# Clone the VM
|
||||
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
|
||||
|
||||
begin
|
||||
start = Time.now
|
||||
vm[vm['template']].CloneVM_Task(
|
||||
:folder => $vsphere[vm['template']].find_folder(folder),
|
||||
:name => vm['hostname'],
|
||||
:spec => spec
|
||||
).wait_for_completion
|
||||
finish = '%.2f' % (Time.now-start)
|
||||
|
||||
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
|
||||
$redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
|
||||
end
|
||||
|
||||
$redis.decr('vmpooler__tasks__clone')
|
||||
|
||||
begin
|
||||
$graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
|
||||
rescue
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
# Destroy a VM
|
||||
def destroy_vm vm, pool
|
||||
Thread.new {
|
||||
$redis.srem('vmpooler__completed__'+pool, vm)
|
||||
$redis.hdel('vmpooler__active__'+pool, vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
|
||||
host = $vsphere[pool].find_vm(vm) ||
|
||||
$vsphere[pool].find_vm_heavy(vm)[vm]
|
||||
|
||||
if (host)
|
||||
start = Time.now
|
||||
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState) and
|
||||
(host.runtime.powerState == 'poweredOn')
|
||||
)
|
||||
$logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
|
||||
host.PowerOffVM_Task.wait_for_completion
|
||||
end
|
||||
|
||||
host.Destroy_Task.wait_for_completion
|
||||
finish = '%.2f' % (Time.now-start)
|
||||
|
||||
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
||||
|
||||
$graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_pool pool
|
||||
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
|
||||
|
||||
$threads[pool['name']] = Thread.new {
|
||||
$vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
|
||||
|
||||
loop do
|
||||
# INVENTORY
|
||||
inventory = {}
|
||||
begin
|
||||
base = $vsphere[pool['name']].find_pool(pool['pool'])
|
||||
|
||||
base.vm.each do |vm|
|
||||
if (
|
||||
(! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
|
||||
)
|
||||
$redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
|
||||
|
||||
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
||||
end
|
||||
|
||||
inventory[vm['name']] = 1
|
||||
end
|
||||
rescue
|
||||
end
|
||||
|
||||
# RUNNING
|
||||
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
if (pool['running_ttl'])
|
||||
begin
|
||||
check_running_vm(vm, pool['name'], pool['running_ttl'])
|
||||
rescue
|
||||
end
|
||||
else
|
||||
begin
|
||||
check_running_vm(vm, pool['name'], '720')
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# READY
|
||||
$redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
begin
|
||||
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# PENDING
|
||||
$redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
|
||||
pool['timeout'] ||= 15
|
||||
|
||||
if (inventory[vm])
|
||||
begin
|
||||
check_pending_vm(vm, pool['name'], pool['timeout'])
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# COMPLETED
|
||||
$redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
begin
|
||||
destroy_vm(vm, pool['name'])
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
|
||||
$redis.srem('vmpooler__completed__'+pool['name'], vm)
|
||||
$redis.hdel('vmpooler__active__'+pool['name'], vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
end
|
||||
else
|
||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
||||
$redis.srem('vmpooler__completed__'+pool['name'], vm)
|
||||
$redis.hdel('vmpooler__active__'+pool['name'], vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
end
|
||||
end
|
||||
|
||||
# DISCOVERED
|
||||
$redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
|
||||
['pending', 'ready', 'running', 'completed'].each do |queue|
|
||||
if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
|
||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
||||
$redis.srem('vmpooler__discovered__'+pool['name'], vm)
|
||||
end
|
||||
end
|
||||
|
||||
if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
|
||||
$redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
|
||||
end
|
||||
end
|
||||
|
||||
# LONG-RUNNING
|
||||
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
|
||||
if ($redis.hget('vmpooler__active__'+pool['name'], vm))
|
||||
running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
|
||||
if (
|
||||
($config[:config]['vm_lifetime'] > 0) and
|
||||
(running > $config[:config]['vm_lifetime'])
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# REPOPULATE
|
||||
total = $redis.scard('vmpooler__ready__'+pool['name']) +
|
||||
$redis.scard('vmpooler__pending__'+pool['name'])
|
||||
|
||||
begin
|
||||
if (defined? $graphite)
|
||||
$graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
|
||||
$graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
|
||||
end
|
||||
rescue
|
||||
end
|
||||
|
||||
if (total < pool['size'])
|
||||
(1..(pool['size'] - total)).each { |i|
|
||||
|
||||
if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
|
||||
begin
|
||||
$redis.incr('vmpooler__tasks__clone')
|
||||
|
||||
clone_vm(
|
||||
pool['template'],
|
||||
pool['pool'],
|
||||
pool['folder'],
|
||||
pool['datastore']
|
||||
)
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
|
||||
$redis.decr('vmpooler__tasks__clone')
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
sleep(1)
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def execute!
|
||||
$logger.log('d', "starting vmpooler")
|
||||
|
||||
# Clear out the tasks manager, as we don't know about any tasks at this point
|
||||
$redis.set('vmpooler__tasks__clone', 0)
|
||||
|
||||
loop do
|
||||
$pools.each do |pool|
|
||||
if (! $threads[pool['name']])
|
||||
check_pool(pool)
|
||||
else
|
||||
if (! $threads[pool['name']].alive?)
|
||||
$logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
|
||||
check_pool(pool)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
sleep(1)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 10 KiB |
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 3.2 KiB |
180
lib/vmpooler/vsphere_helper.rb
Normal file
180
lib/vmpooler/vsphere_helper.rb
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
module Vmpooler
|
||||
class VsphereHelper
|
||||
def initialize vInfo = {}
|
||||
begin
|
||||
require 'rbvmomi'
|
||||
rescue LoadError
|
||||
raise "Unable to load RbVmomi, please ensure its installed"
|
||||
end
|
||||
|
||||
config_file = File.expand_path('vmpooler.yaml')
|
||||
vsphere = YAML.load_file(config_file)[:vsphere]
|
||||
|
||||
@connection = RbVmomi::VIM.connect :host => vsphere['server'],
|
||||
:user => vsphere['username'],
|
||||
:password => vsphere['password'],
|
||||
:insecure => true
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_datastore datastorename
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
datacenter.find_datastore(datastorename)
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_folder foldername
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
base = datacenter.vmFolder
|
||||
folders = foldername.split('/')
|
||||
folders.each do |folder|
|
||||
case base
|
||||
when RbVmomi::VIM::Folder
|
||||
base = base.childEntity.find { |f| f.name == folder }
|
||||
else
|
||||
abort "Unexpected object type encountered (#{base.class}) while finding folder"
|
||||
end
|
||||
end
|
||||
|
||||
base
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_pool poolname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
base = datacenter.hostFolder
|
||||
pools = poolname.split('/')
|
||||
pools.each do |pool|
|
||||
case base
|
||||
when RbVmomi::VIM::Folder
|
||||
base = base.childEntity.find { |f| f.name == pool }
|
||||
when RbVmomi::VIM::ClusterComputeResource
|
||||
base = base.resourcePool.resourcePool.find { |f| f.name == pool }
|
||||
when RbVmomi::VIM::ResourcePool
|
||||
base = base.resourcePool.find { |f| f.name == pool }
|
||||
else
|
||||
abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
|
||||
end
|
||||
end
|
||||
|
||||
base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
|
||||
base
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_vm vmname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
@connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_vm_heavy vmname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
|
||||
containerView = get_base_vm_container_from @connection
|
||||
propertyCollector = @connection.propertyCollector
|
||||
|
||||
objectSet = [{
|
||||
:obj => containerView,
|
||||
:skip => true,
|
||||
:selectSet => [ RbVmomi::VIM::TraversalSpec.new({
|
||||
:name => 'gettingTheVMs',
|
||||
:path => 'view',
|
||||
:skip => false,
|
||||
:type => 'ContainerView'
|
||||
}) ]
|
||||
}]
|
||||
|
||||
propSet = [{
|
||||
:pathSet => [ 'name' ],
|
||||
:type => 'VirtualMachine'
|
||||
}]
|
||||
|
||||
results = propertyCollector.RetrievePropertiesEx({
|
||||
:specSet => [{
|
||||
:objectSet => objectSet,
|
||||
:propSet => propSet
|
||||
}],
|
||||
:options => { :maxObjects => nil }
|
||||
})
|
||||
|
||||
vms = {}
|
||||
results.objects.each do |result|
|
||||
name = result.propSet.first.val
|
||||
next unless vmname.include? name
|
||||
vms[name] = result.obj
|
||||
end
|
||||
|
||||
while results.token do
|
||||
results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
|
||||
results.objects.each do |result|
|
||||
name = result.propSet.first.val
|
||||
next unless vmname.include? name
|
||||
vms[name] = result.obj
|
||||
end
|
||||
end
|
||||
|
||||
vms
|
||||
end
|
||||
|
||||
|
||||
|
||||
def get_base_vm_container_from connection
|
||||
begin
|
||||
connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
viewManager = connection.serviceContent.viewManager
|
||||
viewManager.CreateContainerView({
|
||||
:container => connection.serviceContent.rootFolder,
|
||||
:recursive => true,
|
||||
:type => [ 'VirtualMachine' ]
|
||||
})
|
||||
end
|
||||
|
||||
|
||||
|
||||
def close
|
||||
@connection.close
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -1,180 +0,0 @@
|
|||
require 'yaml' unless defined?(YAML)
|
||||
require 'rubygems' unless defined?(Gem)
|
||||
|
||||
class VsphereHelper
|
||||
def initialize vInfo = {}
|
||||
begin
|
||||
require 'rbvmomi'
|
||||
rescue LoadError
|
||||
raise "Unable to load RbVmomi, please ensure its installed"
|
||||
end
|
||||
|
||||
Dir.chdir(File.dirname(__FILE__))
|
||||
|
||||
config_file = File.expand_path('../vmpooler.yaml')
|
||||
vsphere = YAML.load_file(config_file)[:vsphere]
|
||||
|
||||
@connection = RbVmomi::VIM.connect :host => vsphere['server'],
|
||||
:user => vsphere['username'],
|
||||
:password => vsphere['password'],
|
||||
:insecure => true
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_datastore datastorename
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
datacenter.find_datastore(datastorename)
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_folder foldername
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
base = datacenter.vmFolder
|
||||
folders = foldername.split('/')
|
||||
folders.each do |folder|
|
||||
case base
|
||||
when RbVmomi::VIM::Folder
|
||||
base = base.childEntity.find { |f| f.name == folder }
|
||||
else
|
||||
abort "Unexpected object type encountered (#{base.class}) while finding folder"
|
||||
end
|
||||
end
|
||||
|
||||
base
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_pool poolname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
datacenter = @connection.serviceInstance.find_datacenter
|
||||
base = datacenter.hostFolder
|
||||
pools = poolname.split('/')
|
||||
pools.each do |pool|
|
||||
case base
|
||||
when RbVmomi::VIM::Folder
|
||||
base = base.childEntity.find { |f| f.name == pool }
|
||||
when RbVmomi::VIM::ClusterComputeResource
|
||||
base = base.resourcePool.resourcePool.find { |f| f.name == pool }
|
||||
when RbVmomi::VIM::ResourcePool
|
||||
base = base.resourcePool.find { |f| f.name == pool }
|
||||
else
|
||||
abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
|
||||
end
|
||||
end
|
||||
|
||||
base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
|
||||
base
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_vm vmname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
@connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
|
||||
end
|
||||
|
||||
|
||||
|
||||
def find_vm_heavy vmname
|
||||
begin
|
||||
@connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
|
||||
containerView = get_base_vm_container_from @connection
|
||||
propertyCollector = @connection.propertyCollector
|
||||
|
||||
objectSet = [{
|
||||
:obj => containerView,
|
||||
:skip => true,
|
||||
:selectSet => [ RbVmomi::VIM::TraversalSpec.new({
|
||||
:name => 'gettingTheVMs',
|
||||
:path => 'view',
|
||||
:skip => false,
|
||||
:type => 'ContainerView'
|
||||
}) ]
|
||||
}]
|
||||
|
||||
propSet = [{
|
||||
:pathSet => [ 'name' ],
|
||||
:type => 'VirtualMachine'
|
||||
}]
|
||||
|
||||
results = propertyCollector.RetrievePropertiesEx({
|
||||
:specSet => [{
|
||||
:objectSet => objectSet,
|
||||
:propSet => propSet
|
||||
}],
|
||||
:options => { :maxObjects => nil }
|
||||
})
|
||||
|
||||
vms = {}
|
||||
results.objects.each do |result|
|
||||
name = result.propSet.first.val
|
||||
next unless vmname.include? name
|
||||
vms[name] = result.obj
|
||||
end
|
||||
|
||||
while results.token do
|
||||
results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
|
||||
results.objects.each do |result|
|
||||
name = result.propSet.first.val
|
||||
next unless vmname.include? name
|
||||
vms[name] = result.obj
|
||||
end
|
||||
end
|
||||
|
||||
vms
|
||||
end
|
||||
|
||||
|
||||
|
||||
def get_base_vm_container_from connection
|
||||
begin
|
||||
connection.serviceInstance.CurrentTime
|
||||
rescue
|
||||
initialize()
|
||||
end
|
||||
|
||||
viewManager = connection.serviceContent.viewManager
|
||||
viewManager.CreateContainerView({
|
||||
:container => connection.serviceContent.rootFolder,
|
||||
:recursive => true,
|
||||
:type => [ 'VirtualMachine' ]
|
||||
})
|
||||
end
|
||||
|
||||
|
||||
|
||||
def close
|
||||
@connection.close
|
||||
end
|
||||
end
|
||||
|
||||
459
vmpooler
459
vmpooler
|
|
@ -1,463 +1,14 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
require 'json'
|
||||
require 'rbvmomi'
|
||||
require 'redis'
|
||||
require 'time'
|
||||
require 'timeout'
|
||||
require 'yaml'
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
$:.unshift(File.dirname(__FILE__))
|
||||
require 'lib/logger'
|
||||
require 'lib/require_relative'
|
||||
require 'lib/vsphere_helper'
|
||||
|
||||
Dir.chdir(File.dirname(__FILE__))
|
||||
require 'rubygems' unless defined?(Gem)
|
||||
require 'lib/vmpooler'
|
||||
|
||||
# Load the configuration file
|
||||
config_file = File.expand_path('vmpooler.yaml')
|
||||
$config = YAML.load_file(config_file)
|
||||
|
||||
pools = $config[:pools]
|
||||
vsphere = $config[:vsphere]
|
||||
redis = $config[:redis]
|
||||
|
||||
# Load logger library
|
||||
$logger = Logger.new $config[:config]['logfile']
|
||||
|
||||
# Load Graphite helper library (if configured)
|
||||
if (defined? $config[:config]['graphite'])
|
||||
require 'lib/graphite'
|
||||
$graphite = Graphite.new $config[:config]['graphite']
|
||||
end
|
||||
|
||||
# Set some defaults
|
||||
$config[:config]['task_limit'] ||= 10
|
||||
$config[:config]['vm_checktime'] ||= 15
|
||||
$config[:config]['vm_lifetime'] ||= 24
|
||||
$config[:redis] ||= Hash.new
|
||||
$config[:redis]['server'] ||= 'localhost'
|
||||
|
||||
# Connect to Redis
|
||||
$redis = Redis.new(:host => $config[:redis]['server'])
|
||||
|
||||
# vSphere object
|
||||
$vsphere = {}
|
||||
|
||||
# Our thread-tracker object
|
||||
$threads = {}
|
||||
|
||||
|
||||
|
||||
# Check the state of a VM
|
||||
def check_pending_vm vm, pool, timeout
|
||||
Thread.new {
|
||||
host = $vsphere[pool].find_vm(vm)
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.summary) and
|
||||
(host.summary.guest) and
|
||||
(host.summary.guest.hostName) and
|
||||
(host.summary.guest.hostName == vm)
|
||||
)
|
||||
begin
|
||||
Socket.getaddrinfo(vm, nil)
|
||||
rescue
|
||||
end
|
||||
|
||||
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
|
||||
|
||||
$logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
|
||||
end
|
||||
else
|
||||
clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
|
||||
|
||||
if (
|
||||
(clone_stamp) and
|
||||
(((Time.now - Time.parse(clone_stamp))/60) > timeout)
|
||||
)
|
||||
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_ready_vm vm, pool, ttl
|
||||
Thread.new {
|
||||
if (ttl > 0)
|
||||
if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
|
||||
end
|
||||
end
|
||||
|
||||
check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
|
||||
|
||||
if (
|
||||
(! check_stamp) or
|
||||
(((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
|
||||
)
|
||||
$redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
|
||||
|
||||
host = $vsphere[pool].find_vm(vm) ||
|
||||
$vsphere[pool].find_vm_heavy(vm)[vm]
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState) and
|
||||
(host.runtime.powerState != 'poweredOn')
|
||||
)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
|
||||
end
|
||||
|
||||
if (
|
||||
(host.summary.guest) and
|
||||
(host.summary.guest.hostName) and
|
||||
(host.summary.guest.hostName != vm)
|
||||
)
|
||||
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
|
||||
end
|
||||
else
|
||||
$redis.srem('vmpooler__ready__'+pool, vm)
|
||||
|
||||
$logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
|
||||
end
|
||||
|
||||
begin
|
||||
Timeout::timeout(5) {
|
||||
TCPSocket.new vm, 22
|
||||
}
|
||||
rescue
|
||||
if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_running_vm vm, pool, ttl
|
||||
Thread.new {
|
||||
host = $vsphere[pool].find_vm(vm)
|
||||
|
||||
if (host)
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState != 'poweredOn')
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
|
||||
else
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.bootTime)
|
||||
((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
# Clone a VM
|
||||
def clone_vm template, pool, folder, datastore
|
||||
Thread.new {
|
||||
vm = {}
|
||||
|
||||
if template =~ /\//
|
||||
templatefolders = template.split('/')
|
||||
vm['template'] = templatefolders.pop
|
||||
end
|
||||
|
||||
if templatefolders
|
||||
vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
|
||||
else
|
||||
raise "Please provide a full path to the template"
|
||||
end
|
||||
|
||||
if vm['template'].length == 0
|
||||
raise "Unable to find template '#{vm['template']}'!"
|
||||
end
|
||||
|
||||
# Generate a randomized hostname
|
||||
o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
|
||||
vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
|
||||
|
||||
# Add VM to Redis inventory ('pending' pool)
|
||||
$redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
|
||||
$redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
|
||||
|
||||
# Annotate with creation time, origin template, etc.
|
||||
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
|
||||
:annotation => JSON.pretty_generate({
|
||||
name: vm['hostname'],
|
||||
created_by: $config[:vsphere]['username'],
|
||||
base_template: vm['template'],
|
||||
creation_timestamp: Time.now.utc
|
||||
})
|
||||
)
|
||||
|
||||
# Put the VM in the specified folder and resource pool
|
||||
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
|
||||
:datastore => $vsphere[vm['template']].find_datastore(datastore),
|
||||
:pool => $vsphere[vm['template']].find_pool(pool),
|
||||
:diskMoveType => :moveChildMostDiskBacking
|
||||
)
|
||||
|
||||
# Create a clone spec
|
||||
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
|
||||
:location => relocateSpec,
|
||||
:config => configSpec,
|
||||
:powerOn => true,
|
||||
:template => false
|
||||
)
|
||||
|
||||
# Clone the VM
|
||||
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
|
||||
|
||||
begin
|
||||
start = Time.now
|
||||
vm[vm['template']].CloneVM_Task(
|
||||
:folder => $vsphere[vm['template']].find_folder(folder),
|
||||
:name => vm['hostname'],
|
||||
:spec => spec
|
||||
).wait_for_completion
|
||||
finish = '%.2f' % (Time.now-start)
|
||||
|
||||
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
|
||||
$redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
|
||||
end
|
||||
|
||||
$redis.decr('vmpooler__tasks__clone')
|
||||
|
||||
begin
|
||||
$graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
|
||||
rescue
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
# Destroy a VM
|
||||
def destroy_vm vm, pool
|
||||
Thread.new {
|
||||
$redis.srem('vmpooler__completed__'+pool, vm)
|
||||
$redis.hdel('vmpooler__active__'+pool, vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
|
||||
host = $vsphere[pool].find_vm(vm) ||
|
||||
$vsphere[pool].find_vm_heavy(vm)[vm]
|
||||
|
||||
if (host)
|
||||
start = Time.now
|
||||
|
||||
if (
|
||||
(host.runtime) and
|
||||
(host.runtime.powerState) and
|
||||
(host.runtime.powerState == 'poweredOn')
|
||||
)
|
||||
$logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
|
||||
host.PowerOffVM_Task.wait_for_completion
|
||||
end
|
||||
|
||||
host.Destroy_Task.wait_for_completion
|
||||
finish = '%.2f' % (Time.now-start)
|
||||
|
||||
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
||||
|
||||
$graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
def check_pool pool
|
||||
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
|
||||
|
||||
$threads[pool['name']] = Thread.new {
|
||||
$vsphere[pool['name']] ||= VsphereHelper.new
|
||||
Thread.new { Vmpooler::API.new.execute! }
|
||||
Thread.new { Vmpooler::PoolManager.new.execute! }
|
||||
|
||||
loop do
|
||||
# INVENTORY
|
||||
inventory = {}
|
||||
begin
|
||||
base = $vsphere[pool['name']].find_pool(pool['pool'])
|
||||
|
||||
base.vm.each do |vm|
|
||||
if (
|
||||
(! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
|
||||
(! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
|
||||
)
|
||||
$redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
|
||||
|
||||
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
||||
end
|
||||
|
||||
inventory[vm['name']] = 1
|
||||
end
|
||||
rescue
|
||||
end
|
||||
|
||||
# RUNNING
|
||||
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
if (pool['running_ttl'])
|
||||
begin
|
||||
check_running_vm(vm, pool['name'], pool['running_ttl'])
|
||||
rescue
|
||||
end
|
||||
else
|
||||
begin
|
||||
check_running_vm(vm, pool['name'], '720')
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# READY
|
||||
$redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
begin
|
||||
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# PENDING
|
||||
$redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
|
||||
pool['timeout'] ||= 15
|
||||
|
||||
if (inventory[vm])
|
||||
begin
|
||||
check_pending_vm(vm, pool['name'], pool['timeout'])
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# COMPLETED
|
||||
$redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
|
||||
if (inventory[vm])
|
||||
begin
|
||||
destroy_vm(vm, pool['name'])
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
|
||||
$redis.srem('vmpooler__completed__'+pool['name'], vm)
|
||||
$redis.hdel('vmpooler__active__'+pool['name'], vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
end
|
||||
else
|
||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
||||
$redis.srem('vmpooler__completed__'+pool['name'], vm)
|
||||
$redis.hdel('vmpooler__active__'+pool['name'], vm)
|
||||
$redis.del('vmpooler__vm__'+vm)
|
||||
end
|
||||
end
|
||||
|
||||
# DISCOVERED
|
||||
$redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
|
||||
['pending', 'ready', 'running', 'completed'].each do |queue|
|
||||
if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
|
||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
||||
$redis.srem('vmpooler__discovered__'+pool['name'], vm)
|
||||
end
|
||||
end
|
||||
|
||||
if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
|
||||
$redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
|
||||
end
|
||||
end
|
||||
|
||||
# LONG-RUNNING
|
||||
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
|
||||
if ($redis.hget('vmpooler__active__'+pool['name'], vm))
|
||||
running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
|
||||
if (
|
||||
($config[:config]['vm_lifetime'] > 0) and
|
||||
(running > $config[:config]['vm_lifetime'])
|
||||
)
|
||||
$redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
|
||||
|
||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# REPOPULATE
|
||||
total = $redis.scard('vmpooler__ready__'+pool['name']) +
|
||||
$redis.scard('vmpooler__pending__'+pool['name'])
|
||||
|
||||
begin
|
||||
if (defined? $graphite)
|
||||
$graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
|
||||
$graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
|
||||
end
|
||||
rescue
|
||||
end
|
||||
|
||||
if (total < pool['size'])
|
||||
(1..(pool['size'] - total)).each { |i|
|
||||
|
||||
if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
|
||||
begin
|
||||
$redis.incr('vmpooler__tasks__clone')
|
||||
|
||||
clone_vm(
|
||||
pool['template'],
|
||||
pool['pool'],
|
||||
pool['folder'],
|
||||
pool['datastore']
|
||||
)
|
||||
rescue
|
||||
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
|
||||
$redis.decr('vmpooler__tasks__clone')
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
sleep(1)
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
|
||||
|
||||
$logger.log('d', "starting vmpooler")
|
||||
|
||||
# Clear out the tasks manager, as we don't know about any tasks at this point
|
||||
$redis.set('vmpooler__tasks__clone', 0)
|
||||
|
||||
loop do
|
||||
pools.each do |pool|
|
||||
if (! $threads[pool['name']])
|
||||
check_pool(pool)
|
||||
else
|
||||
if (! $threads[pool['name']].alive?)
|
||||
$logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
|
||||
check_pool(pool)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
sleep(1)
|
||||
end
|
||||
|
||||
|
|
|
|||
231
vmpooler-api
231
vmpooler-api
|
|
@ -1,231 +0,0 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
require 'rubygems'
|
||||
|
||||
require 'json'
|
||||
require 'open-uri'
|
||||
require 'redis'
|
||||
require 'sinatra'
|
||||
require 'yaml'
|
||||
|
||||
$:.unshift(File.dirname(__FILE__))
|
||||
require 'lib/logger'
|
||||
require 'lib/require_relative'
|
||||
|
||||
Dir.chdir(File.dirname(__FILE__))
|
||||
|
||||
# Load the configuration file
|
||||
config_file = File.expand_path('vmpooler.yaml')
|
||||
$config = YAML.load_file(config_file)
|
||||
|
||||
pools = $config[:pools]
|
||||
redis = $config[:redis]
|
||||
|
||||
# Load logger library
|
||||
$logger = Logger.new $config[:config]['logfile']
|
||||
|
||||
# Set some defaults
|
||||
$config[:redis] ||= Hash.new
|
||||
$config[:redis]['server'] ||= 'localhost'
|
||||
|
||||
# Connect to Redis
|
||||
$redis = Redis.new(:host => $config[:redis]['server'])
|
||||
|
||||
# Sinatra!
|
||||
get '/' do
|
||||
erb :dashboard, locals: {
|
||||
site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
|
||||
}
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/numbers/?' do
|
||||
result = Hash.new
|
||||
result['pending'] = 0
|
||||
result['cloning'] = 0
|
||||
result['booting'] = 0
|
||||
result['ready'] = 0
|
||||
result['running'] = 0
|
||||
result['completed'] = 0
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
|
||||
result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
|
||||
result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
|
||||
result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
|
||||
end
|
||||
|
||||
result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
|
||||
result['booting'] = result['pending'].to_i - result['cloning'].to_i
|
||||
result['booting'] = 0 if result['booting'] < 0
|
||||
result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/pool/?' do
|
||||
result = Hash.new
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
result[pool['name']] ||= Hash.new
|
||||
result[pool['name']]['size'] = pool['size']
|
||||
result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
|
||||
end
|
||||
|
||||
if ( params[:history] )
|
||||
if ( $config[:config]['graphite'] )
|
||||
history ||= Hash.new
|
||||
|
||||
begin
|
||||
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
|
||||
history = JSON.parse( buffer )
|
||||
|
||||
history.each do |pool|
|
||||
if pool['target'] =~ /.*\.(.*)$/
|
||||
pool['name'] = $1
|
||||
|
||||
if ( result[pool['name']] )
|
||||
pool['last'] = result[pool['name']]['size']
|
||||
result[pool['name']]['history'] ||= Array.new
|
||||
|
||||
pool['datapoints'].each do |metric|
|
||||
8.times do |n|
|
||||
if ( metric[0] )
|
||||
pool['last'] = metric[0].to_i
|
||||
result[pool['name']]['history'].push( metric[0].to_i )
|
||||
else
|
||||
result[pool['name']]['history'].push( pool['last'] )
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue
|
||||
end
|
||||
else
|
||||
$config[:pools].each do |pool|
|
||||
result[pool['name']] ||= Hash.new
|
||||
result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/dashboard/stats/vmpooler/running/?' do
|
||||
result = Hash.new
|
||||
|
||||
$config[:pools].each do |pool|
|
||||
running = $redis.scard( 'vmpooler__running__' + pool['name'] )
|
||||
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
|
||||
|
||||
result[pool['major']] ||= Hash.new
|
||||
|
||||
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
|
||||
end
|
||||
|
||||
if ( params[:history] )
|
||||
if ( $config[:config]['graphite'] )
|
||||
begin
|
||||
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
|
||||
JSON.parse( buffer ).each do |pool|
|
||||
if pool['target'] =~ /.*\.(.*)$/
|
||||
pool['name'] = $1
|
||||
|
||||
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
|
||||
|
||||
result[pool['major']]['history'] ||= Array.new
|
||||
|
||||
for i in 0..pool['datapoints'].length
|
||||
if (
|
||||
pool['datapoints'][i] and
|
||||
pool['datapoints'][i][0]
|
||||
)
|
||||
pool['last'] = pool['datapoints'][i][0]
|
||||
|
||||
result[pool['major']]['history'][i] ||= 0
|
||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
|
||||
else
|
||||
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
rescue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
content_type :json
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/vm/?' do
|
||||
content_type :json
|
||||
|
||||
result = []
|
||||
|
||||
pools.each do |pool|
|
||||
result.push(pool['name'])
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
get '/vm/:template/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
result[params[:template]] = {}
|
||||
result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
post '/vm/:template/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
result[params[:template]] = {}
|
||||
|
||||
if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
|
||||
vm = $redis.spop('vmpooler__ready__'+params[:template])
|
||||
|
||||
unless (vm.nil?)
|
||||
$redis.sadd('vmpooler__running__'+params[:template], vm)
|
||||
$redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
|
||||
|
||||
result[params[:template]]['ok'] = true
|
||||
result[params[:template]]['hostname'] = vm
|
||||
else
|
||||
result[params[:template]]['ok'] = false
|
||||
end
|
||||
else
|
||||
result[params[:template]]['ok'] = false
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
delete '/vm/:hostname/?' do
|
||||
content_type :json
|
||||
|
||||
result = {}
|
||||
|
||||
result['ok'] = false
|
||||
|
||||
pools.each do |pool|
|
||||
if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
|
||||
$redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
|
||||
$redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
|
||||
result['ok'] = true
|
||||
end
|
||||
end
|
||||
|
||||
JSON.pretty_generate(result)
|
||||
end
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue