puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 01:58:41 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #17 from sschneid/development

Browse source 
Merge 'development' branch
This commit is contained in:
Scott Schneider 2014-06-10 10:32:02 -07:00
commit 0d31a43339
23 changed files with 1110 additions and 939 deletions
Download patch file Download diff file Expand all files Collapse all files

50
README.md
View file

@ -84,21 +84,63 @@ $ curl --url vmpooler.company.com/vm
]
```
#### POST /vm
Useful for batch operations; post JSON (see format below), get back VMs.
```
$ curl -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url vmpooler.company.com/vm
```
```json
{
"ok": true,
"debian-7-i386": {
"hostname": [
"o41xtodlvnvu5cw",
"khirruvwfjlmx3y"
]
},
"debian-7-x86_64": {
"hostname": "y91qbrpbfj6d13q"
}
}
```
#### POST /vm/<pool>
Check-out a VM.
Check-out a VM or VMs.
```
$ curl -d --url vmpooler.company.com/vm/debian-7-i386
```
```json
{
"ok": true,
"debian-7-i386": {
"ok": true,
"hostname": "fq6qlpjlsskycq6"
}
}
```
Multiple VMs can be requested by using multiple query parameters in the URL:
```
$ curl -d --url vmpooler.company.com/vm/debian-7-i386+debian-7-i386+debian-7-x86_64
```
```json
{
"ok": true,
"debian-7-i386": {
"hostname": [
"sc0o4xqtodlul5w",
"4m4dkhqiufnjmxy"
]
},
"debian-7-x86_64": {
"hostname": "zb91y9qbrbf6d3q"
}
}
```
#### DELETE /vm/<hostnamename>
Schedule a checked-out VM for deletion.
@ -121,9 +163,9 @@ A dashboard is provided to offer real-time statistics and historical graphs. It
[Graphite](http://graphite.wikidot.com/) is required for historical data retrieval. See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for details.
## Author
## Authors and Contributors
Scott Schneider (sschneid@gmail.com)
- Scott Schneider (sschneid@gmail.com)
## License

18
lib/graphite.rb
View file

@ -1,18 +0,0 @@
require 'rubygems' unless defined?(Gem)
class Graphite
def initialize(
s = 'graphite'
)
@server = s
end
def log path, value
Thread.new {
socket = TCPSocket.new(@server, 2003)
socket.puts "#{path} #{value} #{Time.now.to_i}"
socket.close
}
end
end

20
lib/logger.rb
View file

@ -1,20 +0,0 @@
require 'rubygems' unless defined?(Gem)
class Logger
def initialize(
f = '/var/log/vmpooler.log'
)
@file = f
end
def log level, string
time = Time.new
stamp = time.strftime('%Y-%m-%d %H:%M:%S')
puts "[#{stamp}] #{string}"
open(@file, 'a') do |f|
f.puts "[#{stamp}] #{string}"
end
end
end

21
lib/require_relative.rb
View file

@ -1,21 +0,0 @@
# require_relative was introduced in 1.9.2. This makes it
# available to younger rubies. It trys hard to not re-require
# files.
unless Kernel.respond_to?(:require_relative)
module Kernel
def require_relative(path)
desired_path = File.expand_path('../'+path.to_str, caller[0])
shortest = desired_path
$:.each do |path|
path += '/'
if desired_path.index(path) == 0
candidate = desired_path.sub(path, '')
shortest = candidate if candidate.size < shortest.size
end
end
require shortest
end
end
end

21
lib/vmpooler.rb Normal file
View file

@ -0,0 +1,21 @@
require 'rubygems' unless defined?(Gem)
module Vmpooler
require 'json'
require 'open-uri'
require 'rbvmomi'
require 'redis'
require 'sinatra/base'
require 'time'
require 'timeout'
require 'yaml'
%w( api graphite logger pool_manager vsphere_helper ).each do |lib|
begin
require "vmpooler/#{lib}"
rescue LoadError
require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
end
end
end

304
lib/vmpooler/api.rb Normal file
View file

@ -0,0 +1,304 @@
module Vmpooler
class API
def initialize
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
# Set some defaults
$config[:redis] ||= Hash.new
$config[:redis]['server'] ||= 'localhost'
if ($config[:graphite]['server'])
$config[:graphite]['prefix'] ||= 'vmpooler'
end
# Connect to Redis
$redis = Redis.new(:host => $config[:redis]['server'])
end
def execute!
my_app = Sinatra.new {
set :environment, 'production'
get '/' do
erb :dashboard, locals: {
site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
}
end
get '/dashboard/stats/vmpooler/numbers/?' do
result = Hash.new
result['pending'] = 0
result['cloning'] = 0
result['booting'] = 0
result['ready'] = 0
result['running'] = 0
result['completed'] = 0
$config[:pools].each do |pool|
result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
end
result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
result['booting'] = result['pending'].to_i - result['cloning'].to_i
result['booting'] = 0 if result['booting'] < 0
result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
content_type :json
JSON.pretty_generate(result)
end
get '/dashboard/stats/vmpooler/pool/?' do
result = Hash.new
$config[:pools].each do |pool|
result[pool['name']] ||= Hash.new
result[pool['name']]['size'] = pool['size']
result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
end
if ( params[:history] )
if ( $config[:graphite]['server'] )
history ||= Hash.new
begin
buffer = open(
'http://'+$config[:graphite]['server']+'/render?target='+$config[:graphite]['prefix']+'.ready.*&from=-1hour&format=json'
).read
history = JSON.parse( buffer )
history.each do |pool|
if pool['target'] =~ /.*\.(.*)$/
pool['name'] = $1
if ( result[pool['name']] )
pool['last'] = result[pool['name']]['size']
result[pool['name']]['history'] ||= Array.new
pool['datapoints'].each do |metric|
8.times do |n|
if ( metric[0] )
pool['last'] = metric[0].to_i
result[pool['name']]['history'].push( metric[0].to_i )
else
result[pool['name']]['history'].push( pool['last'] )
end
end
end
end
end
end
rescue
end
else
$config[:pools].each do |pool|
result[pool['name']] ||= Hash.new
result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
end
end
end
content_type :json
JSON.pretty_generate(result)
end
get '/dashboard/stats/vmpooler/running/?' do
result = Hash.new
$config[:pools].each do |pool|
running = $redis.scard( 'vmpooler__running__' + pool['name'] )
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
result[pool['major']] ||= Hash.new
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
end
if ( params[:history] )
if ( $config[:graphite]['server'] )
begin
buffer = open(
'http://'+$config[:graphite]['server']+'/render?target='+$config[:graphite]['prefix']+'.running.*&from=-1hour&format=json'
).read
JSON.parse( buffer ).each do |pool|
if pool['target'] =~ /.*\.(.*)$/
pool['name'] = $1
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
result[pool['major']]['history'] ||= Array.new
for i in 0..pool['datapoints'].length
if (
pool['datapoints'][i] and
pool['datapoints'][i][0]
)
pool['last'] = pool['datapoints'][i][0]
result[pool['major']]['history'][i] ||= 0
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
else
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
end
end
end
end
rescue
end
end
end
content_type :json
JSON.pretty_generate(result)
end
get '/vm/?' do
content_type :json
result = []
$config[:pools].each do |pool|
result.push(pool['name'])
end
JSON.pretty_generate(result)
end
post '/vm/?' do
content_type :json
result = {}
available = 1
jdata = JSON.parse(request.body.read)
jdata.each do |template, count|
if ( $redis.scard('vmpooler__ready__'+template) < count.to_i )
available = 0
end
end
if ( available == 1 )
result['ok'] = true
jdata.each do |template, count|
result[template] ||= {}
count.to_i.times do |i|
vm = $redis.spop('vmpooler__ready__'+template)
unless (vm.nil?)
$redis.sadd('vmpooler__running__'+template, vm)
$redis.hset('vmpooler__active__'+template, vm, Time.now)
result[template] ||= {}
if ( result[template]['hostname'] )
result[template]['hostname'] = [result[template]['hostname']] if ! result[template]['hostname'].is_a?(Array)
result[template]['hostname'].push(vm)
else
result[template]['hostname'] = vm
end
else
result['ok'] = false
end
end
end
else
result['ok'] = false
end
JSON.pretty_generate(result)
end
get '/vm/:template/?' do
content_type :json
result = {}
result[params[:template]] = {}
result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
JSON.pretty_generate(result)
end
post '/vm/:template/?' do
content_type :json
result = {}
request = {}
params[:template].split('+').each do |template|
request[template] ||= 0
request[template] = request[template] + 1
end
available = 1
request.keys.each do |template|
if ( $redis.scard('vmpooler__ready__'+template) < request[template] )
available = 0
end
end
if ( available == 1 )
result['ok'] = true
params[:template].split('+').each do |template|
result[template] ||= {}
vm = $redis.spop('vmpooler__ready__'+template)
unless (vm.nil?)
$redis.sadd('vmpooler__running__'+template, vm)
$redis.hset('vmpooler__active__'+template, vm, Time.now)
result[template] ||= {}
if ( result[template]['hostname'] )
result[template]['hostname'] = [result[template]['hostname']] if ! result[template]['hostname'].is_a?(Array)
result[template]['hostname'].push(vm)
else
result[template]['hostname'] = vm
end
else
result['ok'] = false
end
end
else
result['ok'] = false
end
JSON.pretty_generate(result)
end
delete '/vm/:hostname/?' do
content_type :json
result = {}
result['ok'] = false
$config[:pools].each do |pool|
if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
$redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
$redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
result['ok'] = true
end
end
JSON.pretty_generate(result)
end
}
my_app.run!
end
end
end

20
lib/vmpooler/graphite.rb Normal file
View file

@ -0,0 +1,20 @@
require 'rubygems' unless defined?(Gem)
module Vmpooler
class Graphite
def initialize(
s = 'graphite'
)
@server = s
end
def log path, value
Thread.new {
socket = TCPSocket.new(@server, 2003)
socket.puts "#{path} #{value} #{Time.now.to_i}"
socket.close
}
end
end
end

21
lib/vmpooler/logger.rb Normal file
View file

@ -0,0 +1,21 @@
require 'rubygems' unless defined?(Gem)
module Vmpooler
class Logger
def initialize(
f = '/var/log/vmpooler.log'
)
@file = f
end
def log level, string
time = Time.new
stamp = time.strftime('%Y-%m-%d %H:%M:%S')
open(@file, 'a') do |f|
f.puts "[#{stamp}] #{string}"
end
end
end
end

451
lib/vmpooler/pool_manager.rb Normal file
View file

@ -0,0 +1,451 @@
module Vmpooler
class PoolManager
def initialize
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
$pools = $config[:pools]
# Set some defaults
$config[:config]['task_limit'] ||= 10
$config[:config]['vm_checktime'] ||= 15
$config[:config]['vm_lifetime'] ||= 24
$config[:redis] ||= Hash.new
$config[:redis]['server'] ||= 'localhost'
# Load logger library
$logger = Vmpooler::Logger.new $config[:config]['logfile']
# Load Graphite helper library (if configured)
if (defined? $config[:graphite]['server'])
$config[:graphite]['prefix'] ||= 'vmpooler'
$graphite = Vmpooler::Graphite.new $config[:graphite]['server']
end
# Connect to Redis
$redis = Redis.new(:host => $config[:redis]['server'])
# vSphere object
$vsphere = {}
# Our thread-tracker object
$threads = {}
end
# Check the state of a VM
def check_pending_vm vm, pool, timeout
Thread.new {
host = $vsphere[pool].find_vm(vm)
if (host)
if (
(host.summary) and
(host.summary.guest) and
(host.summary.guest.hostName) and
(host.summary.guest.hostName == vm)
)
begin
Socket.getaddrinfo(vm, nil)
rescue
end
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
$logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
end
else
clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
if (
(clone_stamp) and
(((Time.now - Time.parse(clone_stamp))/60) > timeout)
)
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
end
end
}
end
def check_ready_vm vm, pool, ttl
Thread.new {
if (ttl > 0)
if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
end
end
check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
if (
(! check_stamp) or
(((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
)
$redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
host = $vsphere[pool].find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm]
if (host)
if (
(host.runtime) and
(host.runtime.powerState) and
(host.runtime.powerState != 'poweredOn')
)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
end
if (
(host.summary.guest) and
(host.summary.guest.hostName) and
(host.summary.guest.hostName != vm)
)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
end
else
$redis.srem('vmpooler__ready__'+pool, vm)
$logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
end
begin
Timeout::timeout(5) {
TCPSocket.new vm, 22
}
rescue
if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
end
end
end
}
end
def check_running_vm vm, pool, ttl
Thread.new {
host = $vsphere[pool].find_vm(vm)
if (host)
if (
(host.runtime) and
(host.runtime.powerState != 'poweredOn')
)
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
else
if (
(host.runtime) and
(host.runtime.bootTime)
((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
)
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
end
end
end
}
end
# Clone a VM
def clone_vm template, pool, folder, datastore
Thread.new {
vm = {}
if template =~ /\//
templatefolders = template.split('/')
vm['template'] = templatefolders.pop
end
if templatefolders
vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
else
raise "Please provide a full path to the template"
end
if vm['template'].length == 0
raise "Unable to find template '#{vm['template']}'!"
end
# Generate a randomized hostname
o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
# Add VM to Redis inventory ('pending' pool)
$redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
$redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
# Annotate with creation time, origin template, etc.
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
:annotation => JSON.pretty_generate({
name: vm['hostname'],
created_by: $config[:vsphere]['username'],
base_template: vm['template'],
creation_timestamp: Time.now.utc
})
)
# Put the VM in the specified folder and resource pool
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
:datastore => $vsphere[vm['template']].find_datastore(datastore),
:pool => $vsphere[vm['template']].find_pool(pool),
:diskMoveType => :moveChildMostDiskBacking
)
# Create a clone spec
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
:location => relocateSpec,
:config => configSpec,
:powerOn => true,
:template => false
)
# Clone the VM
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
begin
start = Time.now
vm[vm['template']].CloneVM_Task(
:folder => $vsphere[vm['template']].find_folder(folder),
:name => vm['hostname'],
:spec => spec
).wait_for_completion
finish = '%.2f' % (Time.now-start)
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
rescue
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
$redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
end
$redis.decr('vmpooler__tasks__clone')
begin
$graphite.log($config[:graphite]['prefix']+".clone.#{vm['template']}", finish) if defined? $graphite
rescue
end
}
end
# Destroy a VM
def destroy_vm vm, pool
Thread.new {
$redis.srem('vmpooler__completed__'+pool, vm)
$redis.hdel('vmpooler__active__'+pool, vm)
$redis.del('vmpooler__vm__'+vm)
host = $vsphere[pool].find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm]
if (host)
start = Time.now
if (
(host.runtime) and
(host.runtime.powerState) and
(host.runtime.powerState == 'poweredOn')
)
$logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
host.PowerOffVM_Task.wait_for_completion
end
host.Destroy_Task.wait_for_completion
finish = '%.2f' % (Time.now-start)
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
$graphite.log($config[:graphite]['prefix']+".destroy.#{pool}", finish) if defined? $graphite
end
}
end
def check_pool pool
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
$threads[pool['name']] = Thread.new {
$vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
loop do
# INVENTORY
inventory = {}
begin
base = $vsphere[pool['name']].find_pool(pool['pool'])
base.vm.each do |vm|
if (
(! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
)
$redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
end
inventory[vm['name']] = 1
end
rescue
end
# RUNNING
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
if (inventory[vm])
if (pool['running_ttl'])
begin
check_running_vm(vm, pool['name'], pool['running_ttl'])
rescue
end
else
begin
check_running_vm(vm, pool['name'], '720')
rescue
end
end
end
end
# READY
$redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
if (inventory[vm])
begin
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
rescue
end
end
end
# PENDING
$redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
pool['timeout'] ||= 15
if (inventory[vm])
begin
check_pending_vm(vm, pool['name'], pool['timeout'])
rescue
end
end
end
# COMPLETED
$redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
if (inventory[vm])
begin
destroy_vm(vm, pool['name'])
rescue
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
$redis.srem('vmpooler__completed__'+pool['name'], vm)
$redis.hdel('vmpooler__active__'+pool['name'], vm)
$redis.del('vmpooler__vm__'+vm)
end
else
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
$redis.srem('vmpooler__completed__'+pool['name'], vm)
$redis.hdel('vmpooler__active__'+pool['name'], vm)
$redis.del('vmpooler__vm__'+vm)
end
end
# DISCOVERED
$redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
['pending', 'ready', 'running', 'completed'].each do |queue|
if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
$redis.srem('vmpooler__discovered__'+pool['name'], vm)
end
end
if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
$redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
end
end
# LONG-RUNNING
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
if ($redis.hget('vmpooler__active__'+pool['name'], vm))
running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
if (
($config[:config]['vm_lifetime'] > 0) and
(running > $config[:config]['vm_lifetime'])
)
$redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
end
end
end
# REPOPULATE
total = $redis.scard('vmpooler__ready__'+pool['name']) +
$redis.scard('vmpooler__pending__'+pool['name'])
begin
if (defined? $graphite)
$graphite.log($config[:graphite]['prefix']+'.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
$graphite.log($config[:graphite]['prefix']+'.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
end
rescue
end
if (total < pool['size'])
(1..(pool['size'] - total)).each { |i|
if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
begin
$redis.incr('vmpooler__tasks__clone')
clone_vm(
pool['template'],
pool['pool'],
pool['folder'],
pool['datastore']
)
rescue
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
$redis.decr('vmpooler__tasks__clone')
end
end
}
end
sleep(1)
end
}
end
def execute!
$logger.log('d', "starting vmpooler")
# Clear out the tasks manager, as we don't know about any tasks at this point
$redis.set('vmpooler__tasks__clone', 0)
loop do
$pools.each do |pool|
if (! $threads[pool['name']])
check_pool(pool)
else
if (! $threads[pool['name']].alive?)
$logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
check_pool(pool)
end
end
end
sleep(1)
end
end
end
end

2
public/dashboard.css → lib/vmpooler/public/dashboard.css
View file

@ -140,5 +140,5 @@ body,
.tick {
fill: none;
stroke: #eee;
stroke: #ddd;
}

BIN
lib/vmpooler/public/img/bg.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 27 KiB

0
public/img/logo.jpg → lib/vmpooler/public/img/logo.jpg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Before After
Before After

0
public/img/spinner.svg → lib/vmpooler/public/img/spinner.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 3.2 KiB

Before After
Before After

0
public/lib/stats-vmpooler-numbers.js → lib/vmpooler/public/lib/stats-vmpooler-numbers.js
View file

23
public/lib/stats-vmpooler-pool.js → lib/vmpooler/public/lib/stats-vmpooler-pool.js
View file

@ -74,6 +74,27 @@ d3.json( pool_url+'?history=1',
.attr( 'width', pool_width )
.attr( 'height', pool_height );
defs = stats_vmpooler_pool__svg[ pool ].append( 'svg:defs' );
defs.append( 'svg:pattern' )
.attr( 'id', 'background' )
.attr( 'patternUnits', 'userSpaceOnUse' )
.attr( 'width', '600px' )
.attr( 'height', '100px' )
.append( 'svg:image' )
.attr( 'xlink:href', '/img/bg.png' )
.attr( 'x', 0 )
.attr( 'y', 0 )
.attr( 'width', '600px' )
.attr( 'height', '100px' );
stats_vmpooler_pool__svg[ pool ]
.append( 'path' )
.attr( 'class', 'area' )
.attr( 'fill', 'url( #background )' )
.attr( 'opacity', '0.50' )
.attr( 'd', area( stats_vmpooler_pool__data[ pool ][ 'r' ] ) );
stats_vmpooler_pool__svg[ pool ]
.append( 'g' )
.attr( 'class', 'x tick' )
@ -140,7 +161,7 @@ d3.json( pool_url+'?history=1',
.append( 'path' )
.attr( 'class', 'area' )
.attr( 'fill', statuscolor )
.attr( 'opacity', '0.25' )
.attr( 'opacity', '0.35' )
.attr( 'd', area( stats_vmpooler_pool__data[ pool ][ 'r' ] ) );
stats_vmpooler_pool__svg[ pool ]

27
public/lib/stats-vmpooler-running.js → lib/vmpooler/public/lib/stats-vmpooler-running.js
View file

@ -118,9 +118,9 @@ d3.json( running_url+'?history=1',
var svg = d3.select( '#stats-vmpooler-running' )
.append( 'svg' )
.attr( 'height', running_height )
.attr( 'width', '100%' )
.style( 'margin-top', '15px' )
.style( 'margin-bottom', '10px' )
.attr( 'width', ( document.getElementById( 'stats-vmpooler-running' ).offsetWidth - 35 ) )
.style( 'margin', '15px 0px 0px 0px' )
.style( 'padding', '0px 10px 10px 10px' )
.append( 'g' );
var mysvg = svg.selectAll( '#stats-vmpooler-running' )
@ -128,6 +128,27 @@ d3.json( running_url+'?history=1',
.enter()
.append( 'g' );
defs = mysvg.append( 'svg:defs' );
defs.append( 'svg:pattern' )
.attr( 'id', 'background' )
.attr( 'patternUnits', 'userSpaceOnUse' )
.attr( 'width', '600px' )
.attr( 'height', '100px' )
.append( 'svg:image' )
.attr( 'xlink:href', '/img/bg.png' )
.attr( 'x', 0 )
.attr( 'y', 0 )
.attr( 'width', '600px' )
.attr( 'height', '100px' );
mysvg
.append( 'path' )
.attr( 'class', 'area' )
.attr( 'fill', 'url( #background )' )
.attr( 'opacity', '0.50' )
.attr( 'd', function( d ) { return area( d.values ); } );
mysvg.append( 'path' )
.attr( 'd', function( d ) { return area( d.values ); } )
.attr( 'clas', 'area' )

0
views/dashboard.erb → lib/vmpooler/views/dashboard.erb
View file

0
views/layout.erb → lib/vmpooler/views/layout.erb
View file

174
lib/vmpooler/vsphere_helper.rb Normal file
View file

@ -0,0 +1,174 @@
require 'rubygems' unless defined?(Gem)
module Vmpooler
class VsphereHelper
def initialize vInfo = {}
config_file = File.expand_path('vmpooler.yaml')
vsphere = YAML.load_file(config_file)[:vsphere]
@connection = RbVmomi::VIM.connect :host => vsphere['server'],
:user => vsphere['username'],
:password => vsphere['password'],
:insecure => true
end
def find_datastore datastorename
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
datacenter.find_datastore(datastorename)
end
def find_folder foldername
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.vmFolder
folders = foldername.split('/')
folders.each do |folder|
case base
when RbVmomi::VIM::Folder
base = base.childEntity.find { |f| f.name == folder }
else
abort "Unexpected object type encountered (#{base.class}) while finding folder"
end
end
base
end
def find_pool poolname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.hostFolder
pools = poolname.split('/')
pools.each do |pool|
case base
when RbVmomi::VIM::Folder
base = base.childEntity.find { |f| f.name == pool }
when RbVmomi::VIM::ClusterComputeResource
base = base.resourcePool.resourcePool.find { |f| f.name == pool }
when RbVmomi::VIM::ResourcePool
base = base.resourcePool.find { |f| f.name == pool }
else
abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
end
end
base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
base
end
def find_vm vmname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
@connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
end
def find_vm_heavy vmname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
containerView = get_base_vm_container_from @connection
propertyCollector = @connection.propertyCollector
objectSet = [{
:obj => containerView,
:skip => true,
:selectSet => [ RbVmomi::VIM::TraversalSpec.new({
:name => 'gettingTheVMs',
:path => 'view',
:skip => false,
:type => 'ContainerView'
}) ]
}]
propSet = [{
:pathSet => [ 'name' ],
:type => 'VirtualMachine'
}]
results = propertyCollector.RetrievePropertiesEx({
:specSet => [{
:objectSet => objectSet,
:propSet => propSet
}],
:options => { :maxObjects => nil }
})
vms = {}
results.objects.each do |result|
name = result.propSet.first.val
next unless vmname.include? name
vms[name] = result.obj
end
while results.token do
results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
results.objects.each do |result|
name = result.propSet.first.val
next unless vmname.include? name
vms[name] = result.obj
end
end
vms
end
def get_base_vm_container_from connection
begin
connection.serviceInstance.CurrentTime
rescue
initialize()
end
viewManager = connection.serviceContent.viewManager
viewManager.CreateContainerView({
:container => connection.serviceContent.rootFolder,
:recursive => true,
:type => [ 'VirtualMachine' ]
})
end
def close
@connection.close
end
end
end

180
lib/vsphere_helper.rb
View file

@ -1,180 +0,0 @@
require 'yaml' unless defined?(YAML)
require 'rubygems' unless defined?(Gem)
class VsphereHelper
def initialize vInfo = {}
begin
require 'rbvmomi'
rescue LoadError
raise "Unable to load RbVmomi, please ensure its installed"
end
Dir.chdir(File.dirname(__FILE__))
config_file = File.expand_path('../vmpooler.yaml')
vsphere = YAML.load_file(config_file)[:vsphere]
@connection = RbVmomi::VIM.connect :host => vsphere['server'],
:user => vsphere['username'],
:password => vsphere['password'],
:insecure => true
end
def find_datastore datastorename
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
datacenter.find_datastore(datastorename)
end
def find_folder foldername
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.vmFolder
folders = foldername.split('/')
folders.each do |folder|
case base
when RbVmomi::VIM::Folder
base = base.childEntity.find { |f| f.name == folder }
else
abort "Unexpected object type encountered (#{base.class}) while finding folder"
end
end
base
end
def find_pool poolname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.hostFolder
pools = poolname.split('/')
pools.each do |pool|
case base
when RbVmomi::VIM::Folder
base = base.childEntity.find { |f| f.name == pool }
when RbVmomi::VIM::ClusterComputeResource
base = base.resourcePool.resourcePool.find { |f| f.name == pool }
when RbVmomi::VIM::ResourcePool
base = base.resourcePool.find { |f| f.name == pool }
else
abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
end
end
base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
base
end
def find_vm vmname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
@connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
end
def find_vm_heavy vmname
begin
@connection.serviceInstance.CurrentTime
rescue
initialize()
end
vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
containerView = get_base_vm_container_from @connection
propertyCollector = @connection.propertyCollector
objectSet = [{
:obj => containerView,
:skip => true,
:selectSet => [ RbVmomi::VIM::TraversalSpec.new({
:name => 'gettingTheVMs',
:path => 'view',
:skip => false,
:type => 'ContainerView'
}) ]
}]
propSet = [{
:pathSet => [ 'name' ],
:type => 'VirtualMachine'
}]
results = propertyCollector.RetrievePropertiesEx({
:specSet => [{
:objectSet => objectSet,
:propSet => propSet
}],
:options => { :maxObjects => nil }
})
vms = {}
results.objects.each do |result|
name = result.propSet.first.val
next unless vmname.include? name
vms[name] = result.obj
end
while results.token do
results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
results.objects.each do |result|
name = result.propSet.first.val
next unless vmname.include? name
vms[name] = result.obj
end
end
vms
end
def get_base_vm_container_from connection
begin
connection.serviceInstance.CurrentTime
rescue
initialize()
end
viewManager = connection.serviceContent.viewManager
viewManager.CreateContainerView({
:container => connection.serviceContent.rootFolder,
:recursive => true,
:type => [ 'VirtualMachine' ]
})
end
def close
@connection.close
end
end

461
vmpooler
View file

@ -1,463 +1,14 @@
#!/usr/bin/ruby
require 'json'
require 'rbvmomi'
require 'redis'
require 'time'
require 'timeout'
require 'yaml'
#!/usr/bin/env ruby
$:.unshift(File.dirname(__FILE__))
require 'lib/logger'
require 'lib/require_relative'
require 'lib/vsphere_helper'
Dir.chdir(File.dirname(__FILE__))
require 'rubygems' unless defined?(Gem)
require 'lib/vmpooler'
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
pools = $config[:pools]
vsphere = $config[:vsphere]
redis = $config[:redis]
# Load logger library
$logger = Logger.new $config[:config]['logfile']
# Load Graphite helper library (if configured)
if (defined? $config[:config]['graphite'])
require 'lib/graphite'
$graphite = Graphite.new $config[:config]['graphite']
end
# Set some defaults
$config[:config]['task_limit'] ||= 10
$config[:config]['vm_checktime'] ||= 15
$config[:config]['vm_lifetime'] ||= 24
$config[:redis] ||= Hash.new
$config[:redis]['server'] ||= 'localhost'
# Connect to Redis
$redis = Redis.new(:host => $config[:redis]['server'])
# vSphere object
$vsphere = {}
# Our thread-tracker object
$threads = {}
# Check the state of a VM
def check_pending_vm vm, pool, timeout
Thread.new {
host = $vsphere[pool].find_vm(vm)
if (host)
if (
(host.summary) and
(host.summary.guest) and
(host.summary.guest.hostName) and
(host.summary.guest.hostName == vm)
)
begin
Socket.getaddrinfo(vm, nil)
rescue
end
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
$logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
end
else
clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
if (
(clone_stamp) and
(((Time.now - Time.parse(clone_stamp))/60) > timeout)
)
$redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
end
end
}
end
def check_ready_vm vm, pool, ttl
Thread.new {
if (ttl > 0)
if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
end
end
check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
if (
(! check_stamp) or
(((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
)
$redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
host = $vsphere[pool].find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm]
if (host)
if (
(host.runtime) and
(host.runtime.powerState) and
(host.runtime.powerState != 'poweredOn')
)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
end
if (
(host.summary.guest) and
(host.summary.guest.hostName) and
(host.summary.guest.hostName != vm)
)
$redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
end
else
$redis.srem('vmpooler__ready__'+pool, vm)
$logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
end
begin
Timeout::timeout(5) {
TCPSocket.new vm, 22
}
rescue
if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
end
end
end
}
end
def check_running_vm vm, pool, ttl
Thread.new {
host = $vsphere[pool].find_vm(vm)
if (host)
if (
(host.runtime) and
(host.runtime.powerState != 'poweredOn')
)
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
else
if (
(host.runtime) and
(host.runtime.bootTime)
((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
)
$redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
end
end
end
}
end
# Clone a VM
def clone_vm template, pool, folder, datastore
Thread.new {
vm = {}
if template =~ /\//
templatefolders = template.split('/')
vm['template'] = templatefolders.pop
end
if templatefolders
vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
else
raise "Please provide a full path to the template"
end
if vm['template'].length == 0
raise "Unable to find template '#{vm['template']}'!"
end
# Generate a randomized hostname
o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
# Add VM to Redis inventory ('pending' pool)
$redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
$redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
# Annotate with creation time, origin template, etc.
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
:annotation => JSON.pretty_generate({
name: vm['hostname'],
created_by: $config[:vsphere]['username'],
base_template: vm['template'],
creation_timestamp: Time.now.utc
})
)
# Put the VM in the specified folder and resource pool
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
:datastore => $vsphere[vm['template']].find_datastore(datastore),
:pool => $vsphere[vm['template']].find_pool(pool),
:diskMoveType => :moveChildMostDiskBacking
)
# Create a clone spec
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
:location => relocateSpec,
:config => configSpec,
:powerOn => true,
:template => false
)
# Clone the VM
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
begin
start = Time.now
vm[vm['template']].CloneVM_Task(
:folder => $vsphere[vm['template']].find_folder(folder),
:name => vm['hostname'],
:spec => spec
).wait_for_completion
finish = '%.2f' % (Time.now-start)
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
rescue
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
$redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
end
$redis.decr('vmpooler__tasks__clone')
begin
$graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
rescue
end
}
end
# Destroy a VM
def destroy_vm vm, pool
Thread.new {
$redis.srem('vmpooler__completed__'+pool, vm)
$redis.hdel('vmpooler__active__'+pool, vm)
$redis.del('vmpooler__vm__'+vm)
host = $vsphere[pool].find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm]
if (host)
start = Time.now
if (
(host.runtime) and
(host.runtime.powerState) and
(host.runtime.powerState == 'poweredOn')
)
$logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
host.PowerOffVM_Task.wait_for_completion
end
host.Destroy_Task.wait_for_completion
finish = '%.2f' % (Time.now-start)
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
$graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
end
}
end
def check_pool pool
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
$threads[pool['name']] = Thread.new {
$vsphere[pool['name']] ||= VsphereHelper.new
loop do
# INVENTORY
inventory = {}
begin
base = $vsphere[pool['name']].find_pool(pool['pool'])
base.vm.each do |vm|
if (
(! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
(! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
)
$redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
end
inventory[vm['name']] = 1
end
rescue
end
# RUNNING
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
if (inventory[vm])
if (pool['running_ttl'])
begin
check_running_vm(vm, pool['name'], pool['running_ttl'])
rescue
end
else
begin
check_running_vm(vm, pool['name'], '720')
rescue
end
end
end
end
# READY
$redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
if (inventory[vm])
begin
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
rescue
end
end
end
# PENDING
$redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
pool['timeout'] ||= 15
if (inventory[vm])
begin
check_pending_vm(vm, pool['name'], pool['timeout'])
rescue
end
end
end
# COMPLETED
$redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
if (inventory[vm])
begin
destroy_vm(vm, pool['name'])
rescue
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
$redis.srem('vmpooler__completed__'+pool['name'], vm)
$redis.hdel('vmpooler__active__'+pool['name'], vm)
$redis.del('vmpooler__vm__'+vm)
end
else
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
$redis.srem('vmpooler__completed__'+pool['name'], vm)
$redis.hdel('vmpooler__active__'+pool['name'], vm)
$redis.del('vmpooler__vm__'+vm)
end
end
# DISCOVERED
$redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
['pending', 'ready', 'running', 'completed'].each do |queue|
if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
$redis.srem('vmpooler__discovered__'+pool['name'], vm)
end
end
if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
$redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
end
end
# LONG-RUNNING
$redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
if ($redis.hget('vmpooler__active__'+pool['name'], vm))
running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
if (
($config[:config]['vm_lifetime'] > 0) and
(running > $config[:config]['vm_lifetime'])
)
$redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
end
end
end
# REPOPULATE
total = $redis.scard('vmpooler__ready__'+pool['name']) +
$redis.scard('vmpooler__pending__'+pool['name'])
begin
if (defined? $graphite)
$graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
$graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
end
rescue
end
if (total < pool['size'])
(1..(pool['size'] - total)).each { |i|
if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
begin
$redis.incr('vmpooler__tasks__clone')
clone_vm(
pool['template'],
pool['pool'],
pool['folder'],
pool['datastore']
)
rescue
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
$redis.decr('vmpooler__tasks__clone')
end
end
}
end
sleep(1)
end
}
end
$logger.log('d', "starting vmpooler")
# Clear out the tasks manager, as we don't know about any tasks at this point
$redis.set('vmpooler__tasks__clone', 0)
Thread.new { Vmpooler::API.new.execute! }
Thread.new { Vmpooler::PoolManager.new.execute! }
loop do
pools.each do |pool|
if (! $threads[pool['name']])
check_pool(pool)
else
if (! $threads[pool['name']].alive?)
$logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
check_pool(pool)
end
end
end
sleep(1)
sleep(10)
end

231
vmpooler-api
View file

@ -1,231 +0,0 @@
#!/usr/bin/ruby
require 'rubygems'
require 'json'
require 'open-uri'
require 'redis'
require 'sinatra'
require 'yaml'
$:.unshift(File.dirname(__FILE__))
require 'lib/logger'
require 'lib/require_relative'
Dir.chdir(File.dirname(__FILE__))
# Load the configuration file
config_file = File.expand_path('vmpooler.yaml')
$config = YAML.load_file(config_file)
pools = $config[:pools]
redis = $config[:redis]
# Load logger library
$logger = Logger.new $config[:config]['logfile']
# Set some defaults
$config[:redis] ||= Hash.new
$config[:redis]['server'] ||= 'localhost'
# Connect to Redis
$redis = Redis.new(:host => $config[:redis]['server'])
# Sinatra!
get '/' do
erb :dashboard, locals: {
site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
}
end
get '/dashboard/stats/vmpooler/numbers/?' do
result = Hash.new
result['pending'] = 0
result['cloning'] = 0
result['booting'] = 0
result['ready'] = 0
result['running'] = 0
result['completed'] = 0
$config[:pools].each do |pool|
result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
end
result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
result['booting'] = result['pending'].to_i - result['cloning'].to_i
result['booting'] = 0 if result['booting'] < 0
result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
content_type :json
JSON.pretty_generate(result)
end
get '/dashboard/stats/vmpooler/pool/?' do
result = Hash.new
$config[:pools].each do |pool|
result[pool['name']] ||= Hash.new
result[pool['name']]['size'] = pool['size']
result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
end
if ( params[:history] )
if ( $config[:config]['graphite'] )
history ||= Hash.new
begin
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
history = JSON.parse( buffer )
history.each do |pool|
if pool['target'] =~ /.*\.(.*)$/
pool['name'] = $1
if ( result[pool['name']] )
pool['last'] = result[pool['name']]['size']
result[pool['name']]['history'] ||= Array.new
pool['datapoints'].each do |metric|
8.times do |n|
if ( metric[0] )
pool['last'] = metric[0].to_i
result[pool['name']]['history'].push( metric[0].to_i )
else
result[pool['name']]['history'].push( pool['last'] )
end
end
end
end
end
end
rescue
end
else
$config[:pools].each do |pool|
result[pool['name']] ||= Hash.new
result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
end
end
end
content_type :json
JSON.pretty_generate(result)
end
get '/dashboard/stats/vmpooler/running/?' do
result = Hash.new
$config[:pools].each do |pool|
running = $redis.scard( 'vmpooler__running__' + pool['name'] )
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
result[pool['major']] ||= Hash.new
result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
end
if ( params[:history] )
if ( $config[:config]['graphite'] )
begin
buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
JSON.parse( buffer ).each do |pool|
if pool['target'] =~ /.*\.(.*)$/
pool['name'] = $1
pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
result[pool['major']]['history'] ||= Array.new
for i in 0..pool['datapoints'].length
if (
pool['datapoints'][i] and
pool['datapoints'][i][0]
)
pool['last'] = pool['datapoints'][i][0]
result[pool['major']]['history'][i] ||= 0
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
else
result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
end
end
end
end
rescue
end
end
end
content_type :json
JSON.pretty_generate(result)
end
get '/vm/?' do
content_type :json
result = []
pools.each do |pool|
result.push(pool['name'])
end
JSON.pretty_generate(result)
end
get '/vm/:template/?' do
content_type :json
result = {}
result[params[:template]] = {}
result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
JSON.pretty_generate(result)
end
post '/vm/:template/?' do
content_type :json
result = {}
result[params[:template]] = {}
if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
vm = $redis.spop('vmpooler__ready__'+params[:template])
unless (vm.nil?)
$redis.sadd('vmpooler__running__'+params[:template], vm)
$redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
result[params[:template]]['ok'] = true
result[params[:template]]['hostname'] = vm
else
result[params[:template]]['ok'] = false
end
else
result[params[:template]]['ok'] = false
end
JSON.pretty_generate(result)
end
delete '/vm/:hostname/?' do
content_type :json
result = {}
result['ok'] = false
pools.each do |pool|
if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
$redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
$redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
result['ok'] = true
end
end
JSON.pretty_generate(result)
end

25
vmpooler.yaml.example
View file

@ -49,6 +49,26 @@
:redis:
server: 'redis.company.com'
# :graphite:
#
# This section contains the connection information required to store
# historical data in an external Graphite database.
#
# Available configuration parameters:
#
# - server
# The FQDN hostname of the Graphite server.
# (optional)
#
# - prefix
# The prefix to use while storing Graphite data.
# (optional; default: 'vmpooler')
# Example:
:graphite:
server: 'graphite.company.com'
# :config:
#
# This section contains global configuration information.
@ -63,10 +83,6 @@
# The path to vmpooler's log file.
# (optional; default: '/var/log/vmpooler.log')
#
# - graphite
# The FQDN hostname of the Graphite server.
# (optional)
#
# - task_limit
# The number of concurrent VMware vSphere tasks to perform.
# (optional; default: '10')
@ -84,7 +100,6 @@
:config:
site_name: 'vmpooler'
logfile: '/var/log/vmpooler.log'
graphite: 'graphite.company.com'
task_limit: 10
vm_checktime: 15
vm_lifetime: 12