mirror of
https://github.com/puppetlabs/vmpooler.git
synced 2026-01-26 10:08:40 -05:00
Log empty pools
Make a note in the logfile when a pool is detected to be empty. Also: - vmpooler__empty__<pool> Redis key to determine when to log - lifetime/TTL checks moved to `_check_running_vm` method - no longer pay attention to VMware-based 'host.runtime.bootTime' This PR implements a bunch of other stuff to account for rspec testing: - Thread creation and looping in `check_pool` - Everything else in `_check_pool`
This commit is contained in:
Scott Schneider
parent
1d483b1374
commit
821ffd866a
2 changed files with 184 additions and 159 deletions
|
|
@ -142,14 +142,20 @@ module Vmpooler
|
||||||
if host
|
if host
|
||||||
queue_from, queue_to = 'running', 'completed'
|
queue_from, queue_to = 'running', 'completed'
|
||||||
|
|
||||||
|
# Check that VM is powered on
|
||||||
if (host.runtime) &&
|
if (host.runtime) &&
|
||||||
(host.runtime.powerState != 'poweredOn')
|
(host.runtime.powerState != 'poweredOn')
|
||||||
move_vm_queue(pool, vm, queue_from, queue_to, 'appears to be powered off or dead')
|
move_vm_queue(pool, vm, queue_from, queue_to, 'appears to be powered off or dead')
|
||||||
else
|
end
|
||||||
if (host.runtime) &&
|
|
||||||
(host.runtime.bootTime)
|
# Check that VM is within defined lifetime
|
||||||
((((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
|
checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
|
||||||
move_vm_queue(pool, vm, queue_from, queue_to, "reached end of TTL after #{ttl} minutes")
|
if checkouttime
|
||||||
|
running = (Time.now - Time.parse(checkouttime)) / 60 / 60
|
||||||
|
|
||||||
|
if (ttl.to_i > 0) &&
|
||||||
|
(running.to_i >= ttl.to_i)
|
||||||
|
move_vm_queue(pool, vm, queue_from, queue_to, "reached end of TTL after #{ttl} hours")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -289,165 +295,147 @@ module Vmpooler
|
||||||
def check_pool(pool)
|
def check_pool(pool)
|
||||||
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
|
$logger.log('d', "[*] [#{pool['name']}] starting worker thread")
|
||||||
|
|
||||||
|
$vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
|
||||||
|
|
||||||
$threads[pool['name']] = Thread.new do
|
$threads[pool['name']] = Thread.new do
|
||||||
$vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
|
|
||||||
|
|
||||||
loop do
|
loop do
|
||||||
# INVENTORY
|
_check_pool(pool)
|
||||||
inventory = {}
|
|
||||||
begin
|
|
||||||
base = $vsphere[pool['name']].find_folder(pool['folder'])
|
|
||||||
|
|
||||||
base.childEntity.each do |vm|
|
|
||||||
if
|
|
||||||
(! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) &&
|
|
||||||
(! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) &&
|
|
||||||
(! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) &&
|
|
||||||
(! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) &&
|
|
||||||
(! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']))
|
|
||||||
|
|
||||||
$redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
|
|
||||||
|
|
||||||
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
|
||||||
end
|
|
||||||
|
|
||||||
inventory[vm['name']] = 1
|
|
||||||
end
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
|
|
||||||
# RUNNING
|
|
||||||
$redis.smembers('vmpooler__running__' + pool['name']).each do |vm|
|
|
||||||
if inventory[vm]
|
|
||||||
if pool['running_ttl']
|
|
||||||
begin
|
|
||||||
check_running_vm(vm, pool['name'], pool['running_ttl'])
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
else
|
|
||||||
begin
|
|
||||||
check_running_vm(vm, pool['name'], '720')
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# READY
|
|
||||||
$redis.smembers('vmpooler__ready__' + pool['name']).each do |vm|
|
|
||||||
if inventory[vm]
|
|
||||||
begin
|
|
||||||
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# PENDING
|
|
||||||
$redis.smembers('vmpooler__pending__' + pool['name']).each do |vm|
|
|
||||||
unless pool['timeout']
|
|
||||||
if $config[:config]['timeout']
|
|
||||||
pool['timeout'] = $config[:config]['timeout']
|
|
||||||
else
|
|
||||||
pool['timeout'] = 15
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
if inventory[vm]
|
|
||||||
begin
|
|
||||||
check_pending_vm(vm, pool['name'], pool['timeout'])
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# COMPLETED
|
|
||||||
$redis.smembers('vmpooler__completed__' + pool['name']).each do |vm|
|
|
||||||
if inventory[vm]
|
|
||||||
begin
|
|
||||||
destroy_vm(vm, pool['name'])
|
|
||||||
rescue
|
|
||||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
|
|
||||||
$redis.srem('vmpooler__completed__' + pool['name'], vm)
|
|
||||||
$redis.hdel('vmpooler__active__' + pool['name'], vm)
|
|
||||||
$redis.del('vmpooler__vm__' + vm)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
|
||||||
$redis.srem('vmpooler__completed__' + pool['name'], vm)
|
|
||||||
$redis.hdel('vmpooler__active__' + pool['name'], vm)
|
|
||||||
$redis.del('vmpooler__vm__' + vm)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# DISCOVERED
|
|
||||||
$redis.smembers('vmpooler__discovered__' + pool['name']).each do |vm|
|
|
||||||
%w(pending ready running completed).each do |queue|
|
|
||||||
if $redis.sismember('vmpooler__' + queue + '__' + pool['name'], vm)
|
|
||||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
|
||||||
$redis.srem('vmpooler__discovered__' + pool['name'], vm)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
if $redis.sismember('vmpooler__discovered__' + pool['name'], vm)
|
|
||||||
$redis.smove('vmpooler__discovered__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# LONG-RUNNING
|
|
||||||
$redis.smembers('vmpooler__running__' + pool['name']).each do |vm|
|
|
||||||
if $redis.hget('vmpooler__active__' + pool['name'], vm)
|
|
||||||
running = (Time.now - Time.parse($redis.hget('vmpooler__active__' + pool['name'], vm))) / 60 / 60
|
|
||||||
lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime']
|
|
||||||
|
|
||||||
if
|
|
||||||
(lifetime.to_i > 0) &&
|
|
||||||
(running.to_i >= lifetime.to_i)
|
|
||||||
|
|
||||||
$redis.smove('vmpooler__running__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
|
|
||||||
|
|
||||||
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{lifetime} hours")
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# REPOPULATE
|
|
||||||
total = $redis.scard('vmpooler__ready__' + pool['name']) +
|
|
||||||
$redis.scard('vmpooler__pending__' + pool['name'])
|
|
||||||
|
|
||||||
begin
|
|
||||||
if defined? $graphite
|
|
||||||
$graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
|
|
||||||
$graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
|
|
||||||
end
|
|
||||||
rescue
|
|
||||||
end
|
|
||||||
|
|
||||||
if total < pool['size']
|
|
||||||
(1..(pool['size'] - total)).each do |_i|
|
|
||||||
if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit']
|
|
||||||
begin
|
|
||||||
$redis.incr('vmpooler__tasks__clone')
|
|
||||||
|
|
||||||
clone_vm(
|
|
||||||
pool['template'],
|
|
||||||
pool['folder'],
|
|
||||||
pool['datastore'],
|
|
||||||
pool['clone_target']
|
|
||||||
)
|
|
||||||
rescue
|
|
||||||
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
|
|
||||||
$redis.decr('vmpooler__tasks__clone')
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
sleep(5)
|
sleep(5)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def _check_pool(pool)
|
||||||
|
# INVENTORY
|
||||||
|
inventory = {}
|
||||||
|
begin
|
||||||
|
base = $vsphere[pool['name']].find_folder(pool['folder'])
|
||||||
|
|
||||||
|
base.childEntity.each do |vm|
|
||||||
|
if
|
||||||
|
(! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) &&
|
||||||
|
(! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) &&
|
||||||
|
(! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) &&
|
||||||
|
(! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) &&
|
||||||
|
(! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']))
|
||||||
|
|
||||||
|
$redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
|
||||||
|
|
||||||
|
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
||||||
|
end
|
||||||
|
|
||||||
|
inventory[vm['name']] = 1
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
|
||||||
|
# RUNNING
|
||||||
|
$redis.smembers('vmpooler__running__' + pool['name']).each do |vm|
|
||||||
|
if inventory[vm]
|
||||||
|
begin
|
||||||
|
check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12)
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# READY
|
||||||
|
$redis.smembers('vmpooler__ready__' + pool['name']).each do |vm|
|
||||||
|
if inventory[vm]
|
||||||
|
begin
|
||||||
|
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# PENDING
|
||||||
|
$redis.smembers('vmpooler__pending__' + pool['name']).each do |vm|
|
||||||
|
if inventory[vm]
|
||||||
|
begin
|
||||||
|
check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15)
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# COMPLETED
|
||||||
|
$redis.smembers('vmpooler__completed__' + pool['name']).each do |vm|
|
||||||
|
if inventory[vm]
|
||||||
|
begin
|
||||||
|
destroy_vm(vm, pool['name'])
|
||||||
|
rescue
|
||||||
|
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
|
||||||
|
$redis.srem('vmpooler__completed__' + pool['name'], vm)
|
||||||
|
$redis.hdel('vmpooler__active__' + pool['name'], vm)
|
||||||
|
$redis.del('vmpooler__vm__' + vm)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
||||||
|
$redis.srem('vmpooler__completed__' + pool['name'], vm)
|
||||||
|
$redis.hdel('vmpooler__active__' + pool['name'], vm)
|
||||||
|
$redis.del('vmpooler__vm__' + vm)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# DISCOVERED
|
||||||
|
$redis.smembers('vmpooler__discovered__' + pool['name']).each do |vm|
|
||||||
|
%w(pending ready running completed).each do |queue|
|
||||||
|
if $redis.sismember('vmpooler__' + queue + '__' + pool['name'], vm)
|
||||||
|
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
||||||
|
$redis.srem('vmpooler__discovered__' + pool['name'], vm)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $redis.sismember('vmpooler__discovered__' + pool['name'], vm)
|
||||||
|
$redis.smove('vmpooler__discovered__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# REPOPULATE
|
||||||
|
ready = $redis.scard('vmpooler__ready__' + pool['name'])
|
||||||
|
total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
|
||||||
|
|
||||||
|
begin
|
||||||
|
if defined? $graphite
|
||||||
|
$graphite.log($config[:graphite]['prefix'] + '.ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name']))
|
||||||
|
$graphite.log($config[:graphite]['prefix'] + '.running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name']))
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
|
||||||
|
if $redis.get('vmpooler__empty__' + pool['name'])
|
||||||
|
unless ready == 0
|
||||||
|
$redis.del('vmpooler__empty__' + pool['name'])
|
||||||
|
end
|
||||||
|
else
|
||||||
|
if ready == 0
|
||||||
|
$redis.set('vmpooler__empty__' + pool['name'], 'true')
|
||||||
|
$logger.log('s', "[!] [#{pool['name']}] is empty")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if total < pool['size']
|
||||||
|
(1..(pool['size'] - total)).each do |_i|
|
||||||
|
if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
|
||||||
|
begin
|
||||||
|
$redis.incr('vmpooler__tasks__clone')
|
||||||
|
|
||||||
|
clone_vm(
|
||||||
|
pool['template'],
|
||||||
|
pool['folder'],
|
||||||
|
pool['datastore'],
|
||||||
|
pool['clone_target']
|
||||||
|
)
|
||||||
|
rescue
|
||||||
|
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
|
||||||
|
$redis.decr('vmpooler__tasks__clone')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def execute!
|
def execute!
|
||||||
$logger.log('d', 'starting vmpooler')
|
$logger.log('d', 'starting vmpooler')
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -169,6 +169,7 @@ describe 'Pool Manager' do
|
||||||
allow(vm_host).to receive(:runtime).and_return true
|
allow(vm_host).to receive(:runtime).and_return true
|
||||||
allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff'
|
allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff'
|
||||||
|
|
||||||
|
expect(redis).to receive(:hget)
|
||||||
expect(redis).to receive(:smove)
|
expect(redis).to receive(:smove)
|
||||||
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
|
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
|
||||||
|
|
||||||
|
|
@ -179,10 +180,10 @@ describe 'Pool Manager' do
|
||||||
allow(pool_helper).to receive(:find_vm).and_return vm_host
|
allow(pool_helper).to receive(:find_vm).and_return vm_host
|
||||||
allow(vm_host).to receive(:runtime).and_return true
|
allow(vm_host).to receive(:runtime).and_return true
|
||||||
allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn'
|
allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn'
|
||||||
allow(vm_host).to receive_message_chain(:runtime, :bootTime).and_return Time.parse('2005-01-01')
|
|
||||||
|
|
||||||
|
expect(redis).to receive(:hget).with('vmpooler__active__pool1', 'vm1').and_return((Time.now - timeout*60*60).to_s)
|
||||||
expect(redis).to receive(:smove)
|
expect(redis).to receive(:smove)
|
||||||
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} minutes")
|
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} hours")
|
||||||
|
|
||||||
subject._check_running_vm(vm, pool, timeout)
|
subject._check_running_vm(vm, pool, timeout)
|
||||||
end
|
end
|
||||||
|
|
@ -211,4 +212,40 @@ describe 'Pool Manager' do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe '#_check_pool' do
|
||||||
|
let(:pool_helper) { double('pool') }
|
||||||
|
let(:vsphere) { {pool => pool_helper} }
|
||||||
|
let(:config) { {
|
||||||
|
config: { task_limit: 10 },
|
||||||
|
pools: [ {'name' => 'pool1', 'size' => 5} ]
|
||||||
|
} }
|
||||||
|
|
||||||
|
before do
|
||||||
|
expect(subject).not_to be_nil
|
||||||
|
$vsphere = vsphere
|
||||||
|
allow(logger).to receive(:log)
|
||||||
|
allow(pool_helper).to receive(:find_folder)
|
||||||
|
allow(redis).to receive(:smembers).with('vmpooler__pending__pool1').and_return([])
|
||||||
|
allow(redis).to receive(:smembers).with('vmpooler__ready__pool1').and_return([])
|
||||||
|
allow(redis).to receive(:smembers).with('vmpooler__running__pool1').and_return([])
|
||||||
|
allow(redis).to receive(:smembers).with('vmpooler__completed__pool1').and_return([])
|
||||||
|
allow(redis).to receive(:smembers).with('vmpooler__discovered__pool1').and_return([])
|
||||||
|
allow(redis).to receive(:set)
|
||||||
|
allow(redis).to receive(:get).with('vmpooler__tasks__clone').and_return(0)
|
||||||
|
allow(redis).to receive(:get).with('vmpooler__empty__pool1').and_return(nil)
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'logging' do
|
||||||
|
|
||||||
|
it 'logs empty pool' do
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
|
||||||
|
allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
|
||||||
|
|
||||||
|
expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
|
||||||
|
subject._check_pool(config[:pools][0])
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue