puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 01:58:41 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Eliminate duplicate VM object lookups where possible (#269)

Browse source 
* Minimize duplicate checking of VMs

This commit updates check_pool pending, running and ready methods to greatly reduce instances in which the VM object is retrieved. Without this change get_vm is run for each of these check_pool steps even though the VM is already validated as being in inventory being running the check. This is left for checking running VMs when the VM is no longer ready. Without this change there is an unnecessarily large volume of VM object checks.

* Make hostname mismatch checking optional

This commit makes hostname mismatch checking optional on a pool and global config level. The default behavior of checking this is preserved. When disabled _check_ready_vm will not run get_vm, which allows for ready VMs to be checked without retrieving an object from vsphere and waiting for a free provider connection. Without this change it is not possible to disable VM object retrieval in _check_vm_ready.

* Check if a hostname return is empty string

This commit checks whether a hostname returned is an empty string.
Without this change a VM that returns a hostname with a empty string
will report as having a hostname mismatch, which may happen before all
VM data is updated.

* Only check hostname for mismatch after time since ready has past

Configure hostname checking so it is not checked until a VM has been
ready for greater than one minute. Without this change hostname checking
will often fail because the configured value has not propogated to the
VM on some platforms before this check is run.
This commit is contained in:
mattkirby 2018-07-13 10:06:44 -07:00 • committed by Samuel
parent 7e5ef2f4e5
commit 070199e877
2 changed files with 164 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

135
lib/vmpooler/pool_manager.rb
View file

@ -49,13 +49,8 @@ module Vmpooler
mutex = vm_mutex(vm) mutex = vm_mutex(vm)
return if mutex.locked? return if mutex.locked?
mutex.synchronize do mutex.synchronize do
host = provider.get_vm(pool, vm)
unless host
fail_pending_vm(vm, pool, timeout, false)
return
end
if provider.vm_ready?(pool, vm) if provider.vm_ready?(pool, vm)
move_pending_vm_to_ready(vm, pool, host) move_pending_vm_to_ready(vm, pool)
else else
fail_pending_vm(vm, pool, timeout) fail_pending_vm(vm, pool, timeout)
end end
@ -86,25 +81,27 @@ module Vmpooler
false false
end end
def move_pending_vm_to_ready(vm, pool, host) def move_pending_vm_to_ready(vm, pool)
if host['hostname'] == vm clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
begin finish = format('%.2f', Time.now - Time.parse(clone_time)) if clone_time
Socket.getaddrinfo(vm, nil) # WTF? I assume this is just priming the local DNS resolver cache?!?!
rescue # rubocop:disable Lint/HandleExceptions
# Do not care about errors what-so-ever
end
clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone') $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
finish = format('%.2f', Time.now - Time.parse(clone_time)) if clone_time $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
$redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
$redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm) # last boot time is displayed in API, and used by alarming script
$redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself? $redis.hset('vmpooler__lastboot', pool, Time.now)
# last boot time is displayed in API, and used by alarming script $metrics.timing("time_to_ready_state.#{pool}", finish)
$redis.hset('vmpooler__lastboot', pool, Time.now) $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
$metrics.timing("time_to_ready_state.#{pool}", finish) end
$logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
end def vm_still_ready?(pool_name, vm_name, provider)
# Check if the VM is still ready/available
return true if provider.vm_ready?(pool_name, vm_name)
raise("VM #{vm_name} is not ready")
rescue
move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
end end
def check_ready_vm(vm, pool, ttl, provider) def check_ready_vm(vm, pool, ttl, provider)
@ -112,7 +109,7 @@ module Vmpooler
begin begin
_check_ready_vm(vm, pool, ttl, provider) _check_ready_vm(vm, pool, ttl, provider)
rescue => err rescue => err
$logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : #{err}") $logger.log('s', "[!] [#{pool['name']}] '#{vm}' failed while checking a ready vm : #{err}")
raise raise
end end
end end
@ -126,53 +123,50 @@ module Vmpooler
check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
host = provider.get_vm(pool, vm)
# Check if the host even exists
unless host
$redis.srem('vmpooler__ready__' + pool, vm)
$logger.log('s', "[!] [#{pool}] '#{vm}' not found in inventory, removed from 'ready' queue")
return
end
$redis.hset('vmpooler__vm__' + vm, 'check', Time.now) $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
# Check if the VM is not powered on, before checking TTL
unless host['powerstate'].casecmp('poweredon').zero?
$redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
return
end
# Check if the hosts TTL has expired # Check if the hosts TTL has expired
if ttl > 0 if ttl > 0
# host['boottime'] may be nil if host is not powered on # host['boottime'] may be nil if host is not powered on
if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
$redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
return return
end end
end end
# Check if the hostname has magically changed from underneath Pooler return if has_mismatched_hostname?(vm, pool, provider)
if host['hostname'] != vm
$redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
return
end
# Check if the VM is still ready/available vm_still_ready?(pool['name'], vm, provider)
begin
raise("VM #{vm} is not ready") unless provider.vm_ready?(pool, vm)
rescue
if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
else
$logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue")
end
end
end end
end end
def has_mismatched_hostname?(vm, pool, provider)
check_hostname = pool['check_hostname_for_mismatch']
check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
return if check_hostname == false
# Wait one minute before checking a VM for hostname mismatch
# When checking as soon as the VM passes the ready test the instance
# often doesn't report its hostname yet causing the VM to be removed immediately
vm_ready_time = $redis.hget("vmpooler__vm__#{vm}", 'ready')
if vm_ready_time
wait_before_checking = 60
time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
return unless time_since_ready > wait_before_checking
end
# Check if the hostname has magically changed from underneath Pooler
vm_hash = provider.get_vm(pool['name'], vm)
hostname = vm_hash['hostname']
return if hostname.empty?
return if hostname == vm
$redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
return true
end
def check_running_vm(vm, pool, ttl, provider) def check_running_vm(vm, pool, ttl, provider)
Thread.new do Thread.new do
begin begin
@ -188,17 +182,26 @@ module Vmpooler
mutex = vm_mutex(vm) mutex = vm_mutex(vm)
return if mutex.locked? return if mutex.locked?
mutex.synchronize do mutex.synchronize do
host = provider.get_vm(pool, vm) # Check that VM is within defined lifetime
checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
if checkouttime
running = (Time.now - Time.parse(checkouttime)) / 60 / 60
if host if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
# Check that VM is within defined lifetime move_vm_queue(pool, vm, 'running', 'completed', "reached end of TTL after #{ttl} hours")
checkouttime = $redis.hget('vmpooler__active__' + pool, vm) return
if checkouttime end
running = (Time.now - Time.parse(checkouttime)) / 60 / 60 end
if (ttl.to_i > 0) && (running.to_i >= ttl.to_i) if provider.vm_ready?(pool, vm)
move_vm_queue(pool, vm, 'running', 'completed', "reached end of TTL after #{ttl} hours") return
end else
host = provider.get_vm(pool, vm)
if host
return
else
move_vm_queue(pool, vm, 'running', 'completed', "is no longer in inventory, removing from running")
end end
end end
end end
@ -761,7 +764,7 @@ module Vmpooler
if inventory[vm] if inventory[vm]
begin begin
pool_check_response[:checked_ready_vms] += 1 pool_check_response[:checked_ready_vms] += 1
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, provider) check_ready_vm(vm, pool, pool['ready_ttl'] || 0, provider)
rescue => err rescue => err
$logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}") $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}")
end end

229
spec/unit/pool_manager_spec.rb
View file

@ -64,23 +64,11 @@ EOT
expect(subject).not_to be_nil expect(subject).not_to be_nil
end end
context 'host does not exist or not in pool' do
it 'calls fail_pending_vm' do
expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, false)
subject._check_pending_vm(vm, pool, timeout, provider)
end
end
context 'host is in pool' do context 'host is in pool' do
before do
expect(provider).to receive(:get_vm).with(pool,vm).and_return(host)
end
it 'calls move_pending_vm_to_ready if host is ready' do it 'calls move_pending_vm_to_ready if host is ready' do
expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(true) expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(true)
expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool, host) expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool)
subject._check_pending_vm(vm, pool, timeout, provider) subject._check_pending_vm(vm, pool, timeout, provider)
end end
@ -178,35 +166,23 @@ EOT
subject.fail_pending_vm(vm, pool, timeout,true) subject.fail_pending_vm(vm, pool, timeout,true)
end end
end end
describe '#move_pending_vm_to_ready' do describe '#move_pending_vm_to_ready' do
let(:host) { { 'hostname' => vm }} let(:host) { { 'hostname' => vm }}
before do before do
expect(subject).not_to be_nil expect(subject).not_to be_nil
allow(Socket).to receive(:getaddrinfo)
end end
before(:each) do before(:each) do
create_pending_vm(pool,vm) create_pending_vm(pool,vm)
end end
context 'when hostname does not match VM name' do
it 'should not take any action' do
expect(logger).to receive(:log).exactly(0).times
expect(Socket).to receive(:getaddrinfo).exactly(0).times
host['hostname'] = 'different_name'
subject.move_pending_vm_to_ready(vm, pool, host)
end
end
context 'when hostname matches VM name' do context 'when hostname matches VM name' do
it 'should move the VM from pending to ready pool' do it 'should move the VM from pending to ready pool' do
expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
end end
@ -214,39 +190,39 @@ EOT
it 'should log a message' do it 'should log a message' do
expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
end end
it 'should receive time_to_ready_state metric' do it 'should receive time_to_ready_state metric' do
redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s) redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/) expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/)
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
end end
it 'should set the boot time in redis' do it 'should set the boot time in redis' do
redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s) redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil
# TODO Should we inspect the value to see if it's valid? # TODO Should we inspect the value to see if it's valid?
end end
it 'should not determine boot timespan if clone start time not set' do it 'should not determine boot timespan if clone start time not set' do
expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to eq("") # Possible implementation bug here. Should still be nil here expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to eq("") # Possible implementation bug here. Should still be nil here
end end
it 'should raise error if clone start time is not parsable' do it 'should raise error if clone start time is not parsable' do
redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate') redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
expect{subject.move_pending_vm_to_ready(vm, pool, host)}.to raise_error(/iamnotparsable_asdate/) expect{subject.move_pending_vm_to_ready(vm, pool)}.to raise_error(/iamnotparsable_asdate/)
end end
it 'should save the last boot time' do it 'should save the last boot time' do
expect(redis.hget('vmpooler__lastboot', pool)).to be(nil) expect(redis.hget('vmpooler__lastboot', pool)).to be(nil)
subject.move_pending_vm_to_ready(vm, pool, host) subject.move_pending_vm_to_ready(vm, pool)
expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil) expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil)
end end
end end
@ -254,6 +230,7 @@ EOT
describe '#check_ready_vm' do describe '#check_ready_vm' do
let(:ttl) { 0 } let(:ttl) { 0 }
let(:poolconfig) { config[:pools][0] }
before do before do
expect(subject).not_to be_nil expect(subject).not_to be_nil
@ -261,15 +238,16 @@ EOT
it 'calls _check_ready_vm' do it 'calls _check_ready_vm' do
expect(Thread).to receive(:new).and_yield expect(Thread).to receive(:new).and_yield
expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, provider) expect(subject).to receive(:_check_ready_vm).with(vm, poolconfig, ttl, provider)
subject.check_ready_vm(vm, pool, ttl, provider) subject.check_ready_vm(vm, poolconfig, ttl, provider)
end end
end end
describe '#_check_ready_vm' do describe '#_check_ready_vm' do
let(:ttl) { 0 } let(:ttl) { 0 }
let(:host) { {} } let(:host) { {} }
let(:poolconfig) { config[:pools][0] }
before(:each) do before(:each) do
create_ready_vm(pool,vm) create_ready_vm(pool,vm)
@ -287,40 +265,17 @@ EOT
check_stamp = (Time.now - 60).to_s check_stamp = (Time.now - 60).to_s
redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp) redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp)
expect(provider).to receive(:get_vm).exactly(0).times expect(provider).to receive(:get_vm).exactly(0).times
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp) expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp)
end end
end end
context 'a VM that does not exist' do
before do
expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
end
it 'should not set the current check timestamp' do
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
subject._check_ready_vm(vm, pool, ttl, provider)
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
end
it 'should log a message' do
expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory, removed from 'ready' queue")
subject._check_ready_vm(vm, pool, ttl, provider)
end
it 'should remove the VM from the ready queue' do
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
subject._check_ready_vm(vm, pool, ttl, provider)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
end
end
context 'a VM that has never been checked' do context 'a VM that has never been checked' do
let(:last_check_date) { Date.new(2001,1,1).to_s } let(:last_check_date) { Date.new(2001,1,1).to_s }
it 'should set the current check timestamp' do it 'should set the current check timestamp' do
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil
end end
end end
@ -333,7 +288,7 @@ EOT
it 'should set the current check timestamp' do it 'should set the current check timestamp' do
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date) expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date)
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date) expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date)
end end
@ -343,63 +298,11 @@ EOT
end end
it 'should only set the next check interval' do it 'should only set the next check interval' do
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
end end
end end
context 'is turned off' do context 'has correct name and is not ready' do
before(:each) do
host['powerstate'] = 'PoweredOff'
end
it 'should move the VM to the completed queue' do
expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
subject._check_ready_vm(vm, pool, ttl, provider)
end
it 'should move the VM to the completed queue in Redis' do
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
subject._check_ready_vm(vm, pool, ttl, provider)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
end
it 'should log messages about being powered off' do
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
subject._check_ready_vm(vm, pool, ttl, provider)
end
end
context 'is turned on, a name mismatch' do
before(:each) do
host['hostname'] = 'different_name'
end
it 'should move the VM to the completed queue' do
expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
subject._check_ready_vm(vm, pool, ttl, provider)
end
it 'should move the VM to the completed queue in Redis' do
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
subject._check_ready_vm(vm, pool, ttl, provider)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
end
it 'should log messages about being misnamed' do
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
subject._check_ready_vm(vm, pool, ttl, provider)
end
end
context 'is turned on, with correct name and is not ready' do
before(:each) do before(:each) do
expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(false) expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(false)
end end
@ -407,13 +310,13 @@ EOT
it 'should move the VM to the completed queue' do it 'should move the VM to the completed queue' do
expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
end end
it 'should move the VM to the completed queue in Redis' do it 'should move the VM to the completed queue in Redis' do
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
end end
@ -421,7 +324,72 @@ EOT
it 'should log messages about being unreachable' do it 'should log messages about being unreachable' do
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
subject._check_ready_vm(vm, pool, ttl, provider) subject._check_ready_vm(vm, poolconfig, ttl, provider)
end
end
context 'with hostname mismatch checking enabled' do
context 'when less than 60 seconds since a VM moved to ready' do
before(:each) do
redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
end
it 'should return nil' do
expect(subject._check_ready_vm(vm, poolconfig, ttl, provider)).to be_nil
end
end
context 'with a hostname mismatch' do
let(:different_hostname) { 'different_name' }
before(:each) do
expect(provider).to receive(:get_vm).with(pool,vm).and_return(host)
host['hostname'] = different_hostname
end
it 'should move the VM to the completed queue' do
expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
subject._check_ready_vm(vm, poolconfig, ttl, provider)
end
it 'should move the VM to the completed queue in Redis' do
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
subject._check_ready_vm(vm, poolconfig, ttl, provider)
expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
end
it 'should log messages about being misnamed' do
expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' has mismatched hostname #{different_hostname}, removed from 'ready' queue")
subject._check_ready_vm(vm, poolconfig, ttl, provider)
end
end
end
context 'with hostname mismatch checking disabled on the pool' do
before(:each) do
config[:pools][0]['check_hostname_for_mismatch'] = false
end
it 'should not run get_vm' do
expect(provider).to_not receive(:get_vm)
subject._check_ready_vm(vm, poolconfig, ttl, provider)
end
end
context 'with hostname mismatch checking disabled in config' do
before(:each) do
config[:config]['check_ready_vm_hostname_for_mismatch'] = false
end
it 'should not run get_vm' do
expect(provider).to_not receive(:get_vm)
subject._check_ready_vm(vm, poolconfig, ttl, provider)
end end
end end
end end
@ -435,7 +403,7 @@ EOT
it 'should return' do it 'should return' do
expect(subject).to receive(:vm_mutex).and_return(mutex) expect(subject).to receive(:vm_mutex).and_return(mutex)
expect(subject._check_ready_vm(vm, pool, ttl, provider)).to be_nil expect(subject._check_ready_vm(vm, poolconfig, ttl, provider)).to be_nil
end end
end end
end end
@ -472,32 +440,24 @@ EOT
allow(provider).to receive(:get_vm).with(pool,vm).and_return(host) allow(provider).to receive(:get_vm).with(pool,vm).and_return(host)
end end
it 'does nothing with a missing VM' do it 'moves a missing VM to the completed queue' do
expect(provider).to receive(:vm_ready?).and_return(false)
expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil) expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
subject._check_running_vm(vm, pool, timeout, provider) subject._check_running_vm(vm, pool, timeout, provider)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
end end
context 'valid host' do context 'valid host' do
it 'should not move VM when not poweredOn' do
# I'm not sure this test is useful. There is no codepath
# in _check_running_vm that looks at Power State
host['powerstate'] = 'PoweredOff'
expect(logger).not_to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
subject._check_running_vm(vm, pool, timeout, provider)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
end
it 'should not move VM if it has no checkout time' do it 'should not move VM if it has no checkout time' do
expect(provider).to receive(:vm_ready?).and_return(true)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
subject._check_running_vm(vm, pool, 0, provider) subject._check_running_vm(vm, pool, 0, provider)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
end end
it 'should not move VM if TTL is zero' do it 'should not move VM if TTL is zero' do
expect(provider).to receive(:vm_ready?).and_return(true)
redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s) redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
subject._check_running_vm(vm, pool, 0, provider) subject._check_running_vm(vm, pool, 0, provider)
@ -2943,6 +2903,7 @@ EOT
end end
context 'Ready VM in the inventory' do context 'Ready VM in the inventory' do
let(:poolconfig) { config[:pools][0] }
before(:each) do before(:each) do
expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response)
allow(subject).to receive(:check_ready_vm) allow(subject).to receive(:check_ready_vm)
@ -2966,13 +2927,13 @@ EOT
big_lifetime = 2000 big_lifetime = 2000
config[:pools][0]['ready_ttl'] = big_lifetime config[:pools][0]['ready_ttl'] = big_lifetime
expect(subject).to receive(:check_ready_vm).with(vm,pool,big_lifetime,provider) expect(subject).to receive(:check_ready_vm).with(vm,poolconfig,big_lifetime,provider)
subject._check_pool(pool_object,provider) subject._check_pool(pool_object,provider)
end end
it 'should use a pool TTL of zero if none set' do it 'should use a pool TTL of zero if none set' do
expect(subject).to receive(:check_ready_vm).with(vm,pool,0,provider) expect(subject).to receive(:check_ready_vm).with(vm,poolconfig,0,provider)
subject._check_pool(pool_object,provider) subject._check_pool(pool_object,provider)
end end