Do not run duplicate instances of inventory check for a pool

This commit updates check_pool inventory check to prevent multiple instances from running at once. Without this change the inventory check may run in multiple threads simultaneously.
This commit is contained in:
kirby@puppetlabs.com 2018-06-28 20:32:44 -07:00
parent 8be578493a
commit df89617fdc
2 changed files with 37 additions and 12 deletions

View file

@ -708,21 +708,24 @@ module Vmpooler
# INVENTORY # INVENTORY
inventory = {} inventory = {}
begin begin
provider.vms_in_pool(pool['name']).each do |vm| mutex = pool_mutex(pool['name'])
if !$redis.sismember('vmpooler__running__' + pool['name'], vm['name']) && mutex.synchronize do
!$redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) && provider.vms_in_pool(pool['name']).each do |vm|
!$redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) && if !$redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
!$redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) && !$redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
!$redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) && !$redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
!$redis.sismember('vmpooler__migrating__' + pool['name'], vm['name']) !$redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
!$redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
!$redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
pool_check_response[:discovered_vms] += 1 pool_check_response[:discovered_vms] += 1
$redis.sadd('vmpooler__discovered__' + pool['name'], vm['name']) $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue") $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
end
inventory[vm['name']] = 1
end end
inventory[vm['name']] = 1
end end
rescue => err rescue => err
$logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while inspecting inventory: #{err}") $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while inspecting inventory: #{err}")

View file

@ -2685,6 +2685,7 @@ EOT
let(:pool_object) { config[:pools][0] } let(:pool_object) { config[:pools][0] }
let(:new_vm) { 'newvm'} let(:new_vm) { 'newvm'}
let(:pool_name) { pool_object['name'] } let(:pool_name) { pool_object['name'] }
let(:mutex) { Mutex.new }
before do before do
expect(subject).not_to be_nil expect(subject).not_to be_nil
@ -2790,6 +2791,27 @@ EOT
end end
end end
end end
it 'should get the pool mutex' do
expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)
subject._check_pool(pool_object,provider)
end
it 'should run synchronize' do
expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)
expect(mutex).to receive(:synchronize).at_least(:once)
subject._check_pool(pool_object,provider)
end
it 'should yield when locked' do
expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)
mutex.lock
expect(mutex).to receive(:synchronize).and_yield
subject._check_pool(pool_object,provider)
end
end end
# RUNNING # RUNNING