puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 10:08:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add mutex object for managing pool configuration updates

Browse source 
This commit adds a config_changes object for ensuring that pool configuration changes are synchronized across multiple running threads, removing the possibility of two threads attempting to update something at once, without relying on redis data. Without this change this is managed crudely by specifying in redis that a configuration update is taking place. This redis data is left so the REPOPULATE section of _check_pool can still identify when a configuration change is in progress, and prevent a pool from repopulating at that time.
This commit is contained in:
kirby@puppetlabs.com 2018-05-23 13:37:17 -07:00
parent 2d679168d7
commit ddecb8b8d0
1 changed files with 52 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

92
lib/vmpooler/pool_manager.rb
View file

@ -21,6 +21,9 @@ module Vmpooler
# Our thread-tracker object # Our thread-tracker object
$threads = {} $threads = {}
# Pool mutex
@reconfigure_pool = {}
end end
def config def config
@ -560,12 +563,15 @@ module Vmpooler
# Ensure templates are evaluated for delta disk creation on startup # Ensure templates are evaluated for delta disk creation on startup
return if $redis.hget('vmpooler__config__updating', pool['name']) return if $redis.hget('vmpooler__config__updating', pool['name'])
unless $redis.hget('vmpooler__template__prepared', pool['name']) unless $redis.hget('vmpooler__template__prepared', pool['name'])
begin mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
$redis.hset('vmpooler__config__updating', pool['name'], 1) mutex.synchronize do
provider.create_template_delta_disks(pool) begin
$redis.hset('vmpooler__template__prepared', pool['name'], pool['template']) $redis.hset('vmpooler__config__updating', pool['name'], 1)
ensure provider.create_template_delta_disks(pool)
$redis.hdel('vmpooler__config__updating', pool['name']) $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
ensure
$redis.hdel('vmpooler__config__updating', pool['name'])
end
end end
end end
end end
@ -575,49 +581,55 @@ module Vmpooler
$redis.hset('vmpooler__template', pool['name'], pool['template']) unless $redis.hget('vmpooler__template', pool['name']) $redis.hset('vmpooler__template', pool['name'], pool['template']) unless $redis.hget('vmpooler__template', pool['name'])
return unless $redis.hget('vmpooler__config__template', pool['name']) return unless $redis.hget('vmpooler__config__template', pool['name'])
unless $redis.hget('vmpooler__config__template', pool['name']) == $redis.hget('vmpooler__template', pool['name']) unless $redis.hget('vmpooler__config__template', pool['name']) == $redis.hget('vmpooler__template', pool['name'])
# Ensure we are only updating a template once
return if $redis.hget('vmpooler__config__updating', pool['name']) return if $redis.hget('vmpooler__config__updating', pool['name'])
begin # Ensure we are only updating a template once
$redis.hset('vmpooler__config__updating', pool['name'], 1) mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
mutex.synchronize do
old_template_name = $redis.hget('vmpooler__template', pool['name']) begin
new_template_name = $redis.hget('vmpooler__config__template', pool['name']) $redis.hset('vmpooler__config__updating', pool['name'], 1)
pool['template'] = new_template_name old_template_name = $redis.hget('vmpooler__template', pool['name'])
$redis.hset('vmpooler__template', pool['name'], new_template_name) new_template_name = $redis.hget('vmpooler__config__template', pool['name'])
$logger.log('s', "[*] [#{pool['name']}] template updated from #{old_template_name} to #{new_template_name}") pool['template'] = new_template_name
# Remove all ready and pending VMs so new instances are created from the new template $redis.hset('vmpooler__template', pool['name'], new_template_name)
if $redis.scard("vmpooler__ready__#{pool['name']}") > 0 $logger.log('s', "[*] [#{pool['name']}] template updated from #{old_template_name} to #{new_template_name}")
$logger.log('s', "[*] [#{pool['name']}] removing ready and pending instances") # Remove all ready and pending VMs so new instances are created from the new template
$redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm| if $redis.scard("vmpooler__ready__#{pool['name']}") > 0
$redis.smove("vmpooler__ready__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm) $logger.log('s', "[*] [#{pool['name']}] removing ready instances")
$redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
move_vm_queue(pool['name'], vm, 'ready', 'completed')
end
end end
end if $redis.scard("vmpooler__pending__#{pool['name']}") > 0
if $redis.scard("vmpooler__pending__#{pool['name']}") > 0 $logger.log('s', "[*] [#{pool['name']}] removing pending instances")
$redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm| $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
$redis.smove("vmpooler__pending__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm) move_vm_queue(pool['name'], vm, 'pending', 'completed')
end
end end
# Prepare template for deployment
$logger.log('s', "[*] [#{pool['name']}] creating template deltas")
provider.create_template_delta_disks(pool)
$logger.log('s', "[*] [#{pool['name']}] template deltas have been created")
ensure
$redis.hdel('vmpooler__config__updating', pool['name'])
end end
# Prepare template for deployment
$logger.log('s', "[*] [#{pool['name']}] creating template deltas")
provider.create_template_delta_disks(pool)
$logger.log('s', "[*] [#{pool['name']}] template deltas have been created")
ensure
$redis.hdel('vmpooler__config__updating', pool['name'])
end end
end end
end end
def remove_excess_vms(pool, provider, ready, total) def remove_excess_vms(pool, provider, ready, total)
unless ready.nil? unless total == 0
if total > pool['size'] mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
difference = ready - pool['size'] mutex.synchronize do
difference.times do if total > pool['size']
next_vm = $redis.spop("vmpooler__ready__#{pool['name']}") difference = ready - pool['size']
move_vm_queue(pool['name'], next_vm, 'ready', 'completed') difference.times do
end next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
if total > ready move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
$redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm| end
move_vm_queue(pool['name'], vm, 'pending', 'completed') if total > ready
$redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
move_vm_queue(pool['name'], vm, 'pending', 'completed')
end
end end
end end
end end