Add mutex object for managing pool configuration updates

This commit adds a config_changes object for ensuring that pool configuration changes are synchronized across multiple running threads, removing the possibility of two threads attempting to update something at once, without relying on redis data. Without this change this is managed crudely by specifying in redis that a configuration update is taking place. This redis data is left so the REPOPULATE section of _check_pool can still identify when a configuration change is in progress, and prevent a pool from repopulating at that time.

lib/vmpooler/pool_manager.rb

@@ -21,6 +21,9 @@ module Vmpooler
 
       # Our thread-tracker object
       $threads = {}
+
+      # Pool mutex
+      @reconfigure_pool = {}
     end
 
     def config
@@ -560,6 +563,8 @@ module Vmpooler
         # Ensure templates are evaluated for delta disk creation on startup
         return if $redis.hget('vmpooler__config__updating', pool['name'])
         unless $redis.hget('vmpooler__template__prepared', pool['name'])
+          mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
+          mutex.synchronize do
             begin
               $redis.hset('vmpooler__config__updating', pool['name'], 1)
               provider.create_template_delta_disks(pool)
@@ -570,16 +575,18 @@ module Vmpooler
           end
         end
       end
+    end
 
     def update_pool_template(pool, provider)
       $redis.hset('vmpooler__template', pool['name'], pool['template']) unless $redis.hget('vmpooler__template', pool['name'])
       return unless $redis.hget('vmpooler__config__template', pool['name'])
       unless $redis.hget('vmpooler__config__template', pool['name']) == $redis.hget('vmpooler__template', pool['name'])
-        # Ensure we are only updating a template once
         return if $redis.hget('vmpooler__config__updating', pool['name'])
+        # Ensure we are only updating a template once
+        mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
+        mutex.synchronize do
           begin
             $redis.hset('vmpooler__config__updating', pool['name'], 1)
-
             old_template_name = $redis.hget('vmpooler__template', pool['name'])
             new_template_name = $redis.hget('vmpooler__config__template', pool['name'])
             pool['template'] = new_template_name
@@ -587,14 +594,15 @@ module Vmpooler
             $logger.log('s', "[*] [#{pool['name']}] template updated from #{old_template_name} to #{new_template_name}")
             # Remove all ready and pending VMs so new instances are created from the new template
             if $redis.scard("vmpooler__ready__#{pool['name']}") > 0
-            $logger.log('s', "[*] [#{pool['name']}] removing ready and pending instances")
+              $logger.log('s', "[*] [#{pool['name']}] removing ready instances")
               $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
-              $redis.smove("vmpooler__ready__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+                move_vm_queue(pool['name'], vm, 'ready', 'completed')
               end
             end
             if $redis.scard("vmpooler__pending__#{pool['name']}") > 0
+              $logger.log('s', "[*] [#{pool['name']}] removing pending instances")
               $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
-              $redis.smove("vmpooler__pending__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+                move_vm_queue(pool['name'], vm, 'pending', 'completed')
               end
             end
             # Prepare template for deployment
@@ -606,9 +614,12 @@ module Vmpooler
           end
         end
       end
+    end
 
     def remove_excess_vms(pool, provider, ready, total)
-      unless ready.nil?
+      unless total == 0
+        mutex = @reconfigure_pool[pool['name']] || @reconfigure_pool[pool['name']] = Mutex.new
+        mutex.synchronize do
           if total > pool['size']
             difference = ready - pool['size']
             difference.times do
@@ -623,6 +634,7 @@ module Vmpooler
           end
         end
       end
+    end
 
     def _check_pool(pool, provider)
       pool_check_response = {