(POOLER-47) Send clone errors up

This commit updates vmpooler to ensure clone errors, and other pool manager errors are raised to the parent method. Without this change vmpooler gets stuck after a connection fails during clone operations and will not attempt to clone again.

lib/vmpooler/pool_manager.rb

@@ -271,6 +271,7 @@ module Vmpooler
           rescue => err
             $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}")
             $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname'])
+            raise
           end
 
           $redis.decr('vmpooler__tasks__clone')
@@ -278,6 +279,7 @@ module Vmpooler
           $metrics.timing("clone.#{vm['template']}", finish)
         rescue => err
           $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}")
+          raise
         end
       end
     end
@@ -579,7 +581,8 @@ module Vmpooler
 
           inventory[vm['name']] = 1
         end
-      rescue
+      rescue => err
+        $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while inspecting inventory: #{err}")
       end
 
       # RUNNING
@@ -588,7 +591,8 @@ module Vmpooler
           begin
             vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
             check_running_vm(vm, pool['name'], vm_lifetime, vsphere)
-          rescue
+          rescue => err
+            $logger.log('d', "[!] [#{pool['name']}] _check_pool with an error while evaluating running VMs: #{err}")
           end
         end
       end
@@ -598,7 +602,8 @@ module Vmpooler
         if inventory[vm]
           begin
             check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere)
-          rescue
+          rescue => err
+            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}")
           end
         end
       end
@@ -609,7 +614,8 @@ module Vmpooler
         if inventory[vm]
           begin
             check_pending_vm(vm, pool['name'], pool_timeout, vsphere)
-          rescue
+          rescue => err
+            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating pending VMs: #{err}")
           end
         else
           fail_pending_vm(vm, pool['name'], pool_timeout, false)
@@ -621,11 +627,12 @@ module Vmpooler
         if inventory[vm]
           begin
             destroy_vm(vm, pool['name'], vsphere)
-          rescue
+          rescue => err
             $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
             $redis.srem("vmpooler__completed__#{pool['name']}", vm)
             $redis.hdel("vmpooler__active__#{pool['name']}", vm)
             $redis.del("vmpooler__vm__#{vm}")
+            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating completed VMs: #{err}")
           end
         else
           $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
@@ -636,6 +643,7 @@ module Vmpooler
       end
 
       # DISCOVERED
+      begin
         $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm|
           %w(pending ready running completed).each do |queue|
             if $redis.sismember("vmpooler__#{queue}__#{pool['name']}", vm)
@@ -648,6 +656,9 @@ module Vmpooler
             $redis.smove("vmpooler__discovered__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
           end
         end
+      rescue => err
+        $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating discovered VMs: #{err}")
+      end
 
       # MIGRATIONS
       $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm|
@@ -694,10 +705,14 @@ module Vmpooler
             rescue => err
               $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
               $redis.decr('vmpooler__tasks__clone')
+              raise
             end
           end
         end
       end
+    rescue => err
+      $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error: #{err}")
+      raise
     end
 
     def execute!