(POOLER-133) Identify when a ready VM has failed

This commit fixes checking of a VM that has already been identified as ready. Without this change a ready VM that has failed will be identified as having failed, but will not successfully be removed from the ready queue. Additionally, the default vm_checktime value has been reduced from 15 to 1 to ensure that ready VMs are checked within one minute of the time they have reached the ready state by default.

Lastly, the docker-compose files are updated to specify that the redis
instance used is a local redis instance.

lib/vmpooler.rb

@@ -49,7 +49,7 @@ module Vmpooler
     # Set some configuration defaults
     parsed_config[:config]['task_limit'] = ENV['TASK_LIMIT'] || parsed_config[:config]['task_limit'] || 10
     parsed_config[:config]['migration_limit'] = ENV['MIGRATION_LIMIT'] if ENV['MIGRATION_LIMIT']
-    parsed_config[:config]['vm_checktime'] = ENV['VM_CHECKTIME'] || parsed_config[:config]['vm_checktime'] || 15
+    parsed_config[:config]['vm_checktime'] = ENV['VM_CHECKTIME'] || parsed_config[:config]['vm_checktime'] || 1
     parsed_config[:config]['vm_lifetime'] = ENV['VM_LIFETIME'] || parsed_config[:config]['vm_lifetime']  || 24
     parsed_config[:config]['prefix'] = ENV['VM_PREFIX'] || parsed_config[:config]['prefix'] || ''
 
@@ -100,6 +100,9 @@ module Vmpooler
       parsed_config[:pools] = load_pools_from_redis(redis)
     end
 
+    # Create an index of pools by title
+    parsed_config[:pool_index] = pool_index(parsed_config[:pools])
+
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
       if pool['alias']
@@ -161,4 +164,14 @@ module Vmpooler
   def self.pools(conf)
     conf[:pools]
   end
+
+  def self.pool_index(pools)
+    pools_hash = {}
+    index = 0
+    for pool in pools
+      pools_hash[pool['name']] = index
+      index += 1
+    end
+    pools_hash
+  end
 end

lib/vmpooler/pool_manager.rb

@@ -134,18 +134,18 @@ module Vmpooler
       move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
     end
 
-    def check_ready_vm(vm, pool, ttl, provider)
+    def check_ready_vm(vm, pool_name, ttl, provider)
       Thread.new do
         begin
-          _check_ready_vm(vm, pool, ttl, provider)
+          _check_ready_vm(vm, pool_name, ttl, provider)
         rescue => err
-          $logger.log('s', "[!] [#{pool['name']}] '#{vm}' failed while checking a ready vm : #{err}")
+          $logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{err}")
           raise
         end
       end
     end
 
-    def _check_ready_vm(vm, pool, ttl, provider)
+    def _check_ready_vm(vm, pool_name, ttl, provider)
       # Periodically check that the VM is available
       mutex = vm_mutex(vm)
       return if mutex.locked?
@@ -158,21 +158,22 @@ module Vmpooler
         if ttl > 0
           # host['boottime'] may be nil if host is not powered on
           if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
-            $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
+            $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
 
-            $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
+            $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
             return
           end
         end
 
-        return if has_mismatched_hostname?(vm, pool, provider)
+        return if has_mismatched_hostname?(vm, pool_name, provider)
 
-        vm_still_ready?(pool['name'], vm, provider)
+        vm_still_ready?(pool_name, vm, provider)
       end
     end
 
-    def has_mismatched_hostname?(vm, pool, provider)
-      check_hostname = pool['check_hostname_for_mismatch']
+    def has_mismatched_hostname?(vm, pool_name, provider)
+      pool_config = $config[:pools][$config[:pool_index][pool_name]]
+      check_hostname = pool_config['check_hostname_for_mismatch']
       check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
       return if check_hostname == false
 
@@ -187,15 +188,15 @@ module Vmpooler
       end
 
       # Check if the hostname has magically changed from underneath Pooler
-      vm_hash = provider.get_vm(pool['name'], vm)
+      vm_hash = provider.get_vm(pool_name, vm)
       return unless vm_hash.is_a? Hash
       hostname = vm_hash['hostname']
 
       return if hostname.nil?
       return if hostname.empty?
       return if hostname == vm
-      $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
-      $logger.log('d', "[!] [#{pool['name']}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
+      $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
+      $logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
       return true
     end