(POOLER-140) Ensure a VM is alive at checkout

This commit duplicates the vm_ready? check to the API layer to allow for API to validate that a VM is alive at checkout. Without this change API relies upon the checks in pool_manager validating pools. This change should allow for additional insight into whether a machine is in a ready state and resopnding at checkout time.

lib/vmpooler/api/helpers.rb

@@ -470,6 +470,29 @@ module Vmpooler
       rescue
         false
       end
+
+      def open_socket(host, domain = nil, timeout = 1, port = 22, &_block)
+        Timeout.timeout(timeout) do
+          target_host = host
+          target_host = "#{host}.#{domain}" if domain
+          sock = TCPSocket.new target_host, port
+          begin
+            yield sock if block_given?
+          ensure
+            sock.close
+          end
+        end
+      end
+
+      def vm_ready?(vm_name, domain = nil)
+        begin
+          open_socket(vm_name, domain)
+        rescue => _err
+          return false
+        end
+
+        true
+      end
     end
   end
 end

lib/vmpooler/api/v1.rb

@@ -68,10 +68,17 @@ module Vmpooler
       end
 
       template_backends.each do |template_backend|
-        vm = backend.smembers("vmpooler__ready__#{template_backend}")[-1]
-        if vm
-          backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
-          return [vm, template_backend, template]
+        vms = backend.smembers("vmpooler__ready__#{template_backend}")
+        next if vms.empty?
+        vms.reverse.each do |vm|
+          ready = vm_ready?(vm, config[:domain])
+          if ready
+            backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+            return [vm, template_backend, template]
+          else
+            backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
+            metrics.increment("checkout.nonresponsive.#{template_backend}")
+          end
         end
       end
       [nil, nil, nil]