Merge 1e0b936d12 into 1fa1909986

.gitignore

@@ -1,3 +1,7 @@
 .ruby-version
 Gemfile.lock
+Gemfile.local
 vendor
+vmpooler.yaml
+.bundle
+coverage

Gemfile

@@ -23,3 +23,13 @@ group :test do
   gem 'simplecov', '>= 0.11.2'
   gem 'yarjuf', '>= 2.0'
 end
+
+# Evaluate Gemfile.local if it exists
+if File.exists? "#{__FILE__}.local"
+  eval(File.read("#{__FILE__}.local"), binding)
+end
+
+# Evaluate ~/.gemfile if it exists
+if File.exists?(File.join(Dir.home, '.gemfile'))
+  eval(File.read(File.join(Dir.home, '.gemfile')), binding)
+end

lib/vmpooler.rb

@@ -12,7 +12,7 @@ module Vmpooler
   require 'yaml'
   require 'set'
 
-  %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib|
+  %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd backingservice ).each do |lib|
     begin
       require "vmpooler/#{lib}"
     rescue LoadError
@@ -21,7 +21,7 @@ module Vmpooler
   end
 
   def self.config(filepath='vmpooler.yaml')
-    parsed_config = {}
+  parsed_config = {}
 
     if ENV['VMPOOLER_CONFIG']
       # Load configuration from ENV

lib/vmpooler/api/helpers.rb

@@ -56,6 +56,8 @@ module Vmpooler
 
       def authenticate(auth, username_str, password_str)
         case auth['provider']
+          when 'dummy'
+            return (username_str != password_str)
           when 'ldap'
             require 'rubygems'
             require 'net/ldap'

lib/vmpooler/backingservice.rb

@@ -0,0 +1,8 @@
+# TODO remove dummy for commit history
+%w( base vsphere dummy ).each do |lib|
+  begin
+    require "vmpooler/backingservice/#{lib}"
+  rescue LoadError
+    require File.expand_path(File.join(File.dirname(__FILE__), 'backingservice', lib))
+  end
+end

lib/vmpooler/backingservice/base.rb

@@ -0,0 +1,111 @@
+module Vmpooler
+  class PoolManager
+    class BackingService
+      class Base
+        # These defs must be overidden in child classes
+
+        def initialize(options)
+          @options = options
+        end
+
+        # returns
+        #   [String] Name of the backing service
+        def name
+          'base'
+        end
+
+        #def validate_config(config)
+        #  false
+        #end
+
+        # inputs
+        #  pool : hashtable from config file
+        # returns
+        #   hashtable
+        #     name : name of the device   <---- TODO is this all?
+        def vms_in_pool(pool)
+          fail "#{self.class.name} does not implement vms_in_pool"
+        end
+
+        # inputs
+        #   vm_name: string
+        # returns
+        #    [String] hostname   = Name of the host computer running the vm.  If this is not a Virtual Machine, it returns the vm_name
+        def get_vm_host(vm_name)
+          fail "#{self.class.name} does not implement get_vm_host"
+        end
+
+        # inputs
+        #   vm_name: string
+        # returns
+        #    [String] hostname   = Name of the most appropriate host computer to run this VM.  Useful for load balancing VMs in a cluster
+        #                          If this is not a Virtual Machine, it returns the vm_name
+        def find_least_used_compatible_host(vm_name)
+          fail "#{self.class.name} does not implement find_least_used_compatible_host"
+        end
+
+        # inputs
+        #   vm_name: string
+        #   dest_host_name: string (Name of the host to migrate `vm_name` to)
+        # returns
+        #    [Boolean] Returns true on success or false on failure
+        def migrate_vm_to_host(vm_name, dest_host_name)
+          fail "#{self.class.name} does not implement migrate_vm_to_host"
+        end
+
+        # inputs
+        #   vm_name: string
+        # returns
+        #   nil if it doesn't exist
+        #   Hastable of the VM
+        #    [String] name       = Name of the VM
+        #    [String] hostname   = Name reported by Vmware tools (host.summary.guest.hostName)
+        #    [String] template   = This is the name of template exposed by the API.  It must _match_ the poolname
+        #    [String] poolname   = Name of the pool the VM is located
+        #    [Time]   boottime   = Time when the VM was created/booted
+        #    [String] powerstate = Current power state of a VM.  Valid values (as per vCenter API)
+        #                            - 'PoweredOn','PoweredOff'
+        def get_vm(vm_name)
+          fail "#{self.class.name} does not implement get_vm"
+        end
+
+        # inputs
+        #   pool : hashtable from config file
+        #   new_vmname : string      Name the new VM should use
+        # returns
+        #   Hashtable of the VM as per get_vm
+        def create_vm(pool,new_vmname)
+          fail "#{self.class.name} does not implement create_vm"
+        end
+
+        # inputs
+        #   vm_name: string
+        #   pool: string
+        # returns
+        #   boolean : true if success, false on error
+        def destroy_vm(vm_name,pool)
+          fail "#{self.class.name} does not implement destroy_vm"
+        end
+
+        # inputs
+        #    vm  : string
+        #    pool: string
+        # timeout: int (Seconds)
+        # returns
+        #   result: boolean
+        def is_vm_ready?(vm,pool,timeout)
+          fail "#{self.class.name} does not implement is_vm_ready?"
+        end
+
+        # inputs
+        #    vm : string
+        # returns
+        #   result: boolean
+        def vm_exists?(vm)
+          !get_vm(vm).nil?
+        end
+
+      end
+    end
+  end
+end

lib/vmpooler/backingservice/dummy.rb

@@ -0,0 +1,238 @@
+require 'yaml'
+
+module Vmpooler
+  class PoolManager
+    class BackingService
+      class Dummy < Vmpooler::PoolManager::BackingService::Base
+
+        # Fake VM backing service for testing, with initial configuration set in a simple text YAML filename
+        #   or via YAML in the config file
+        def initialize(options)
+          super(options)
+
+          dummyfilename = options['filename']
+
+          # TODO Accessing @dummylist is not thread safe :-(  Mutexes?
+
+          # This initial_state option is only intended to be used by spec tests
+          @dummylist = options['initial_state'].nil? ? {} : options['initial_state']
+
+          if !dummyfilename.nil? && File.exists?(dummyfilename)
+            @dummylist ||= YAML.load_file(dummyfilename)
+          end
+        end
+
+        def vms_in_pool(pool)
+          get_pool_object(pool['name']).each do |vm|
+            vm
+          end
+        end
+
+        def name
+          'dummy'
+        end
+
+        def get_vm(vm)
+          dummy = get_dummy_vm(vm)
+          return nil if dummy.nil?
+
+          # Randomly power off the VM
+          unless dummy['powerstate'] != 'PoweredOn' || @options['getvm_poweroff_percent'].nil?
+            if 1 + rand(100) <= @options['getvm_poweroff_percent']
+              dummy['powerstate'] = 'PoweredOff'
+              $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off")
+            end
+          end
+
+          # Randomly rename the host
+          unless dummy['hostname'] != dummy['name'] || @options['getvm_rename_percent'].nil?
+            if 1 + rand(100) <= @options['getvm_rename_percent']
+              dummy['hostname'] = 'DUMMY' + dummy['name']
+              $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed")
+            end
+          end
+
+          obj = {}
+          obj['name'] = dummy['name']
+          obj['hostname'] = dummy['hostname']
+          obj['boottime'] = dummy['boottime']
+          obj['template'] = dummy['template']
+          obj['poolname'] = dummy['poolname']
+          obj['powerstate'] = dummy['powerstate']
+
+          obj
+        end
+
+        def find_least_used_compatible_host(vm_name)
+          current_vm = get_dummy_vm(vm_name)
+
+          # Unless migratevm_couldmove_percent is specified, don't migrate
+          return current_vm['vm_host'] if @options['migratevm_couldmove_percent'].nil?
+
+          # Only migrate if migratevm_couldmove_percent is met
+          return current_vm['vm_host'] if 1 + rand(100) > @options['migratevm_couldmove_percent']
+          
+          # Simulate a 10 node cluster and randomly pick a different one
+          new_host = "HOST" + (1 + rand(10)).to_s while new_host == current_vm['vm_host']
+
+          new_host
+        end
+
+        def get_vm_host(vm_name)
+          current_vm = get_dummy_vm(vm_name)
+
+          current_vm.nil? ? fail("VM #{vm_name} does not exist") : current_vm['vm_host']
+        end
+
+        def migrate_vm_to_host(vm_name, dest_host_name)
+          current_vm = get_dummy_vm(vm_name)
+
+          # Inject migration delay
+          unless @options['migratevm_max_time'].nil?
+            migrate_time = 1 + rand(@options['migratevm_max_time'])
+            sleep(migrate_time)
+          end
+
+          # Inject clone failure
+          unless @options['migratevm_fail_percent'].nil?
+            fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['migratevm_fail_percent']
+          end
+
+          current_vm['vm_host'] = dest_host_name
+
+          true
+        end
+
+        def is_vm_ready?(vm,pool,timeout)
+          host = get_dummy_vm(vm)
+          if !host then return false end
+          if host['poolname'] != pool then return false end
+          if host['ready'] then return true end
+
+          Timeout.timeout(timeout) do
+            while host['dummy_state'] != 'RUNNING'
+              sleep(2)
+              host = get_dummy_vm(vm)
+            end
+          end
+
+          # Simulate how long it takes from a VM being powered on until
+          # it's ready to receive a connection
+          sleep(2)
+
+          unless @options['vmready_fail_percent'].nil?
+            fail "Dummy Failure for vmready_fail_percent" if 1 + rand(100) <= @options['vmready_fail_percent']
+          end
+
+          host['ready'] = true
+          true
+        end
+
+        def create_vm(pool,dummy_hostname)
+          template_name = pool['template']
+          pool_name = pool['name']
+
+          vm = {}
+          vm['name'] = dummy_hostname
+          vm['hostname'] = dummy_hostname
+          vm['domain']  = 'dummy.local'
+          # 'vm_template' is the name of the template to use to clone the VM from  <----- Do we need this?!?!?
+          vm['vm_template'] = template_name
+          # 'template' is the Template name in VM Pooler API, in our case that's the poolname.
+          vm['template'] = pool_name
+          vm['poolname'] = pool_name
+          vm['ready'] = false
+          vm['boottime'] = Time.now
+          vm['powerstate'] = 'PoweredOn'
+          vm['vm_host'] = 'HOST1'
+          vm['dummy_state'] = 'UNKNOWN'
+          get_pool_object(pool_name)
+          @dummylist['pool'][pool_name] << vm
+
+          $logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'")
+          begin
+            # Inject clone time delay
+            unless @options['createvm_max_time'].nil?
+              vm['dummy_state'] = 'CLONING'
+              clone_time = 1 + rand(@options['createvm_max_time'])
+              sleep(clone_time)
+            end
+
+            # Inject clone failure
+            unless @options['createvm_fail_percent'].nil?
+              fail "Dummy Failure for createvm_fail_percent" if 1 + rand(100) <= @options['createvm_fail_percent']
+            end
+
+            # Assert the VM is ready for use
+            vm['dummy_state'] = 'RUNNING'
+          rescue => err
+            remove_dummy_vm(dummy_hostname,pool_name)
+            raise
+          end
+
+          get_vm(dummy_hostname)
+        end
+
+        def destroy_vm(vm_name,pool)
+          vm = get_dummy_vm(vm_name)
+          if !vm then return false end
+          if vm['poolname'] != pool then return false end
+
+          # Shutdown down the VM if it's poweredOn
+          if vm['powerstate'] = 'PoweredOn'
+            $logger.log('d', "[ ] [#{pool}] '#{vm_name}' is being shut down")
+
+            # Inject shutdown delay time
+            unless @options['destroyvm_max_shutdown_time'].nil?
+              shutdown_time = 1 + rand(@options['destroyvm_max_shutdown_time'])
+              sleep(shutdown_time)
+            end
+
+            vm['powerstate'] = 'PoweredOff'
+          end
+
+          # Inject destroy VM delay
+          unless @options['destroyvm_max_time'].nil?
+            destroy_time = 1 + rand(@options['destroyvm_max_time'])
+            sleep(destroy_time)
+          end
+
+          # Inject destroy VM failure
+          unless @options['destroyvm_fail_percent'].nil?
+            fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['destroyvm_fail_percent']
+          end
+
+          # 'Destroy' the VM
+          remove_dummy_vm(vm_name,pool)
+
+          true
+        end
+
+        private
+        def remove_dummy_vm(vm_name,pool)
+          return if @dummylist['pool'][pool].nil?
+          new_poollist = @dummylist['pool'][pool].delete_if { |vm| vm['name'] == vm_name }
+          @dummylist['pool'][pool] = new_poollist
+        end
+
+        # Get's the pool config safely from the in-memory hashtable
+        def get_pool_object(pool_name)
+          @dummylist['pool'] = {} if @dummylist['pool'].nil?
+          @dummylist['pool'][pool_name] = [] if @dummylist['pool'][pool_name].nil?
+          
+          return @dummylist['pool'][pool_name]
+        end
+
+        def get_dummy_vm(vm)
+          @dummylist['pool'].keys.each do |poolname|
+            @dummylist['pool'][poolname].each do |poolvm|
+              return poolvm if poolvm['name'] == vm
+            end
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

lib/vmpooler/backingservice/vsphere.rb

@@ -0,0 +1,622 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class PoolManager
+    class BackingService
+      class Vsphere < Vmpooler::PoolManager::BackingService::Base
+        #--------------- Public methods
+
+        def initialize(options)          
+          $credentials = options['credentials']
+          $metrics = options['metrics']
+        end
+
+        def devices_in_pool(pool)
+          base = find_folder(pool['folder'])
+
+          base.childEntity.each do |vm|
+            vm
+          end            
+        end
+
+# def destroy_vm()
+#     # Destroy a VM
+#     def _destroy_vm(vm, pool, vsphere)
+#       $redis.srem('vmpooler__completed__' + pool, vm)
+#       $redis.hdel('vmpooler__active__' + pool, vm)
+#       $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
+
+#       # Auto-expire metadata key
+#       $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+
+# # TODO This is all vSphere specific
+
+#       host = vsphere.find_vm(vm)
+
+#       if host
+#         start = Time.now
+
+#         if
+#           (host.runtime) &&
+#           (host.runtime.powerState) &&
+#           (host.runtime.powerState == 'poweredOn')
+
+#           $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
+#           host.PowerOffVM_Task.wait_for_completion
+#         end
+
+#         host.Destroy_Task.wait_for_completion
+#         finish = '%.2f' % (Time.now - start)
+
+#         $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
+#         $metrics.timing("destroy.#{pool}", finish)
+#       end
+#     end
+# end
+
+        def create_device(pool)
+'12345'  
+            # clone_vm(
+            #   pool['template'],
+            #   pool['folder'],
+            #   pool['datastore'],
+            #   pool['clone_target'],
+            #   vsphere
+            # )
+
+        #   Thread.new do
+        #     begin
+        #         vm = {}
+
+        #         if template =~ /\//
+        #           templatefolders = template.split('/')
+        #           vm['template'] = templatefolders.pop
+        #         end
+
+        #         if templatefolders
+        #           vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template'])
+        #         else
+        #           fail 'Please provide a full path to the template'
+        #         end
+
+        #         if vm['template'].length == 0
+        #           fail "Unable to find template '#{vm['template']}'!"
+        #         end
+
+        #         # Generate a randomized hostname
+        #         o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
+        #         vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
+
+        #         # Add VM to Redis inventory ('pending' pool)
+        #         $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname'])
+        #         $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now)
+        #         $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template'])
+
+        #         # Annotate with creation time, origin template, etc.
+        #         # Add extraconfig options that can be queried by vmtools
+        #         configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
+        #           annotation: JSON.pretty_generate(
+        #               name: vm['hostname'],
+        #               created_by: $config[:vsphere]['username'],
+        #               base_template: vm['template'],
+        #               creation_timestamp: Time.now.utc
+        #           ),
+        #           extraConfig: [
+        #               { key: 'guestinfo.hostname',
+        #                 value: vm['hostname']
+        #               }
+        #           ]
+        #         )
+
+        #         # Choose a clone target
+        #         if target
+        #           $clone_target = vsphere.find_least_used_host(target)
+        #         elsif $config[:config]['clone_target']
+        #           $clone_target = vsphere.find_least_used_host($config[:config]['clone_target'])
+        #         end
+
+        #         # Put the VM in the specified folder and resource pool
+        #         relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+        #           datastore: vsphere.find_datastore(datastore),
+        #           host: $clone_target,
+        #           diskMoveType: :moveChildMostDiskBacking
+        #         )
+
+        #         # Create a clone spec
+        #         spec = RbVmomi::VIM.VirtualMachineCloneSpec(
+        #           location: relocateSpec,
+        #           config: configSpec,
+        #           powerOn: true,
+        #           template: false
+        #         )
+
+        #         # Clone the VM
+        #         $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
+
+        #         begin
+        #           start = Time.now
+        #           vm[vm['template']].CloneVM_Task(
+        #             folder: vsphere.find_folder(folder),
+        #             name: vm['hostname'],
+        #             spec: spec
+        #           ).wait_for_completion
+        #           finish = '%.2f' % (Time.now - start)
+
+        #           $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish)
+        #           $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish)
+
+        #           $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
+        #         rescue => err
+        #           $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}")
+        #           $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname'])
+        #           raise
+        #         end
+
+        #         $redis.decr('vmpooler__tasks__clone')
+
+        #         $metrics.timing("clone.#{vm['template']}", finish)
+        #       rescue => err
+        #         $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}")
+        #         raise
+        #       end
+        #     end
+
+
+        end
+
+#**** When getting the VM details
+      # if (host.summary) &&
+      #     (host.summary.guest) &&
+      #     (host.summary.guest.hostName) &&
+      #     (host.summary.guest.hostName == vm)
+#
+
+        def is_vm_ready?(vm,pool,timeout)
+          fail "!!!!"
+
+
+    # def open_socket(host, domain=nil, timeout=5, port=22, &block)
+    #   Timeout.timeout(timeout) do
+    #     target_host = host
+    #     target_host = "#{host}.#{domain}" if domain
+    #     sock = TCPSocket.new target_host, port
+    #     begin
+    #       yield sock if block_given?
+    #     ensure
+    #       sock.close
+    #     end
+    #   end
+    # end
+
+    # def _check_pending_vm(vm, pool, timeout, vsphere)
+    #   host = vsphere.find_vm(vm)
+
+    #   if ! host
+    #     fail_pending_vm(vm, pool, timeout, false)
+    #     return
+    #   end
+    #   open_socket vm
+    #   move_pending_vm_to_ready(vm, pool, host)
+    # rescue
+    #   fail_pending_vm(vm, pool, timeout)
+    # end
+
+
+        end
+
+
+
+        #--------------- Private methods
+        private
+        ADAPTER_TYPE = 'lsiLogic'
+        DISK_TYPE = 'thin'
+        DISK_MODE = 'persistent'
+
+        def ensure_connected(connection, credentials)
+          connection.serviceInstance.CurrentTime
+        rescue
+          $metrics.increment("connect.open")
+          connect_to_vsphere $credentials
+        end
+
+        def connect_to_vsphere(credentials)
+          @connection = RbVmomi::VIM.connect host: credentials['server'],
+                                            user: credentials['username'],
+                                            password: credentials['password'],
+                                            insecure: credentials['insecure'] || true
+        end
+
+        def add_disk(vm, size, datastore)
+          ensure_connected @connection, $credentials
+
+          return false unless size.to_i > 0
+
+          vmdk_datastore = find_datastore(datastore)
+          vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk"
+
+          controller = find_disk_controller(vm)
+
+          vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec(
+            capacityKb: size.to_i * 1024 * 1024,
+            adapterType: ADAPTER_TYPE,
+            diskType: DISK_TYPE
+          )
+
+          vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo(
+            datastore: vmdk_datastore,
+            diskMode: DISK_MODE,
+            fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}"
+          )
+
+          device = RbVmomi::VIM::VirtualDisk(
+            backing: vmdk_backing,
+            capacityInKB: size.to_i * 1024 * 1024,
+            controllerKey: controller.key,
+            key: -1,
+            unitNumber: find_disk_unit_number(vm, controller)
+          )
+
+          device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec(
+            device: device,
+            operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add')
+          )
+
+          vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec(
+            deviceChange: [device_config_spec]
+          )
+
+          @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
+            datacenter: @connection.serviceInstance.find_datacenter,
+            name: "[#{vmdk_datastore.name}] #{vmdk_file_name}",
+            spec: vmdk_spec
+          ).wait_for_completion
+
+          vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion
+
+          true
+        end
+
+        def find_datastore(datastorename)
+          ensure_connected @connection, $credentials
+
+          datacenter = @connection.serviceInstance.find_datacenter
+          datacenter.find_datastore(datastorename)
+        end
+
+        def find_device(vm, deviceName)
+          ensure_connected @connection, $credentials
+
+          vm.config.hardware.device.each do |device|
+            return device if device.deviceInfo.label == deviceName
+          end
+
+          nil
+        end
+
+        def find_disk_controller(vm)
+          ensure_connected @connection, $credentials
+
+          devices = find_disk_devices(vm)
+
+          devices.keys.sort.each do |device|
+            if devices[device]['children'].length < 15
+              return find_device(vm, devices[device]['device'].deviceInfo.label)
+            end
+          end
+
+          nil
+        end
+
+        def find_disk_devices(vm)
+          ensure_connected @connection, $credentials
+
+          devices = {}
+
+          vm.config.hardware.device.each do |device|
+            if device.is_a? RbVmomi::VIM::VirtualSCSIController
+              if devices[device.controllerKey].nil?
+                devices[device.key] = {}
+                devices[device.key]['children'] = []
+              end
+
+              devices[device.key]['device'] = device
+            end
+
+            if device.is_a? RbVmomi::VIM::VirtualDisk
+              if devices[device.controllerKey].nil?
+                devices[device.controllerKey] = {}
+                devices[device.controllerKey]['children'] = []
+              end
+
+              devices[device.controllerKey]['children'].push(device)
+            end
+          end
+
+          devices
+        end
+
+        def find_disk_unit_number(vm, controller)
+          ensure_connected @connection, $credentials
+
+          used_unit_numbers = []
+          available_unit_numbers = []
+
+          devices = find_disk_devices(vm)
+
+          devices.keys.sort.each do |c|
+            next unless controller.key == devices[c]['device'].key
+            used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber)
+            devices[c]['children'].each do |disk|
+              used_unit_numbers.push(disk.unitNumber)
+            end
+          end
+
+          (0..15).each do |scsi_id|
+            if used_unit_numbers.grep(scsi_id).length <= 0
+              available_unit_numbers.push(scsi_id)
+            end
+          end
+
+          available_unit_numbers.sort[0]
+        end
+
+        def find_folder(foldername)
+          ensure_connected @connection, $credentials
+
+          datacenter = @connection.serviceInstance.find_datacenter
+          base = datacenter.vmFolder
+          folders = foldername.split('/')
+          folders.each do |folder|
+            case base
+              when RbVmomi::VIM::Folder
+                base = base.childEntity.find { |f| f.name == folder }
+              else
+                abort "Unexpected object type encountered (#{base.class}) while finding folder"
+            end
+          end
+
+          base
+        end
+
+        # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference
+        # Params:
+        # +model+:: CPU arch version to match on
+        # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
+        def get_host_utilization(host, model=nil, limit=90)
+          if model
+            return nil unless host_has_cpu_model? host, model
+          end
+          return nil if host.runtime.inMaintenanceMode
+          return nil unless host.overallStatus == 'green'
+
+          cpu_utilization = cpu_utilization_for host
+          memory_utilization = memory_utilization_for host
+
+          return nil if cpu_utilization > limit
+          return nil if memory_utilization > limit
+
+          [ cpu_utilization + memory_utilization, host ]
+        end
+
+        def host_has_cpu_model?(host, model)
+          get_host_cpu_arch_version(host) == model
+        end
+
+        def get_host_cpu_arch_version(host)
+          cpu_model = host.hardware.cpuPkg[0].description
+          cpu_model_parts = cpu_model.split()
+          arch_version = cpu_model_parts[4]
+          arch_version
+        end
+
+        def cpu_utilization_for(host)
+          cpu_usage = host.summary.quickStats.overallCpuUsage
+          cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores
+          (cpu_usage.to_f / cpu_size.to_f) * 100
+        end
+
+        def memory_utilization_for(host)
+          memory_usage = host.summary.quickStats.overallMemoryUsage
+          memory_size = host.summary.hardware.memorySize / 1024 / 1024
+          (memory_usage.to_f / memory_size.to_f) * 100
+        end
+
+        def find_least_used_host(cluster)
+          ensure_connected @connection, $credentials
+
+          cluster_object = find_cluster(cluster)
+          target_hosts = get_cluster_host_utilization(cluster_object)
+          least_used_host = target_hosts.sort[0][1]
+          least_used_host
+        end
+
+        def find_cluster(cluster)
+          datacenter = @connection.serviceInstance.find_datacenter
+          datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster }
+        end
+
+        def get_cluster_host_utilization(cluster)
+          cluster_hosts = []
+          cluster.host.each do |host|
+            host_usage = get_host_utilization(host)
+            cluster_hosts << host_usage if host_usage
+          end
+          cluster_hosts
+        end
+
+        def find_least_used_compatible_host(vm)
+          ensure_connected @connection, $credentials
+
+          source_host = vm.summary.runtime.host
+          model = get_host_cpu_arch_version(source_host)
+          cluster = source_host.parent
+          target_hosts = []
+          cluster.host.each do |host|
+            host_usage = get_host_utilization(host, model)
+            target_hosts << host_usage if host_usage
+          end
+          target_host = target_hosts.sort[0][1]
+          [target_host, target_host.name]
+        end
+
+        def find_pool(poolname)
+          ensure_connected @connection, $credentials
+
+          datacenter = @connection.serviceInstance.find_datacenter
+          base = datacenter.hostFolder
+          pools = poolname.split('/')
+          pools.each do |pool|
+            case base
+              when RbVmomi::VIM::Folder
+                base = base.childEntity.find { |f| f.name == pool }
+              when RbVmomi::VIM::ClusterComputeResource
+                base = base.resourcePool.resourcePool.find { |f| f.name == pool }
+              when RbVmomi::VIM::ResourcePool
+                base = base.resourcePool.find { |f| f.name == pool }
+              else
+                abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
+            end
+          end
+
+          base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool)
+          base
+        end
+
+        def find_snapshot(vm, snapshotname)
+          if vm.snapshot
+            get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname)
+          end
+        end
+
+        def find_vm(vmname)
+          ensure_connected @connection, $credentials
+          find_vm_light(vmname) || find_vm_heavy(vmname)[vmname]
+        end
+
+        def find_vm_light(vmname)
+          ensure_connected @connection, $credentials
+
+          @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname)
+        end
+
+        def find_vm_heavy(vmname)
+          ensure_connected @connection, $credentials
+
+          vmname = vmname.is_a?(Array) ? vmname : [vmname]
+          containerView = get_base_vm_container_from @connection
+          propertyCollector = @connection.propertyCollector
+
+          objectSet = [{
+            obj: containerView,
+            skip: true,
+            selectSet: [RbVmomi::VIM::TraversalSpec.new(
+                name: 'gettingTheVMs',
+                path: 'view',
+                skip: false,
+                type: 'ContainerView'
+            )]
+          }]
+
+          propSet = [{
+            pathSet: ['name'],
+            type: 'VirtualMachine'
+          }]
+
+          results = propertyCollector.RetrievePropertiesEx(
+            specSet: [{
+              objectSet: objectSet,
+              propSet: propSet
+            }],
+            options: { maxObjects: nil }
+          )
+
+          vms = {}
+          results.objects.each do |result|
+            name = result.propSet.first.val
+            next unless vmname.include? name
+            vms[name] = result.obj
+          end
+
+          while results.token
+            results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token)
+            results.objects.each do |result|
+              name = result.propSet.first.val
+              next unless vmname.include? name
+              vms[name] = result.obj
+            end
+          end
+
+          vms
+        end
+
+        def find_vmdks(vmname, datastore)
+          ensure_connected @connection, $credentials
+
+          disks = []
+
+          vmdk_datastore = find_datastore(datastore)
+
+          vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file'
+          vm_files.keys.each do |f|
+            vm_files[f]['layoutEx.file'].each do |l|
+              if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/)
+                disks.push(l)
+              end
+            end
+          end
+
+          disks
+        end
+
+        def get_base_vm_container_from(connection)
+          ensure_connected @connection, $credentials
+
+          viewManager = connection.serviceContent.viewManager
+          viewManager.CreateContainerView(
+            container: connection.serviceContent.rootFolder,
+            recursive: true,
+            type: ['VirtualMachine']
+          )
+        end
+
+        def get_snapshot_list(tree, snapshotname)
+          snapshot = nil
+
+          tree.each do |child|
+            if child.name == snapshotname
+              snapshot ||= child.snapshot
+            else
+              snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname)
+            end
+          end
+
+          snapshot
+        end
+
+        def migrate_vm_host(vm, host)
+          relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host)
+          vm.RelocateVM_Task(spec: relospec).wait_for_completion
+        end
+
+        def close
+          @connection.close
+        end
+
+      end
+    end
+  end
+end
+
+
+
+
+
+
+
+
+
+module Vmpooler
+  class VsphereHelper
+
+  end
+end

lib/vmpooler/logger.rb

@@ -11,9 +11,11 @@ module Vmpooler
     def log(_level, string)
       time = Time.new
       stamp = time.strftime('%Y-%m-%d %H:%M:%S')
-
       open(@file, 'a') do |f|
         f.puts "[#{stamp}] #{string}"
+        if ENV['VMPOOLER_DEBUG']
+          puts "[#{stamp}] #{string}"
+        end        
       end
     end
   end

lib/vmpooler/pool_manager.rb

@@ -12,54 +12,55 @@ module Vmpooler
       # Connect to Redis
       $redis = redis
 
-      # vSphere object
-      $vsphere = {}
+      # per pool VM Backing Services
+      $backing_services = {}
 
       # Our thread-tracker object
       $threads = {}
+
+      # WARNING DEBUG
+      #$logger.log('d',"Flushing REDIS  WARNING!!!")
+      #$redis.flushdb
     end
 
     # Check the state of a VM
-    def check_pending_vm(vm, pool, timeout, vsphere)
+    # DONE
+    def check_pending_vm(vm, pool, timeout, backingservice)
       Thread.new do
-        _check_pending_vm(vm, pool, timeout, vsphere)
-      end
-    end
-
-    def open_socket(host, domain=nil, timeout=5, port=22, &block)
-      Timeout.timeout(timeout) do
-        target_host = host
-        target_host = "#{host}.#{domain}" if domain
-        sock = TCPSocket.new target_host, port
         begin
-          yield sock if block_given?
-        ensure
-          sock.close
+          _check_pending_vm(vm, pool, timeout, backingservice)
+        rescue => err
+          $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}")
+          fail_pending_vm(vm, pool, timeout)
+          raise
         end
       end
     end
 
-    def _check_pending_vm(vm, pool, timeout, vsphere)
-      host = vsphere.find_vm(vm)
-
+    # DONE
+    def _check_pending_vm(vm, pool, timeout, backingservice)
+      host = backingservice.get_vm(vm)
       if ! host
         fail_pending_vm(vm, pool, timeout, false)
         return
       end
-      open_socket vm
-      move_pending_vm_to_ready(vm, pool, host)
-    rescue
-      fail_pending_vm(vm, pool, timeout)
+      if backingservice.is_vm_ready?(vm,pool,timeout)
+        move_pending_vm_to_ready(vm, pool, host)
+      else
+        fail "VM is not ready"
+      end
     end
 
+    # DONE
     def remove_nonexistent_vm(vm, pool)
       $redis.srem("vmpooler__pending__#{pool}", vm)
       $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
     end
 
+    # DONE
     def fail_pending_vm(vm, pool, timeout, exists=true)
       clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone')
-      return if ! clone_stamp
+      return true if ! clone_stamp
 
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
       if time_since_clone > timeout
@@ -70,18 +71,19 @@ module Vmpooler
           remove_nonexistent_vm(vm, pool)
         end
       end
+
+      true
     rescue => err
       $logger.log('d', "Fail pending VM failed with an error: #{err}")
+
+      false
     end
 
+    # DONE
     def move_pending_vm_to_ready(vm, pool, host)
-      if (host.summary) &&
-          (host.summary.guest) &&
-          (host.summary.guest.hostName) &&
-          (host.summary.guest.hostName == vm)
-
+      if host['hostname'] == vm
         begin
-          Socket.getaddrinfo(vm, nil)  # WTF?
+          Socket.getaddrinfo(vm, nil)  # WTF? I assume this is just priming the local DNS resolver cache?!?!
         rescue
         end
 
@@ -91,77 +93,91 @@ module Vmpooler
         $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
         $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish)
 
-        $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
+        $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
       end
     end
 
-    def check_ready_vm(vm, pool, ttl, vsphere)
+    # DONE
+    def check_ready_vm(vm, pool, ttl, backingservice)
       Thread.new do
-        if ttl > 0
-          if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl
-            $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
-
-            $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
-          end
+        begin
+          _check_ready_vm(vm, pool, ttl, backingservice)
+        rescue => err
+          $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : #{err}")
+          raise
         end
+      end
+    end
 
-        check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
+    # DONE
+    def _check_ready_vm(vm, pool, ttl, backingservice)
+      host = backingservice.get_vm(vm)
+      # Check if the host even exists
+      if !host
+        $redis.srem('vmpooler__ready__' + pool, vm)
+        $logger.log('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue")
+        return
+      end
 
+      # Check if the hosts TTL has expired
+      if ttl > 0
+        if (((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl
+          $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
+
+          $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
+          return
+        end
+      end
+
+      # Periodically check that the VM is available
+      check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
+      if
+        (!check_stamp) ||
+        (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime'])
+
+        $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
+
+        # Check if the VM is not powered on
         if
-          (!check_stamp) ||
-          (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime'])
+          (host['powerstate'] != 'PoweredOn')
+          $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
+          $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
+        end
 
-          $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
+        # Check if the hostname has magically changed from underneath Pooler
+        if (host['hostname'] != vm)
+          $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
+          $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
+        end
 
-          host = vsphere.find_vm(vm)
-
-          if host
-            if
-              (host.runtime) &&
-              (host.runtime.powerState) &&
-              (host.runtime.powerState != 'poweredOn')
-
-              $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
-
-              $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
-            end
-
-            if
-              (host.summary.guest) &&
-              (host.summary.guest.hostName) &&
-              (host.summary.guest.hostName != vm)
-
-              $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
-
-              $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
-            end
+        # Check if the VM is still ready/available
+        begin
+          fail "VM #{vm} is not ready" unless backingservice.is_vm_ready?(vm,pool,5)
+        rescue
+          if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
+            $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
           else
-            $redis.srem('vmpooler__ready__' + pool, vm)
-
-            $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
-          end
-
-          begin
-            open_socket vm
-          rescue
-            if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm)
-              $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
-            else
-              $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue")
-            end
+            $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue")
           end
         end
       end
     end
 
-    def check_running_vm(vm, pool, ttl, vsphere)
+    # DONE
+    def check_running_vm(vm, pool, ttl, backingservice)
       Thread.new do
-        _check_running_vm(vm, pool, ttl, vsphere)
+        begin
+          _check_running_vm(vm, pool, ttl, backingservice)
+        rescue => err
+          $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{err}")
+          raise
+        end
       end
     end
 
-    def _check_running_vm(vm, pool, ttl, vsphere)
-      host = vsphere.find_vm(vm)
+    # DONE
+    def _check_running_vm(vm, pool, ttl, backingservice)
+      host = backingservice.get_vm(vm)
 
       if host
         queue_from, queue_to = 'running', 'completed'
@@ -179,151 +195,94 @@ module Vmpooler
       end
     end
 
+    # DONE
     def move_vm_queue(pool, vm, queue_from, queue_to, msg)
       $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
       $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}")
     end
 
-    # Clone a VM
-    def clone_vm(template, folder, datastore, target, vsphere)
+    # DONE
+    def clone_vm(pool, backingservice)
       Thread.new do
         begin
-          vm = {}
-
-          if template =~ /\//
-            templatefolders = template.split('/')
-            vm['template'] = templatefolders.pop
-          end
-
-          if templatefolders
-            vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template'])
-          else
-            fail 'Please provide a full path to the template'
-          end
-
-          if vm['template'].length == 0
-            fail "Unable to find template '#{vm['template']}'!"
-          end
+          pool_name = pool['name']
 
           # Generate a randomized hostname
           o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
-          vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
+          new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
 
           # Add VM to Redis inventory ('pending' pool)
-          $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname'])
-          $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now)
-          $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template'])
-
-          # Annotate with creation time, origin template, etc.
-          # Add extraconfig options that can be queried by vmtools
-          configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
-            annotation: JSON.pretty_generate(
-                name: vm['hostname'],
-                created_by: $config[:vsphere]['username'],
-                base_template: vm['template'],
-                creation_timestamp: Time.now.utc
-            ),
-            extraConfig: [
-                { key: 'guestinfo.hostname',
-                  value: vm['hostname']
-                }
-            ]
-          )
-
-          # Choose a clone target
-          if target
-            $clone_target = vsphere.find_least_used_host(target)
-          elsif $config[:config]['clone_target']
-            $clone_target = vsphere.find_least_used_host($config[:config]['clone_target'])
-          end
-
-          # Put the VM in the specified folder and resource pool
-          relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-            datastore: vsphere.find_datastore(datastore),
-            host: $clone_target,
-            diskMoveType: :moveChildMostDiskBacking
-          )
-
-          # Create a clone spec
-          spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-            location: relocateSpec,
-            config: configSpec,
-            powerOn: true,
-            template: false
-          )
-
-          # Clone the VM
-          $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
+          $redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
+          $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
+          $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
 
           begin
             start = Time.now
-            vm[vm['template']].CloneVM_Task(
-              folder: vsphere.find_folder(folder),
-              name: vm['hostname'],
-              spec: spec
-            ).wait_for_completion
+            backingservice.create_vm(pool,new_vmname)
             finish = '%.2f' % (Time.now - start)
 
-            $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish)
-            $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish)
+            $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
+            $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
+            $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned from '#{pool_name}' in #{finish} seconds")
 
-            $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
+            $metrics.timing("clone.#{pool_name}", finish)
           rescue => err
-            $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}")
-            $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname'])
+            $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed with an error: #{err}")
+            $redis.srem('vmpooler__pending__' + pool_name, new_vmname)
             raise
+          ensure
+            $redis.decr('vmpooler__tasks__clone')
           end
-
-          $redis.decr('vmpooler__tasks__clone')
-
-          $metrics.timing("clone.#{vm['template']}", finish)
         rescue => err
-          $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}")
+          $logger.log('s', "[!] [#{pool['name']}] failed while preparing to clone with an error: #{err}")
           raise
         end
       end
     end
 
     # Destroy a VM
-    def destroy_vm(vm, pool, vsphere)
+    # DONE
+    def destroy_vm(vm, pool, backingservice)
       Thread.new do
-        $redis.srem('vmpooler__completed__' + pool, vm)
-        $redis.hdel('vmpooler__active__' + pool, vm)
-        $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
-
-        # Auto-expire metadata key
-        $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
-
-        host = vsphere.find_vm(vm)
-
-        if host
-          start = Time.now
-
-          if
-            (host.runtime) &&
-            (host.runtime.powerState) &&
-            (host.runtime.powerState == 'poweredOn')
-
-            $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
-            host.PowerOffVM_Task.wait_for_completion
-          end
-
-          host.Destroy_Task.wait_for_completion
-          finish = '%.2f' % (Time.now - start)
-
-          $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
-          $metrics.timing("destroy.#{pool}", finish)
+        begin
+          _destroy_vm(vm, pool, backingservice)
+        rescue => err
+          $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}")
+          raise
         end
       end
     end
 
+    # Destroy a VM
+    # DONE
+    def _destroy_vm(vm, pool, backingservice)
+      $redis.srem('vmpooler__completed__' + pool, vm)
+      $redis.hdel('vmpooler__active__' + pool, vm)
+      $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
+
+      # Auto-expire metadata key
+      $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+
+      start = Time.now
+
+      backingservice.destroy_vm(vm,pool)
+
+      finish = '%.2f' % (Time.now - start)
+      $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
+      $metrics.timing("destroy.#{pool}", finish)
+    end
+
     def create_vm_disk(vm, disk_size, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       Thread.new do
         _create_vm_disk(vm, disk_size, vsphere)
       end
     end
 
     def _create_vm_disk(vm, disk_size, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       host = vsphere.find_vm(vm)
 
       if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0))
@@ -358,12 +317,16 @@ module Vmpooler
     end
 
     def create_vm_snapshot(vm, snapshot_name, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       Thread.new do
         _create_vm_snapshot(vm, snapshot_name, vsphere)
       end
     end
 
     def _create_vm_snapshot(vm, snapshot_name, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       host = vsphere.find_vm(vm)
 
       if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?))
@@ -387,12 +350,16 @@ module Vmpooler
     end
 
     def revert_vm_snapshot(vm, snapshot_name, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       Thread.new do
         _revert_vm_snapshot(vm, snapshot_name, vsphere)
       end
     end
 
     def _revert_vm_snapshot(vm, snapshot_name, vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       host = vsphere.find_vm(vm)
 
       if host
@@ -413,6 +380,8 @@ module Vmpooler
     end
 
     def check_disk_queue
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       $logger.log('d', "[*] [disk_manager] starting worker thread")
 
       $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
@@ -426,6 +395,8 @@ module Vmpooler
     end
 
     def _check_disk_queue(vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       vm = $redis.spop('vmpooler__tasks__disk')
 
       unless vm.nil?
@@ -439,6 +410,8 @@ module Vmpooler
     end
 
     def check_snapshot_queue
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       $logger.log('d', "[*] [snapshot_manager] starting worker thread")
 
       $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
@@ -452,6 +425,8 @@ module Vmpooler
     end
 
     def _check_snapshot_queue(vsphere)
+fail "NOT YET REFACTORED"      
+# TODO This is all vSphere specific
       vm = $redis.spop('vmpooler__tasks__snapshot')
 
       unless vm.nil?
@@ -475,56 +450,55 @@ module Vmpooler
       end
     end
 
+    # DONE
     def migration_limit(migration_limit)
       # Returns migration_limit setting when enabled
       return false if migration_limit == 0 || ! migration_limit
       migration_limit if migration_limit >= 1
     end
 
-    def migrate_vm(vm, pool, vsphere)
+    # DONE
+    def migrate_vm(vm, pool, backingservice)
       Thread.new do
-        _migrate_vm(vm, pool, vsphere)
+        begin
+          _migrate_vm(vm, pool, backingservice)
+        rescue => err
+          $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}")
+          remove_vmpooler_migration_vm(pool, vm)
+        end
       end
     end
 
-    def _migrate_vm(vm, pool, vsphere)
-      begin
-        $redis.srem('vmpooler__migrating__' + pool, vm)
-        vm_object = vsphere.find_vm(vm)
-        parent_host, parent_host_name = get_vm_host_info(vm_object)
-        migration_limit = migration_limit $config[:config]['migration_limit']
-        migration_count = $redis.scard('vmpooler__migration')
+    # DONE
+    def _migrate_vm(vm, pool, backingservice)
+      $redis.srem('vmpooler__migrating__' + pool, vm)
 
-        if ! migration_limit
-          $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}")
+      parent_host_name = backingservice.get_vm_host(vm)
+      migration_limit = migration_limit $config[:config]['migration_limit']
+      migration_count = $redis.scard('vmpooler__migration')
+
+      if ! migration_limit
+        $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}")
+        return
+      else
+        if migration_count >= migration_limit
+          $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
           return
         else
-          if migration_count >= migration_limit
-            $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
-            return
+          $redis.sadd('vmpooler__migration', vm)
+          host_name = backingservice.find_least_used_compatible_host(vm)
+          if host_name == parent_host_name
+            $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
           else
-            $redis.sadd('vmpooler__migration', vm)
-            host, host_name = vsphere.find_least_used_compatible_host(vm_object)
-            if host == parent_host
-              $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
-            else
-              finish = migrate_vm_and_record_timing(vm_object, vm, pool, host, parent_host_name, host_name, vsphere)
-              $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds")
-            end
-            remove_vmpooler_migration_vm(pool, vm)
+            finish = migrate_vm_and_record_timing(vm, pool, parent_host_name, host_name, backingservice)
+            $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds")
           end
+          remove_vmpooler_migration_vm(pool, vm)
         end
-      rescue => err
-        $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}")
-        remove_vmpooler_migration_vm(pool, vm)
       end
     end
 
-    def get_vm_host_info(vm_object)
-      parent_host = vm_object.summary.runtime.host
-      [parent_host, parent_host.name]
-    end
-
+    # DONE
     def remove_vmpooler_migration_vm(pool, vm)
       begin
         $redis.srem('vmpooler__migration', vm)
@@ -533,9 +507,10 @@ module Vmpooler
       end
     end
 
-    def migrate_vm_and_record_timing(vm_object, vm_name, pool, host, source_host_name, dest_host_name, vsphere)
+    # DONE
+    def migrate_vm_and_record_timing(vm_name, pool, source_host_name, dest_host_name, backingservice)
       start = Time.now
-      vsphere.migrate_vm_host(vm_object, host)
+      backingservice.migrate_vm_to_host(vm_name, dest_host_name)
       finish = '%.2f' % (Time.now - start)
       $metrics.timing("migrate.#{pool}", finish)
       $metrics.increment("migrate_from.#{source_host_name}")
@@ -546,26 +521,34 @@ module Vmpooler
       finish
     end
 
+    # DONE
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics
+      case pool['backingservice']
+      when 'vsphere'
+        # TODO what about the helper
+        $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Vsphere.new({ 'metrics' => $metrics}) # TODO Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      when 'dummy'
+        $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Dummy.new($config[:backingservice][:dummy])
+      else
+         $logger.log('s', "[!] backing service #{pool['backingservice']} is unknown for pool [#{pool['name']}]")
+      end
 
       $threads[pool['name']] = Thread.new do
         loop do
-          _check_pool(pool, $vsphere[pool['name']])
-          sleep(5)
+          _check_pool(pool, $backing_services[pool['name']])
+# TODO Should this be configurable?
+          sleep(2) # Should be 5
         end
       end
     end
 
-    def _check_pool(pool, vsphere)
+    def _check_pool(pool,backingservice)
       # INVENTORY
       inventory = {}
       begin
-        base = vsphere.find_folder(pool['folder'])
-
-        base.childEntity.each do |vm|
+        backingservice.vms_in_pool(pool).each do |vm|
           if
             (! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) &&
             (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) &&
@@ -590,7 +573,7 @@ module Vmpooler
         if inventory[vm]
           begin
             vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
-            check_running_vm(vm, pool['name'], vm_lifetime, vsphere)
+            check_running_vm(vm, pool['name'], vm_lifetime, backingservice)
           rescue => err
             $logger.log('d', "[!] [#{pool['name']}] _check_pool with an error while evaluating running VMs: #{err}")
           end
@@ -601,7 +584,7 @@ module Vmpooler
       $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
         if inventory[vm]
           begin
-            check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere)
+            check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, backingservice)
           rescue => err
             $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}")
           end
@@ -613,7 +596,7 @@ module Vmpooler
         pool_timeout = pool['timeout'] || $config[:config]['timeout'] || 15
         if inventory[vm]
           begin
-            check_pending_vm(vm, pool['name'], pool_timeout, vsphere)
+            check_pending_vm(vm, pool['name'], pool_timeout, backingservice)
           rescue => err
             $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating pending VMs: #{err}")
           end
@@ -626,7 +609,7 @@ module Vmpooler
       $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm|
         if inventory[vm]
           begin
-            destroy_vm(vm, pool['name'], vsphere)
+            destroy_vm(vm, pool['name'], backingservice)
           rescue => err
             $redis.srem("vmpooler__completed__#{pool['name']}", vm)
             $redis.hdel("vmpooler__active__#{pool['name']}", vm)
@@ -663,7 +646,7 @@ module Vmpooler
       $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm|
         if inventory[vm]
           begin
-            migrate_vm(vm, pool['name'], vsphere)
+            migrate_vm(vm, pool['name'], backingservice)
           rescue => err
             $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
           end
@@ -693,14 +676,7 @@ module Vmpooler
           if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
             begin
               $redis.incr('vmpooler__tasks__clone')
-
-              clone_vm(
-                pool['template'],
-                pool['folder'],
-                pool['datastore'],
-                pool['clone_target'],
-                vsphere
-              )
+              clone_vm(pool,backingservice)
             rescue => err
               $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
               $redis.decr('vmpooler__tasks__clone')
@@ -721,21 +697,26 @@ module Vmpooler
       $redis.set('vmpooler__tasks__clone', 0)
       # Clear out vmpooler__migrations since stale entries may be left after a restart
       $redis.del('vmpooler__migration')
+      # Set default backingservice for all pools that do not have one defined
+      $config[:pools].each do |pool|
+        pool['backingservice'] = 'vsphere' if pool['backingservice'].nil?
+      end
 
       loop do
-        if ! $threads['disk_manager']
-          check_disk_queue
-        elsif ! $threads['disk_manager'].alive?
-          $logger.log('d', "[!] [disk_manager] worker thread died, restarting")
-          check_disk_queue
-        end
+        # DEBUG TO DO
+        # if ! $threads['disk_manager']
+        #   check_disk_queue
+        # elsif ! $threads['disk_manager'].alive?
+        #   $logger.log('d', "[!] [disk_manager] worker thread died, restarting")
+        #   check_disk_queue
+        # end
 
-        if ! $threads['snapshot_manager']
-          check_snapshot_queue
-        elsif ! $threads['snapshot_manager'].alive?
-          $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting")
-          check_snapshot_queue
-        end
+        # if ! $threads['snapshot_manager']
+        #   check_snapshot_queue
+        # elsif ! $threads['snapshot_manager'].alive?
+        #   $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting")
+        #   check_snapshot_queue
+        # end
 
         $config[:pools].each do |pool|
           if ! $threads[pool['name']]

lib/vmpooler/vsphere_helper.rb

@@ -1,417 +1,417 @@
-require 'rubygems' unless defined?(Gem)
-
-module Vmpooler
-  class VsphereHelper
-    ADAPTER_TYPE = 'lsiLogic'
-    DISK_TYPE = 'thin'
-    DISK_MODE = 'persistent'
-
-    def initialize(config, metrics)
-      $credentials = config[:vsphere]
-      $conf = config[:config]
-      $metrics = metrics
-    end
-
-    def ensure_connected(connection, credentials)
-      connection.serviceInstance.CurrentTime
-    rescue
-      $metrics.increment("connect.open")
-      connect_to_vsphere $credentials
-    end
-
-    def connect_to_vsphere(credentials)
-      max_tries = $conf['max_tries'] || 3
-      retry_factor = $conf['retry_factor'] || 10
-      try = 1
-      begin
-        @connection = RbVmomi::VIM.connect host: credentials['server'],
-                                           user: credentials['username'],
-                                           password: credentials['password'],
-                                           insecure: credentials['insecure'] || true
-        $metrics.increment("connect.open")
-      rescue => err
-        try += 1
-        $metrics.increment("connect.fail")
-        raise err if try == max_tries
-        sleep(try * retry_factor)
-        retry
-      end
-    end
-
-    def add_disk(vm, size, datastore)
-      ensure_connected @connection, $credentials
-
-      return false unless size.to_i > 0
-
-      vmdk_datastore = find_datastore(datastore)
-      vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk"
-
-      controller = find_disk_controller(vm)
-
-      vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec(
-        capacityKb: size.to_i * 1024 * 1024,
-        adapterType: ADAPTER_TYPE,
-        diskType: DISK_TYPE
-      )
-
-      vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo(
-        datastore: vmdk_datastore,
-        diskMode: DISK_MODE,
-        fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}"
-      )
-
-      device = RbVmomi::VIM::VirtualDisk(
-        backing: vmdk_backing,
-        capacityInKB: size.to_i * 1024 * 1024,
-        controllerKey: controller.key,
-        key: -1,
-        unitNumber: find_disk_unit_number(vm, controller)
-      )
-
-      device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec(
-        device: device,
-        operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add')
-      )
-
-      vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec(
-        deviceChange: [device_config_spec]
-      )
-
-      @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
-        datacenter: @connection.serviceInstance.find_datacenter,
-        name: "[#{vmdk_datastore.name}] #{vmdk_file_name}",
-        spec: vmdk_spec
-      ).wait_for_completion
-
-      vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion
-
-      true
-    end
-
-    def find_datastore(datastorename)
-      ensure_connected @connection, $credentials
-
-      datacenter = @connection.serviceInstance.find_datacenter
-      datacenter.find_datastore(datastorename)
-    end
-
-    def find_device(vm, deviceName)
-      ensure_connected @connection, $credentials
-
-      vm.config.hardware.device.each do |device|
-        return device if device.deviceInfo.label == deviceName
-      end
-
-      nil
-    end
-
-    def find_disk_controller(vm)
-      ensure_connected @connection, $credentials
-
-      devices = find_disk_devices(vm)
-
-      devices.keys.sort.each do |device|
-        if devices[device]['children'].length < 15
-          return find_device(vm, devices[device]['device'].deviceInfo.label)
-        end
-      end
-
-      nil
-    end
-
-    def find_disk_devices(vm)
-      ensure_connected @connection, $credentials
-
-      devices = {}
-
-      vm.config.hardware.device.each do |device|
-        if device.is_a? RbVmomi::VIM::VirtualSCSIController
-          if devices[device.controllerKey].nil?
-            devices[device.key] = {}
-            devices[device.key]['children'] = []
-          end
-
-          devices[device.key]['device'] = device
-        end
-
-        if device.is_a? RbVmomi::VIM::VirtualDisk
-          if devices[device.controllerKey].nil?
-            devices[device.controllerKey] = {}
-            devices[device.controllerKey]['children'] = []
-          end
-
-          devices[device.controllerKey]['children'].push(device)
-        end
-      end
-
-      devices
-    end
-
-    def find_disk_unit_number(vm, controller)
-      ensure_connected @connection, $credentials
-
-      used_unit_numbers = []
-      available_unit_numbers = []
-
-      devices = find_disk_devices(vm)
-
-      devices.keys.sort.each do |c|
-        next unless controller.key == devices[c]['device'].key
-        used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber)
-        devices[c]['children'].each do |disk|
-          used_unit_numbers.push(disk.unitNumber)
-        end
-      end
-
-      (0..15).each do |scsi_id|
-        if used_unit_numbers.grep(scsi_id).length <= 0
-          available_unit_numbers.push(scsi_id)
-        end
-      end
-
-      available_unit_numbers.sort[0]
-    end
-
-    def find_folder(foldername)
-      ensure_connected @connection, $credentials
-
-      datacenter = @connection.serviceInstance.find_datacenter
-      base = datacenter.vmFolder
-      folders = foldername.split('/')
-      folders.each do |folder|
-        case base
-          when RbVmomi::VIM::Folder
-            base = base.childEntity.find { |f| f.name == folder }
-          else
-            abort "Unexpected object type encountered (#{base.class}) while finding folder"
-        end
-      end
-
-      base
-    end
-
-    # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference
-    # Params:
-    # +model+:: CPU arch version to match on
-    # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
-    def get_host_utilization(host, model=nil, limit=90)
-      if model
-        return nil unless host_has_cpu_model? host, model
-      end
-      return nil if host.runtime.inMaintenanceMode
-      return nil unless host.overallStatus == 'green'
-
-      cpu_utilization = cpu_utilization_for host
-      memory_utilization = memory_utilization_for host
-
-      return nil if cpu_utilization > limit
-      return nil if memory_utilization > limit
-
-      [ cpu_utilization + memory_utilization, host ]
-    end
-
-    def host_has_cpu_model?(host, model)
-       get_host_cpu_arch_version(host) == model
-    end
-
-    def get_host_cpu_arch_version(host)
-      cpu_model = host.hardware.cpuPkg[0].description
-      cpu_model_parts = cpu_model.split()
-      arch_version = cpu_model_parts[4]
-      arch_version
-    end
-
-    def cpu_utilization_for(host)
-      cpu_usage = host.summary.quickStats.overallCpuUsage
-      cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores
-      (cpu_usage.to_f / cpu_size.to_f) * 100
-    end
-
-    def memory_utilization_for(host)
-      memory_usage = host.summary.quickStats.overallMemoryUsage
-      memory_size = host.summary.hardware.memorySize / 1024 / 1024
-      (memory_usage.to_f / memory_size.to_f) * 100
-    end
-
-    def find_least_used_host(cluster)
-      ensure_connected @connection, $credentials
-
-      cluster_object = find_cluster(cluster)
-      target_hosts = get_cluster_host_utilization(cluster_object)
-      least_used_host = target_hosts.sort[0][1]
-      least_used_host
-    end
-
-    def find_cluster(cluster)
-      datacenter = @connection.serviceInstance.find_datacenter
-      datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster }
-    end
-
-    def get_cluster_host_utilization(cluster)
-      cluster_hosts = []
-      cluster.host.each do |host|
-        host_usage = get_host_utilization(host)
-        cluster_hosts << host_usage if host_usage
-      end
-      cluster_hosts
-    end
-
-    def find_least_used_compatible_host(vm)
-      ensure_connected @connection, $credentials
-
-      source_host = vm.summary.runtime.host
-      model = get_host_cpu_arch_version(source_host)
-      cluster = source_host.parent
-      target_hosts = []
-      cluster.host.each do |host|
-        host_usage = get_host_utilization(host, model)
-        target_hosts << host_usage if host_usage
-      end
-      target_host = target_hosts.sort[0][1]
-      [target_host, target_host.name]
-    end
-
-    def find_pool(poolname)
-      ensure_connected @connection, $credentials
-
-      datacenter = @connection.serviceInstance.find_datacenter
-      base = datacenter.hostFolder
-      pools = poolname.split('/')
-      pools.each do |pool|
-        case base
-          when RbVmomi::VIM::Folder
-            base = base.childEntity.find { |f| f.name == pool }
-          when RbVmomi::VIM::ClusterComputeResource
-            base = base.resourcePool.resourcePool.find { |f| f.name == pool }
-          when RbVmomi::VIM::ResourcePool
-            base = base.resourcePool.find { |f| f.name == pool }
-          else
-            abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
-        end
-      end
-
-      base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool)
-      base
-    end
-
-    def find_snapshot(vm, snapshotname)
-      if vm.snapshot
-        get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname)
-      end
-    end
-
-    def find_vm(vmname)
-      ensure_connected @connection, $credentials
-      find_vm_light(vmname) || find_vm_heavy(vmname)[vmname]
-    end
-
-    def find_vm_light(vmname)
-      ensure_connected @connection, $credentials
-
-      @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname)
-    end
-
-    def find_vm_heavy(vmname)
-      ensure_connected @connection, $credentials
-
-      vmname = vmname.is_a?(Array) ? vmname : [vmname]
-      containerView = get_base_vm_container_from @connection
-      propertyCollector = @connection.propertyCollector
-
-      objectSet = [{
-        obj: containerView,
-        skip: true,
-        selectSet: [RbVmomi::VIM::TraversalSpec.new(
-            name: 'gettingTheVMs',
-            path: 'view',
-            skip: false,
-            type: 'ContainerView'
-        )]
-      }]
-
-      propSet = [{
-        pathSet: ['name'],
-        type: 'VirtualMachine'
-      }]
-
-      results = propertyCollector.RetrievePropertiesEx(
-        specSet: [{
-          objectSet: objectSet,
-          propSet: propSet
-        }],
-        options: { maxObjects: nil }
-      )
-
-      vms = {}
-      results.objects.each do |result|
-        name = result.propSet.first.val
-        next unless vmname.include? name
-        vms[name] = result.obj
-      end
-
-      while results.token
-        results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token)
-        results.objects.each do |result|
-          name = result.propSet.first.val
-          next unless vmname.include? name
-          vms[name] = result.obj
-        end
-      end
-
-      vms
-    end
-
-    def find_vmdks(vmname, datastore)
-      ensure_connected @connection, $credentials
-
-      disks = []
-
-      vmdk_datastore = find_datastore(datastore)
-
-      vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file'
-      vm_files.keys.each do |f|
-        vm_files[f]['layoutEx.file'].each do |l|
-          if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/)
-            disks.push(l)
-          end
-        end
-      end
-
-      disks
-    end
-
-    def get_base_vm_container_from(connection)
-      ensure_connected @connection, $credentials
-
-      viewManager = connection.serviceContent.viewManager
-      viewManager.CreateContainerView(
-        container: connection.serviceContent.rootFolder,
-        recursive: true,
-        type: ['VirtualMachine']
-      )
-    end
-
-    def get_snapshot_list(tree, snapshotname)
-      snapshot = nil
-
-      tree.each do |child|
-        if child.name == snapshotname
-          snapshot ||= child.snapshot
-        else
-          snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname)
-        end
-      end
-
-      snapshot
-    end
-
-    def migrate_vm_host(vm, host)
-      relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host)
-      vm.RelocateVM_Task(spec: relospec).wait_for_completion
-    end
-
-    def close
-      @connection.close
-    end
-  end
-end
+# require 'rubygems' unless defined?(Gem)
+
+# module Vmpooler
+#   class VsphereHelper
+#     ADAPTER_TYPE = 'lsiLogic'
+#     DISK_TYPE = 'thin'
+#     DISK_MODE = 'persistent'
+
+#     def initialize(config, metrics)
+#       $credentials = config[:vsphere]
+#       $conf = config[:config]
+#       $metrics = metrics
+#     end
+
+#     def ensure_connected(connection, credentials)
+#       connection.serviceInstance.CurrentTime
+#     rescue
+#       $metrics.increment("connect.open")
+#       connect_to_vsphere $credentials
+#     end
+
+#     def connect_to_vsphere(credentials)
+#       max_tries = $conf['max_tries'] || 3
+#       retry_factor = $conf['retry_factor'] || 10
+#       try = 1
+#       begin
+#         @connection = RbVmomi::VIM.connect host: credentials['server'],
+#                                            user: credentials['username'],
+#                                            password: credentials['password'],
+#                                            insecure: credentials['insecure'] || true
+#         $metrics.increment("connect.open")
+#       rescue => err
+#         try += 1
+#         $metrics.increment("connect.fail")
+#         raise err if try == max_tries
+#         sleep(try * retry_factor)
+#         retry
+#       end
+#     end
+
+#     def add_disk(vm, size, datastore)
+#       ensure_connected @connection, $credentials
+
+#       return false unless size.to_i > 0
+
+#       vmdk_datastore = find_datastore(datastore)
+#       vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk"
+
+#       controller = find_disk_controller(vm)
+
+#       vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec(
+#         capacityKb: size.to_i * 1024 * 1024,
+#         adapterType: ADAPTER_TYPE,
+#         diskType: DISK_TYPE
+#       )
+
+#       vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo(
+#         datastore: vmdk_datastore,
+#         diskMode: DISK_MODE,
+#         fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}"
+#       )
+
+#       device = RbVmomi::VIM::VirtualDisk(
+#         backing: vmdk_backing,
+#         capacityInKB: size.to_i * 1024 * 1024,
+#         controllerKey: controller.key,
+#         key: -1,
+#         unitNumber: find_disk_unit_number(vm, controller)
+#       )
+
+#       device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec(
+#         device: device,
+#         operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add')
+#       )
+
+#       vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec(
+#         deviceChange: [device_config_spec]
+#       )
+
+#       @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
+#         datacenter: @connection.serviceInstance.find_datacenter,
+#         name: "[#{vmdk_datastore.name}] #{vmdk_file_name}",
+#         spec: vmdk_spec
+#       ).wait_for_completion
+
+#       vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion
+
+#       true
+#     end
+
+#     def find_datastore(datastorename)
+#       ensure_connected @connection, $credentials
+
+#       datacenter = @connection.serviceInstance.find_datacenter
+#       datacenter.find_datastore(datastorename)
+#     end
+
+#     def find_device(vm, deviceName)
+#       ensure_connected @connection, $credentials
+
+#       vm.config.hardware.device.each do |device|
+#         return device if device.deviceInfo.label == deviceName
+#       end
+
+#       nil
+#     end
+
+#     def find_disk_controller(vm)
+#       ensure_connected @connection, $credentials
+
+#       devices = find_disk_devices(vm)
+
+#       devices.keys.sort.each do |device|
+#         if devices[device]['children'].length < 15
+#           return find_device(vm, devices[device]['device'].deviceInfo.label)
+#         end
+#       end
+
+#       nil
+#     end
+
+#     def find_disk_devices(vm)
+#       ensure_connected @connection, $credentials
+
+#       devices = {}
+
+#       vm.config.hardware.device.each do |device|
+#         if device.is_a? RbVmomi::VIM::VirtualSCSIController
+#           if devices[device.controllerKey].nil?
+#             devices[device.key] = {}
+#             devices[device.key]['children'] = []
+#           end
+
+#           devices[device.key]['device'] = device
+#         end
+
+#         if device.is_a? RbVmomi::VIM::VirtualDisk
+#           if devices[device.controllerKey].nil?
+#             devices[device.controllerKey] = {}
+#             devices[device.controllerKey]['children'] = []
+#           end
+
+#           devices[device.controllerKey]['children'].push(device)
+#         end
+#       end
+
+#       devices
+#     end
+
+#     def find_disk_unit_number(vm, controller)
+#       ensure_connected @connection, $credentials
+
+#       used_unit_numbers = []
+#       available_unit_numbers = []
+
+#       devices = find_disk_devices(vm)
+
+#       devices.keys.sort.each do |c|
+#         next unless controller.key == devices[c]['device'].key
+#         used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber)
+#         devices[c]['children'].each do |disk|
+#           used_unit_numbers.push(disk.unitNumber)
+#         end
+#       end
+
+#       (0..15).each do |scsi_id|
+#         if used_unit_numbers.grep(scsi_id).length <= 0
+#           available_unit_numbers.push(scsi_id)
+#         end
+#       end
+
+#       available_unit_numbers.sort[0]
+#     end
+
+#     def find_folder(foldername)
+#       ensure_connected @connection, $credentials
+
+#       datacenter = @connection.serviceInstance.find_datacenter
+#       base = datacenter.vmFolder
+#       folders = foldername.split('/')
+#       folders.each do |folder|
+#         case base
+#           when RbVmomi::VIM::Folder
+#             base = base.childEntity.find { |f| f.name == folder }
+#           else
+#             abort "Unexpected object type encountered (#{base.class}) while finding folder"
+#         end
+#       end
+
+#       base
+#     end
+
+#     # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference
+#     # Params:
+#     # +model+:: CPU arch version to match on
+#     # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
+#     def get_host_utilization(host, model=nil, limit=90)
+#       if model
+#         return nil unless host_has_cpu_model? host, model
+#       end
+#       return nil if host.runtime.inMaintenanceMode
+#       return nil unless host.overallStatus == 'green'
+
+#       cpu_utilization = cpu_utilization_for host
+#       memory_utilization = memory_utilization_for host
+
+#       return nil if cpu_utilization > limit
+#       return nil if memory_utilization > limit
+
+#       [ cpu_utilization + memory_utilization, host ]
+#     end
+
+#     def host_has_cpu_model?(host, model)
+#        get_host_cpu_arch_version(host) == model
+#     end
+
+#     def get_host_cpu_arch_version(host)
+#       cpu_model = host.hardware.cpuPkg[0].description
+#       cpu_model_parts = cpu_model.split()
+#       arch_version = cpu_model_parts[4]
+#       arch_version
+#     end
+
+#     def cpu_utilization_for(host)
+#       cpu_usage = host.summary.quickStats.overallCpuUsage
+#       cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores
+#       (cpu_usage.to_f / cpu_size.to_f) * 100
+#     end
+
+#     def memory_utilization_for(host)
+#       memory_usage = host.summary.quickStats.overallMemoryUsage
+#       memory_size = host.summary.hardware.memorySize / 1024 / 1024
+#       (memory_usage.to_f / memory_size.to_f) * 100
+#     end
+
+#     def find_least_used_host(cluster)
+#       ensure_connected @connection, $credentials
+
+#       cluster_object = find_cluster(cluster)
+#       target_hosts = get_cluster_host_utilization(cluster_object)
+#       least_used_host = target_hosts.sort[0][1]
+#       least_used_host
+#     end
+
+#     def find_cluster(cluster)
+#       datacenter = @connection.serviceInstance.find_datacenter
+#       datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster }
+#     end
+
+#     def get_cluster_host_utilization(cluster)
+#       cluster_hosts = []
+#       cluster.host.each do |host|
+#         host_usage = get_host_utilization(host)
+#         cluster_hosts << host_usage if host_usage
+#       end
+#       cluster_hosts
+#     end
+
+#     def find_least_used_compatible_host(vm)
+#       ensure_connected @connection, $credentials
+
+#       source_host = vm.summary.runtime.host
+#       model = get_host_cpu_arch_version(source_host)
+#       cluster = source_host.parent
+#       target_hosts = []
+#       cluster.host.each do |host|
+#         host_usage = get_host_utilization(host, model)
+#         target_hosts << host_usage if host_usage
+#       end
+#       target_host = target_hosts.sort[0][1]
+#       [target_host, target_host.name]
+#     end
+
+#     def find_pool(poolname)
+#       ensure_connected @connection, $credentials
+
+#       datacenter = @connection.serviceInstance.find_datacenter
+#       base = datacenter.hostFolder
+#       pools = poolname.split('/')
+#       pools.each do |pool|
+#         case base
+#           when RbVmomi::VIM::Folder
+#             base = base.childEntity.find { |f| f.name == pool }
+#           when RbVmomi::VIM::ClusterComputeResource
+#             base = base.resourcePool.resourcePool.find { |f| f.name == pool }
+#           when RbVmomi::VIM::ResourcePool
+#             base = base.resourcePool.find { |f| f.name == pool }
+#           else
+#             abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
+#         end
+#       end
+
+#       base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool)
+#       base
+#     end
+
+#     def find_snapshot(vm, snapshotname)
+#       if vm.snapshot
+#         get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname)
+#       end
+#     end
+
+#     def find_vm(vmname)
+#       ensure_connected @connection, $credentials
+#       find_vm_light(vmname) || find_vm_heavy(vmname)[vmname]
+#     end
+
+#     def find_vm_light(vmname)
+#       ensure_connected @connection, $credentials
+
+#       @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname)
+#     end
+
+#     def find_vm_heavy(vmname)
+#       ensure_connected @connection, $credentials
+
+#       vmname = vmname.is_a?(Array) ? vmname : [vmname]
+#       containerView = get_base_vm_container_from @connection
+#       propertyCollector = @connection.propertyCollector
+
+#       objectSet = [{
+#         obj: containerView,
+#         skip: true,
+#         selectSet: [RbVmomi::VIM::TraversalSpec.new(
+#             name: 'gettingTheVMs',
+#             path: 'view',
+#             skip: false,
+#             type: 'ContainerView'
+#         )]
+#       }]
+
+#       propSet = [{
+#         pathSet: ['name'],
+#         type: 'VirtualMachine'
+#       }]
+
+#       results = propertyCollector.RetrievePropertiesEx(
+#         specSet: [{
+#           objectSet: objectSet,
+#           propSet: propSet
+#         }],
+#         options: { maxObjects: nil }
+#       )
+
+#       vms = {}
+#       results.objects.each do |result|
+#         name = result.propSet.first.val
+#         next unless vmname.include? name
+#         vms[name] = result.obj
+#       end
+
+#       while results.token
+#         results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token)
+#         results.objects.each do |result|
+#           name = result.propSet.first.val
+#           next unless vmname.include? name
+#           vms[name] = result.obj
+#         end
+#       end
+
+#       vms
+#     end
+
+#     def find_vmdks(vmname, datastore)
+#       ensure_connected @connection, $credentials
+
+#       disks = []
+
+#       vmdk_datastore = find_datastore(datastore)
+
+#       vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file'
+#       vm_files.keys.each do |f|
+#         vm_files[f]['layoutEx.file'].each do |l|
+#           if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/)
+#             disks.push(l)
+#           end
+#         end
+#       end
+
+#       disks
+#     end
+
+#     def get_base_vm_container_from(connection)
+#       ensure_connected @connection, $credentials
+
+#       viewManager = connection.serviceContent.viewManager
+#       viewManager.CreateContainerView(
+#         container: connection.serviceContent.rootFolder,
+#         recursive: true,
+#         type: ['VirtualMachine']
+#       )
+#     end
+
+#     def get_snapshot_list(tree, snapshotname)
+#       snapshot = nil
+
+#       tree.each do |child|
+#         if child.name == snapshotname
+#           snapshot ||= child.snapshot
+#         else
+#           snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname)
+#         end
+#       end
+
+#       snapshot
+#     end
+
+#     def migrate_vm_host(vm, host)
+#       relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host)
+#       vm.RelocateVM_Task(spec: relospec).wait_for_completion
+#     end
+
+#     def close
+#       @connection.close
+#     end
+#   end
+# end

spec/vmpooler/backingservice/base_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+# This spec does not really exercise code paths but is merely used
+# to enforce that certain methods are defined in the base classes
+
+describe 'Vmpooler::PoolManager::BackingService::Base' do
+  let(:config) { {} }
+  let(:fake_vm) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
+  }
+
+  subject { Vmpooler::PoolManager::BackingService::Base.new(config) }
+
+  describe '#name' do
+    it 'should be base' do
+      expect(subject.name).to eq('base')
+    end
+  end
+
+  describe '#vms_in_pool' do
+    it 'should raise error' do
+      expect{subject.vms_in_pool('pool')}.to raise_error(/does not implement vms_in_pool/)
+    end
+  end
+
+  describe '#get_vm_host' do
+    it 'should raise error' do
+      expect{subject.get_vm_host('vm')}.to raise_error(/does not implement get_vm_host/)
+    end
+  end
+
+  describe '#find_least_used_compatible_host' do
+    it 'should raise error' do
+      expect{subject.find_least_used_compatible_host('vm')}.to raise_error(/does not implement find_least_used_compatible_host/)
+    end
+  end
+
+  describe '#migrate_vm_to_host' do
+    it 'should raise error' do
+      expect{subject.migrate_vm_to_host('vm','host')}.to raise_error(/does not implement migrate_vm_to_host/)
+    end
+  end
+
+  describe '#get_vm' do
+    it 'should raise error' do
+      expect{subject.get_vm('vm')}.to raise_error(/does not implement get_vm/)
+    end
+  end
+
+  describe '#create_vm' do
+    it 'should raise error' do
+      expect{subject.create_vm('pool','newname')}.to raise_error(/does not implement create_vm/)
+    end
+  end
+
+  describe '#destroy_vm' do
+    it 'should raise error' do
+      expect{subject.destroy_vm('vm','pool')}.to raise_error(/does not implement destroy_vm/)
+    end
+  end
+
+  describe '#is_vm_ready?' do
+    it 'should raise error' do
+      expect{subject.is_vm_ready?('vm','pool','timeout')}.to raise_error(/does not implement is_vm_ready?/)
+    end
+  end
+
+  describe '#vm_exists?' do
+    it 'should raise error' do
+      expect{subject.vm_exists?('vm')}.to raise_error(/does not implement/)
+    end
+
+    it 'should return true when get_vm is returns an object' do
+      allow(subject).to receive(:get_vm).with('vm').and_return(fake_vm)
+
+      expect(subject.vm_exists?('vm')).to eq(true)
+    end
+
+    it 'should return false when get_vm is returns nil' do
+      allow(subject).to receive(:get_vm).with('vm').and_return(nil)
+
+      expect(subject.vm_exists?('vm')).to eq(false)
+    end
+  end
+end

spec/vmpooler/backingservice/dummy_spec.rb

@@ -0,0 +1,408 @@
+require 'spec_helper'
+
+describe 'Vmpooler::PoolManager::BackingService::Dummy' do
+  let(:logger) { double('logger') }
+
+  let (:pool_hash) {
+    pool = {}
+    pool['name'] = 'pool1'
+
+    pool
+  }
+
+  let (:default_config) {
+    # Construct an initial state for testing
+    dummylist = {}
+    dummylist['pool'] = {}
+    # pool1 is a pool of "normal" VMs
+    dummylist['pool']['pool1'] = []
+      # A normal running VM
+      vm = {}
+      vm['name'] = 'vm1'
+      vm['hostname'] = 'vm1'
+      vm['domain']  = 'dummy.local'
+      vm['vm_template'] = 'template1'
+      vm['template'] = 'pool1'
+      vm['poolname'] = 'pool1'
+      vm['ready'] = true
+      vm['boottime'] = Time.now
+      vm['powerstate'] = 'PoweredOn'
+      vm['vm_host'] = 'HOST1'
+      vm['dummy_state'] = 'RUNNING'
+      dummylist['pool']['pool1'] << vm
+
+    # pool2 is a pool of "abnormal" VMs e.g. PoweredOff etc.
+    dummylist['pool']['pool2'] = []
+      # A freshly provisioned VM that is not ready
+      vm = {}
+      vm['name'] = 'vm2'
+      vm['hostname'] = 'vm2'
+      vm['domain']  = 'dummy.local'
+      vm['vm_template'] = 'template1'
+      vm['template'] = 'pool2'
+      vm['poolname'] = 'pool2'
+      vm['ready'] = false
+      vm['boottime'] = Time.now
+      vm['powerstate'] = 'PoweredOn'
+      vm['vm_host'] = 'HOST1'
+      vm['dummy_state'] = 'UNKNOWN'
+      dummylist['pool']['pool2'] << vm
+      # A freshly provisioned VM that is running but not ready
+      vm = {}
+      vm['name'] = 'vm3'
+      vm['hostname'] = 'vm3'
+      vm['domain']  = 'dummy.local'
+      vm['vm_template'] = 'template1'
+      vm['template'] = 'pool2'
+      vm['poolname'] = 'pool2'
+      vm['ready'] = false
+      vm['boottime'] = Time.now
+      vm['powerstate'] = 'PoweredOn'
+      vm['vm_host'] = 'HOST1'
+      vm['dummy_state'] = 'RUNNING'
+      dummylist['pool']['pool2'] << vm
+
+    config = {}
+    config['initial_state'] = dummylist
+
+    config
+  }
+
+  let (:config) { default_config }
+
+  subject { Vmpooler::PoolManager::BackingService::Dummy.new(config) }
+
+  before do
+    allow(logger).to receive(:log)
+    $logger = logger
+  end
+
+  describe '#name' do
+    it 'should be dummy' do
+      expect(subject.name).to eq('dummy')
+    end
+  end
+
+  describe '#vms_in_pool' do
+    it 'should return [] when pool does not exist' do
+      pool = pool_hash
+      pool['name'] = 'pool_does_not_exist'
+
+      vm_list = subject.vms_in_pool(pool)
+
+      expect(vm_list).to eq([])
+    end
+
+    it 'should return an array of VMs when pool exists' do
+      pool = pool_hash
+      pool['name'] = 'pool1'
+
+      vm_list = subject.vms_in_pool(pool)
+
+      expect(vm_list.count).to eq(1)
+    end
+  end
+
+  describe '#get_vm_host' do
+    it 'should return the hostname when VM exists' do
+      expect(subject.get_vm_host('vm1')).to eq('HOST1')
+    end
+
+    it 'should error when VM does not exist' do
+      expect{subject.get_vm_host('doesnotexist')}.to raise_error(RuntimeError)
+    end
+  end
+
+  describe '#find_least_used_compatible_host' do
+    it 'should return the current host' do
+      new_host = subject.find_least_used_compatible_host('vm1')
+      expect(new_host).to eq('HOST1')
+    end
+
+    context 'using migratevm_couldmove_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['migratevm_couldmove_percent'] = 0
+          config
+        }
+
+        it 'should return the current host' do
+          new_host = subject.find_least_used_compatible_host('vm1')
+          expect(new_host).to eq('HOST1')
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['migratevm_couldmove_percent'] = 100
+          config
+        }
+
+        it 'should return a different host' do
+          new_host = subject.find_least_used_compatible_host('vm1')
+          expect(new_host).to_not eq('HOST1')
+        end
+      end
+
+    end
+  end
+
+  describe '#migrate_vm_to_host' do
+    it 'should move to the new host' do
+      expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true)
+      expect(subject.get_vm_host('vm1')).to eq('NEWHOST')
+    end
+
+    context 'using migratevm_fail_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['migratevm_fail_percent'] = 0
+          config
+        }
+
+        it 'should move to the new host' do
+          expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true)
+          expect(subject.get_vm_host('vm1')).to eq('NEWHOST')
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['migratevm_fail_percent'] = 100
+          config
+        }
+
+        it 'should raise an error' do
+          expect{subject.migrate_vm_to_host('vm1','NEWHOST')}.to raise_error(/migratevm_fail_percent/)
+        end
+      end
+    end
+  end
+
+  describe '#get_vm' do
+    it 'should return the VM when VM exists' do
+      vm = subject.get_vm('vm1')
+      expect(vm['name']).to eq('vm1')
+      expect(vm['powerstate']).to eq('PoweredOn')
+      expect(vm['hostname']).to eq(vm['name'])
+    end
+
+    it 'should return nil when VM does not exist' do
+      expect(subject.get_vm('doesnotexist')).to eq(nil)
+    end
+
+    context 'using getvm_poweroff_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['getvm_poweroff_percent'] = 0
+          config
+        }
+
+        it 'will not power off a VM' do
+          vm = subject.get_vm('vm1')
+          expect(vm['name']).to eq('vm1')
+          expect(vm['powerstate']).to eq('PoweredOn')
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['getvm_poweroff_percent'] = 100
+          config
+        }
+
+        it 'will power off a VM' do
+          vm = subject.get_vm('vm1')
+          expect(vm['name']).to eq('vm1')
+          expect(vm['powerstate']).to eq('PoweredOff')
+        end
+      end
+    end
+
+    context 'using getvm_rename_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['getvm_rename_percent'] = 0
+          config
+        }
+
+        it 'will not rename a VM' do
+          vm = subject.get_vm('vm1')
+          expect(vm['name']).to eq('vm1')
+          expect(vm['hostname']).to eq(vm['name'])
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['getvm_rename_percent'] = 100
+          config
+        }
+
+        it 'will rename a VM' do
+          vm = subject.get_vm('vm1')
+          expect(vm['name']).to eq('vm1')
+          expect(vm['hostname']).to_not eq(vm['name'])
+        end
+      end
+    end
+  end
+
+  describe '#create_vm' do
+    it 'should return a new VM' do
+      expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm')
+    end
+
+    it 'should increase the number of VMs in the pool' do
+      pool = pool_hash
+      old_pool_count = subject.vms_in_pool(pool).count
+      new_vm = subject.create_vm(pool_hash,'newvm')
+      expect(subject.vms_in_pool(pool).count).to eq(old_pool_count + 1)
+    end
+
+    context 'using createvm_fail_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['createvm_fail_percent'] = 0
+          config
+        }
+
+        it 'should return a new VM' do
+          expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm')
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['createvm_fail_percent'] = 100
+          config
+        }
+
+        it 'should raise an error' do
+          expect{subject.create_vm(pool_hash,'newvm')}.to raise_error(/createvm_fail_percent/)
+        end
+
+        it 'new VM should not exist' do
+          begin
+            subject.create_vm(pool_hash,'newvm')
+          rescue
+          end
+          expect(subject.get_vm('newvm')).to eq(nil)
+        end
+      end
+    end
+  end
+
+  describe '#destroy_vm' do
+    it 'should return true when destroyed' do
+      expect(subject.destroy_vm('vm1','pool1')).to eq(true)
+    end
+
+    it 'should log if the VM is powered off' do
+      expect(logger).to receive(:log).with('d', "[ ] [pool1] 'vm1' is being shut down")
+      expect(subject.destroy_vm('vm1','pool1')).to eq(true)
+    end
+
+    it 'should return false if VM does not exist' do
+      expect(subject.destroy_vm('doesnotexist','pool1')).to eq(false)
+    end
+
+    it 'should return false if VM is not in the correct pool' do
+      expect(subject.destroy_vm('vm1','differentpool')).to eq(false)
+    end
+
+    context 'using destroyvm_fail_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['destroyvm_fail_percent'] = 0
+          config
+        }
+
+        it 'should return true when destroyed' do
+          expect(subject.destroy_vm('vm1','pool1')).to eq(true)
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['destroyvm_fail_percent'] = 100
+          config
+        }
+
+        it 'should raise an error' do
+          expect{subject.destroy_vm('vm1','pool1')}.to raise_error(/migratevm_fail_percent/)
+        end
+      end
+    end
+  end
+
+  describe '#is_vm_ready?' do
+    it 'should return true if ready' do
+      expect(subject.is_vm_ready?('vm1','pool1',0)).to eq(true)
+    end
+
+    it 'should return false if VM does not exist' do
+      expect(subject.is_vm_ready?('doesnotexist','pool1',0)).to eq(false)
+    end
+
+    it 'should return false if VM is not in the correct pool' do
+      expect(subject.is_vm_ready?('vm1','differentpool',0)).to eq(false)
+    end
+
+    it 'should raise an error if timeout expires' do
+      expect{subject.is_vm_ready?('vm2','pool2',1)}.to raise_error(Timeout::Error)
+    end
+
+    it 'should return true if VM becomes ready' do
+      expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true)
+    end
+
+    context 'using vmready_fail_percent' do
+      describe 'of zero' do
+        let (:config) {
+          config = default_config
+          config['vmready_fail_percent'] = 0
+          config
+        }
+
+        it 'should return true if VM becomes ready' do
+          expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true)
+        end
+      end
+
+      describe 'of 100' do
+        let (:config) {
+          config = default_config
+          config['vmready_fail_percent'] = 100
+          config
+        }
+
+        it 'should raise an error' do
+          expect{subject.is_vm_ready?('vm3','pool2',1)}.to raise_error(/vmready_fail_percent/)
+        end
+      end
+    end
+  end
+
+  describe '#vm_exists?' do
+    it 'should return true when VM exists' do
+      expect(subject.vm_exists?('vm1')).to eq(true)
+    end
+
+    it 'should return true when VM does not exist' do
+      expect(subject.vm_exists?('doesnotexist')).to eq(false)
+    end
+  end
+end

spec/vmpooler/pool_manager_migration_spec.rb

@@ -3,84 +3,96 @@ require 'mock_redis'
 require 'time'
 
 describe 'Pool Manager' do
-  let(:logger) { double('logger') }
   let(:redis) { MockRedis.new }
+  let(:logger) { double('logger') }
   let(:metrics) { Vmpooler::DummyStatsd.new }
   let(:config) {
     {
       config: {
-        'site_name' => 'test pooler',
         'migration_limit' => 2,
-        vsphere: {
-          'server' => 'vsphere.puppet.com',
-          'username' => 'vmpooler@vsphere.local',
-          'password' => '',
-          'insecure' => true
-        },
-        pools: [ {'name' => 'pool1', 'size' => 5, 'folder' => 'pool1_folder'} ],
-        statsd: { 'prefix' => 'stats_prefix'},
-        pool_names: [ 'pool1' ]
       }
     }
   }
-  let(:pool) { config[:config][:pools][0]['name'] }
-  let(:vm) {
-    {
-      'name' => 'vm1',
-      'host' => 'host1',
-      'template' => pool,
-    }
+  let(:backingservice) { double('backingservice') }
+  let(:pool) { 'pool1' }
+  let(:vm) { 'vm1' }
+  let(:timeout) { 5 }
+  let(:host) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
   }
+  let(:vm_host_hostname) { 'host1' }
+
+  subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
+
+  describe "#migration_limit" do
+    it 'return false if config is nil' do
+      expect(subject.migration_limit(nil)).to equal(false)
+    end
+    it 'return false if config is 0' do
+      expect(subject.migration_limit(0)).to equal(false)
+    end
+    it 'return nil if config is -1' do
+      expect(subject.migration_limit(-1)).to equal(nil)
+    end
+    it 'return 1 if config is 1' do
+      expect(subject.migration_limit(1)).to equal(1)
+    end
+    it 'return 100 if config is 100' do
+      expect(subject.migration_limit(100)).to equal(100)
+    end
+  end
 
   describe '#_migrate_vm' do
-    let(:vsphere) { double(pool) }
-    let(:pooler) { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
     context 'evaluates VM for migration and logs host' do
       before do
-        create_migrating_vm vm['name'], pool, redis
-        allow(vsphere).to receive(:find_vm).and_return(vm)
-        allow(pooler).to receive(:get_vm_host_info).and_return([{'name' => 'host1'}, 'host1'])
+        create_migrating_vm vm, pool, redis
+        allow(backingservice).to receive(:get_vm_host).with(vm).and_return(vm_host_hostname)
       end
 
       it 'logs VM host when migration is disabled' do
         config[:config]['migration_limit'] = nil
 
-        expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be true
-        expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}")
+        expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be true
+        expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}")
 
-        pooler._migrate_vm(vm['name'], pool, vsphere)
+        subject._migrate_vm(vm, pool, backingservice)
 
-        expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be false
+        expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be false
       end
 
       it 'verifies that migration_limit greater than or equal to migrations in progress and logs host' do
-        add_vm_to_migration_set vm['name'], redis
+        add_vm_to_migration_set vm, redis
         add_vm_to_migration_set 'vm2', redis
 
-        expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}. No migration will be evaluated since the migration_limit has been reached")
+        expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}. No migration will be evaluated since the migration_limit has been reached")
 
-        pooler._migrate_vm(vm['name'], pool, vsphere)
+        subject._migrate_vm(vm, pool, backingservice)
       end
 
       it 'verifies that migration_limit is less than migrations in progress and logs old host, new host and migration time' do
-        allow(vsphere).to receive(:find_least_used_compatible_host).and_return([{'name' => 'host2'}, 'host2'])
-        allow(vsphere).to receive(:migrate_vm_host)
+        allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2')
+        allow(backingservice).to receive(:migrate_vm_to_host).and_return(true)
 
         expect(redis.hget("vmpooler__vm__#{vm['name']}", 'migration_time'))
         expect(redis.hget("vmpooler__vm__#{vm['name']}", 'checkout_to_migration'))
-        expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm['name']}' migrated from #{vm['host']} to host2 in 0.00 seconds")
+        expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' migrated from #{vm_host_hostname} to host2 in 0.00 seconds")
 
-        pooler._migrate_vm(vm['name'], pool, vsphere)
+        subject._migrate_vm(vm, pool, backingservice)
       end
 
       it 'fails when no suitable host can be found' do
         error = 'ArgumentError: No target host found'
-        allow(vsphere).to receive(:find_least_used_compatible_host)
-        allow(vsphere).to receive(:migrate_vm_host).and_raise(error)
+        allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2')
+        allow(backingservice).to receive(:migrate_vm_to_host).and_raise(error)
 
-        expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm['name']}' migration failed with an error: #{error}")
-
-        pooler._migrate_vm(vm['name'], pool, vsphere)
+        expect{subject._migrate_vm(vm, pool, backingservice)}.to raise_error(error)
       end
     end
   end

spec/vmpooler/pool_manager_snapshot_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+require 'time'
+
+describe 'Pool Manager' do
+  let(:logger) { double('logger') }
+  let(:redis) { double('redis') }
+  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:config) { {} }
+  let(:pool) { 'pool1' }
+  let(:vm) { 'vm1' }
+  let(:timeout) { 5 }
+  let(:host) { double('host') }
+
+  subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
+
+  describe '#_create_vm_snapshot' do
+    let(:snapshot_manager) { 'snapshot_manager' }
+    let(:pool_helper) { double('snapshot_manager') }
+    let(:backingservice) { {snapshot_manager => pool_helper} }
+
+    before do
+      expect(subject).not_to be_nil
+      $backingservice = backingservice
+    end
+
+    context '(valid host)' do
+      let(:vm_host) { double('vmhost') }
+
+      it 'creates a snapshot' do
+        expect(backingservice).to receive(:get_vm).and_return vm_host
+        expect(logger).to receive(:log)
+        expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion)
+        expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s)
+        expect(logger).to receive(:log)
+
+        subject._create_vm_snapshot('testvm', 'testsnapshot', backingservice)
+      end
+    end
+  end
+
+  describe '#_revert_vm_snapshot' do
+    let(:snapshot_manager) { 'snapshot_manager' }
+    let(:pool_helper) { double('snapshot_manager') }
+    let(:backingservice) { {snapshot_manager => pool_helper} }
+
+    before do
+      expect(subject).not_to be_nil
+      $backingservice = backingservice
+    end
+
+    context '(valid host)' do
+      let(:vm_host) { double('vmhost') }
+      let(:vm_snapshot) { double('vmsnapshot') }
+
+      it 'reverts a snapshot' do
+        expect(backingservice).to receive(:get_vm).and_return vm_host
+        expect(backingservice).to receive(:find_snapshot).and_return vm_snapshot
+        expect(logger).to receive(:log)
+        expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion)
+        expect(logger).to receive(:log)
+
+        subject._revert_vm_snapshot('testvm', 'testsnapshot', backingservice)
+      end
+    end
+  end
+
+  describe '#_check_snapshot_queue' do
+    let(:pool_helper) { double('pool') }
+    let(:backingservice) { {pool => pool_helper} }
+
+    before do
+      expect(subject).not_to be_nil
+      $backingservice = backingservice
+    end
+
+    it 'checks appropriate redis queues' do
+      expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot')
+      expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert')
+
+      subject._check_snapshot_queue(backingservice)
+    end
+  end
+end

spec/vmpooler/pool_manager_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'time'
 
-describe 'Pool Manager' do
+describe 'Vmpooler::PoolManager' do
   let(:logger) { double('logger') }
   let(:redis) { double('redis') }
   let(:metrics) { Vmpooler::DummyStatsd.new }
@@ -9,82 +9,397 @@ describe 'Pool Manager' do
   let(:pool) { 'pool1' }
   let(:vm) { 'vm1' }
   let(:timeout) { 5 }
-  let(:host) { double('host') }
+  let(:default_host) {
+    fake_vm = {}
+    fake_vm['name'] = 'vm1'
+    fake_vm['hostname'] = 'vm1'
+    fake_vm['template'] = 'pool1'
+    fake_vm['boottime'] = Time.now
+    fake_vm['powerstate'] = 'PoweredOn'
+
+    fake_vm
+  }
+  let(:host) { default_host }
+  let(:max_int) { 4611686018427387903 } # A really big integer (64bit)
 
   subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) }
 
-  describe '#_check_pending_vm' do
-    let(:pool_helper) { double('pool') }
-    let(:vsphere) { {pool => pool_helper} }
+  describe '#check_pending_vm' do
+    let(:backingservice) { double('backingservice') }
 
     before do
       expect(subject).not_to be_nil
-      $vsphere = vsphere
     end
 
-    context 'host not in pool' do
+    it 'calls _check_pending_vm' do
+      expect(Thread).to receive(:new).and_yield
+      expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice)
+
+      subject.check_pending_vm(vm, pool, timeout, backingservice)
+    end
+
+    it 'calls fail_pending_vm if an error is raised' do
+      expect(Thread).to receive(:new).and_yield
+      allow(logger).to receive(:log)
+      expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error')
+      expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout)
+
+      expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/)
+    end
+
+    it 'logs a message if an error is raised' do
+      expect(Thread).to receive(:new).and_yield
+      expect(logger).to receive(:log)
+      expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error')
+      allow(subject).to receive(:fail_pending_vm).with(vm, pool, timeout)
+
+      expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/)
+    end
+  end
+
+  describe '#_check_pending_vm' do
+    let(:backingservice) { double('backingservice') }
+
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    context 'VM does not exist' do
       it 'calls fail_pending_vm' do
-        allow(vsphere).to receive(:find_vm).and_return(nil)
+        allow(backingservice).to receive(:get_vm).and_return(nil)
         allow(redis).to receive(:hget)
-        subject._check_pending_vm(vm, pool, timeout, vsphere)
+        subject._check_pending_vm(vm, pool, timeout, backingservice)
       end
     end
 
-    context 'host is in pool' do
-      let(:vm_finder) { double('vm_finder') }
-      let(:tcpsocket) { double('TCPSocket') }
-
+    context 'VM is in pool and ready' do
       it 'calls move_pending_vm_to_ready' do
-        allow(subject).to receive(:open_socket).and_return(true)
-        allow(vsphere).to receive(:find_vm).and_return(vm_finder)
-        allow(vm_finder).to receive(:summary).and_return(nil)
+        allow(backingservice).to receive(:get_vm).with(vm).and_return(host)
+        allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(true)
+        allow(subject).to receive(:move_pending_vm_to_ready)
 
-        expect(vm_finder).to receive(:summary).once
-        expect(redis).not_to receive(:hget).with(String, 'clone')
+        subject._check_pending_vm(vm, pool, timeout, backingservice)
+      end
+    end
 
-        subject._check_pending_vm(vm, pool, timeout, vsphere)
+    context 'VM is in pool but not ready' do
+      it 'raises an error' do
+        allow(backingservice).to receive(:get_vm).with(vm).and_return(host)
+        allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(false)
+        allow(subject).to receive(:move_pending_vm_to_ready)
+
+        expect{subject._check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/VM is not ready/)
       end
     end
   end
 
+  describe '#remove_nonexistent_vm' do
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    it 'removes VM from pending in redis' do
+      allow(logger).to receive(:log)
+      expect(redis).to receive(:srem).with("vmpooler__pending__#{pool}", vm)
+
+      subject.remove_nonexistent_vm(vm, pool)
+    end
+
+    it 'logs msg' do
+      allow(redis).to receive(:srem)
+      expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
+
+      subject.remove_nonexistent_vm(vm, pool)
+    end
+  end
+
+  describe '#fail_pending_vm' do
+    before do
+      expect(subject).not_to be_nil
+      allow(logger).to receive(:log)
+    end
+
+    it 'takes no action if VM is not cloning' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil)
+
+      expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true)
+    end
+
+    it 'takes no action if VM is within timeout' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s)
+
+      expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true)
+    end
+
+    it 'moves VM to completed queue if VM has exceeded timeout and exists' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s)
+      expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
+
+      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true)
+    end
+
+    it 'logs message if VM has exceeded timeout and exists' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s)
+      allow(redis).to receive(:smove)
+      expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
+
+      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true)
+    end
+
+    it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s)
+      expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool)
+
+      expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(true)
+    end
+
+    it 'returns false and swallows error if an error is raised' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate')
+      expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(false)
+    end
+
+    it 'logs message if an error is raised' do
+      expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate')
+      expect(logger).to receive(:log).with('d', String)
+
+      subject.fail_pending_vm(vm, pool, timeout,true)
+    end
+  end
+
+  describe 'move_pending_vm_to_ready' do
+    before do
+      expect(subject).not_to be_nil
+      allow(logger).to receive(:log)
+      allow(Socket).to receive(:getaddrinfo)
+    end
+
+    context 'when hostname does not match VM name' do
+      it 'should not take any action' do
+        expect(logger).to receive(:log).exactly(0).times
+        expect(Socket).to receive(:getaddrinfo).exactly(0).times
+
+        bad_host = host
+        bad_host['hostname'] = 'different_name'
+
+        subject.move_pending_vm_to_ready(vm, pool, bad_host)
+      end
+    end
+
+    context 'when hostname matches VM name' do
+      it 'should use the pool in smove' do
+        allow(redis).to receive(:hget)
+        allow(redis).to receive(:hset)
+        expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm)
+
+        subject.move_pending_vm_to_ready(vm, pool, host)
+      end
+
+      it 'should log a message' do
+        allow(redis).to receive(:hget)
+        allow(redis).to receive(:hset)
+        allow(redis).to receive(:smove)
+        expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
+
+        subject.move_pending_vm_to_ready(vm, pool, host)
+      end
+
+      it 'should use clone start time to determine boot timespan' do
+        allow(redis).to receive(:smove)
+        expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s)
+        expect(redis).to receive(:hset).with(String,pool + ':' + vm,String)
+
+        subject.move_pending_vm_to_ready(vm, pool, host)
+      end
+
+      it 'should not determine boot timespan if clone start time not set' do
+        allow(redis).to receive(:smove)
+        expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil)
+        expect(redis).to receive(:hset).with(String,pool + ':' + vm,String).exactly(0).times
+
+        subject.move_pending_vm_to_ready(vm, pool, host)
+      end
+
+      it 'should raise error if clone start time is not parsable' do
+        expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate')
+
+        expect{subject.move_pending_vm_to_ready(vm, pool, host)}.to raise_error(/iamnotparsable_asdate/)
+      end
+    end
+  end
+
+  describe '#check_ready_vm' do
+    let(:backingservice) { double('backingservice') }
+    let (:ttl) { 5 }
+
+    before do
+      expect(subject).not_to be_nil
+    end
+
+    it 'calls _check_pending_vm' do
+      expect(Thread).to receive(:new).and_yield
+      expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice)
+
+      subject.check_ready_vm(vm, pool, ttl, backingservice)
+    end
+
+    it 'logs a message if an error is raised' do
+      expect(Thread).to receive(:new).and_yield
+      expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice).and_raise('an_error')
+      expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : an_error")
+
+      expect{subject.check_ready_vm(vm, pool, ttl, backingservice)}.to raise_error(/an_error/)
+    end
+  end
+
+
+
+
+
+  describe '#_check_ready_vm' do
+    let(:backingservice) { double('backingservice') }
+    let (:ttl) { 5 }
+
+    let(:config) {
+      config = {
+        'config': {}
+      }
+      # Use the configuration defaults
+      config[:config]['vm_checktime'] = 15
+
+      config
+    }
+
+    before do
+      expect(subject).not_to be_nil
+      allow(backingservice).to receive(:get_vm).and_return(host)
+      allow(logger).to receive(:log)
+    end
+
+    context 'a VM that does not need to be checked' do
+      it 'should do nothing' do
+        allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s)
+
+        subject._check_ready_vm(vm, pool, ttl, backingservice)
+      end
+    end
+
+    context 'a VM that does not exist' do
+      it 'should log a message' do
+        allow(backingservice).to receive(:get_vm).and_return(nil)
+        allow(redis).to receive(:srem)
+        expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue")
+
+        subject._check_ready_vm(vm, pool, ttl, backingservice)
+      end
+
+      it 'should remove the VM from the ready queue' do
+        allow(backingservice).to receive(:get_vm).and_return(nil)
+        allow(redis).to receive(:srem).with("vmpooler__ready__#{pool}", vm)
+
+        subject._check_ready_vm(vm, pool, ttl, backingservice)
+      end
+    end
+
+    context 'an old VM' do
+      let (:host) {
+        fake_vm = default_host
+        fake_vm['boottime'] = Time.new(2001,1,1)
+        fake_vm
+      }
+
+      context 'with a TTL of zero' do
+        it 'should do nothing' do
+          #allow(backingservice).to receive(:get_vm).and_return(host)
+          allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s)
+
+          subject._check_ready_vm(vm, pool, 0, backingservice)
+        end
+      end
+
+      context 'with a TTL longer than the VM lifetime' do
+        it 'should do nothing' do
+         # allow(backingservice).to receive(:get_vm).and_return(host)
+          allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s)
+
+          subject._check_ready_vm(vm, pool, max_int, backingservice)
+        end
+      end
+
+      context 'with a TTL shorter than the VM lifetime' do
+        it 'should move the VM to the completed queue' do
+          #allow(backingservice).to receive(:get_vm).and_return(host)
+          allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s)
+          expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)
+
+          subject._check_ready_vm(vm, pool, 1, backingservice)
+        end
+
+        it 'should log a message' do
+          #allow(backingservice).to receive(:get_vm).and_return(host)
+          allow(redis).to receive(:smove)
+          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
+
+          subject._check_ready_vm(vm, pool, ttl, backingservice)
+        end
+      end
+    end
+
+    context 'a VM that has not previously been checked' do
+      before do
+        allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(nil)
+      end
+
+      it 'sets the last check time' do
+        expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'check', Time)
+        allow(backingservice).to receive(:is_vm_ready?).and_return(true)
+
+        subject._check_ready_vm(vm, pool, ttl, backingservice)
+      end
+
+    end
+
+    # TODO Need to test everything inside the check if statement
+
+  end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
   describe '#move_vm_to_ready' do
     before do
       expect(subject).not_to be_nil
     end
 
     context 'a host without correct summary' do
-      it 'does nothing when summary is nil' do
-        allow(host).to receive(:summary).and_return nil
-        subject.move_pending_vm_to_ready(vm, pool, host)
-      end
-
-      it 'does nothing when guest is nil' do
-        allow(host).to receive(:summary).and_return true
-        allow(host).to receive_message_chain(:summary, :guest).and_return nil
-        subject.move_pending_vm_to_ready(vm, pool, host)
-      end
-
       it 'does nothing when hostName is nil' do
-        allow(host).to receive(:summary).and_return true
-        allow(host).to receive_message_chain(:summary, :guest).and_return true
-        allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return nil
+        host['hostname'] = nil
+
         subject.move_pending_vm_to_ready(vm, pool, host)
       end
 
       it 'does nothing when hostName does not match vm' do
-        allow(host).to receive(:summary).and_return true
-        allow(host).to receive_message_chain(:summary, :guest).and_return true
-        allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return 'adifferentvm'
+        host['hostname'] = 'adifferentvm'
+
         subject.move_pending_vm_to_ready(vm, pool, host)
       end
     end
 
     context 'a host with proper summary' do
       before do
-        allow(host).to receive(:summary).and_return true
-        allow(host).to receive_message_chain(:summary, :guest).and_return true
-        allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return vm
-
         allow(redis).to receive(:hget)
         allow(redis).to receive(:smove)
         allow(redis).to receive(:hset)
@@ -145,24 +460,24 @@ describe 'Pool Manager' do
 
   describe '#_check_running_vm' do
     let(:pool_helper) { double('pool') }
-    let(:vsphere) { {pool => pool_helper} }
+    let(:backingservice) { {pool => pool_helper} }
 
     before do
       expect(subject).not_to be_nil
-      $vsphere = vsphere
+      $backingservice = backingservice
     end
 
     it 'does nothing with nil host' do
-      allow(vsphere).to receive(:find_vm).and_return(nil)
+      allow(backingservice).to receive(:get_vm).and_return(nil)
       expect(redis).not_to receive(:smove)
-      subject._check_running_vm(vm, pool, timeout, vsphere)
+      subject._check_running_vm(vm, pool, timeout, backingservice)
     end
 
     context 'valid host' do
       let(:vm_host) { double('vmhost') }
 
       it 'does not move vm when not poweredOn' do
-        allow(vsphere).to receive(:find_vm).and_return vm_host
+        allow(backingservice).to receive(:get_vm).and_return vm_host
         allow(vm_host).to receive(:runtime).and_return true
         allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff'
 
@@ -170,11 +485,11 @@ describe 'Pool Manager' do
         expect(redis).not_to receive(:smove)
         expect(logger).not_to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
 
-        subject._check_running_vm(vm, pool, timeout, vsphere)
+        subject._check_running_vm(vm, pool, timeout, backingservice)
       end
 
       it 'moves vm when poweredOn, but past TTL' do
-        allow(vsphere).to receive(:find_vm).and_return vm_host
+        allow(backingservice).to receive(:get_vm).and_return vm_host
         allow(vm_host).to receive(:runtime).and_return true
         allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn'
 
@@ -182,7 +497,7 @@ describe 'Pool Manager' do
         expect(redis).to receive(:smove)
         expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} hours")
 
-        subject._check_running_vm(vm, pool, timeout, vsphere)
+        subject._check_running_vm(vm, pool, timeout, backingservice)
       end
     end
   end
@@ -209,7 +524,7 @@ describe 'Pool Manager' do
 
   describe '#_check_pool' do
     let(:pool_helper) { double('pool') }
-    let(:vsphere) { {pool => pool_helper} }
+    let(:backingservice) { {pool => pool_helper} }
     let(:config) { {
       config: { task_limit: 10 },
       pools: [ {'name' => 'pool1', 'size' => 5} ]
@@ -217,7 +532,7 @@ describe 'Pool Manager' do
 
     before do
       expect(subject).not_to be_nil
-      $vsphere = vsphere
+      $backingservice = backingservice
       allow(logger).to receive(:log)
       allow(pool_helper).to receive(:find_folder)
       allow(redis).to receive(:smembers).with('vmpooler__pending__pool1').and_return([])
@@ -238,14 +553,14 @@ describe 'Pool Manager' do
         allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0)
 
         expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
-        subject._check_pool(config[:pools][0], vsphere)
+        subject._check_pool(config[:pools][0], backingservice)
       end
     end
   end
 
   describe '#_stats_running_ready' do
     let(:pool_helper) { double('pool') }
-    let(:vsphere) { {pool => pool_helper} }
+    let(:backingservice) { {pool => pool_helper} }
     let(:metrics) { Vmpooler::DummyStatsd.new }
     let(:config) { {
       config: { task_limit: 10 },
@@ -255,7 +570,7 @@ describe 'Pool Manager' do
 
     before do
       expect(subject).not_to be_nil
-      $vsphere = vsphere
+      $backingservice = backingservice
       allow(logger).to receive(:log)
       allow(pool_helper).to receive(:find_folder)
       allow(redis).to receive(:smembers).and_return([])
@@ -275,7 +590,7 @@ describe 'Pool Manager' do
 
         expect(metrics).to receive(:gauge).with('ready.pool1', 1)
         expect(metrics).to receive(:gauge).with('running.pool1', 5)
-        subject._check_pool(config[:pools][0], vsphere)
+        subject._check_pool(config[:pools][0], backingservice)
       end
 
       it 'increments metrics when ready with 0 when pool empty' do
@@ -286,76 +601,8 @@ describe 'Pool Manager' do
 
         expect(metrics).to receive(:gauge).with('ready.pool1', 0)
         expect(metrics).to receive(:gauge).with('running.pool1', 5)
-        subject._check_pool(config[:pools][0], vsphere)
+        subject._check_pool(config[:pools][0], backingservice)
       end
     end
   end
-
-  describe '#_create_vm_snapshot' do
-    let(:snapshot_manager) { 'snapshot_manager' }
-    let(:pool_helper) { double('snapshot_manager') }
-    let(:vsphere) { {snapshot_manager => pool_helper} }
-
-    before do
-      expect(subject).not_to be_nil
-      $vsphere = vsphere
-    end
-
-    context '(valid host)' do
-      let(:vm_host) { double('vmhost') }
-
-      it 'creates a snapshot' do
-        expect(vsphere).to receive(:find_vm).and_return vm_host
-        expect(logger).to receive(:log)
-        expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion)
-        expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s)
-        expect(logger).to receive(:log)
-
-        subject._create_vm_snapshot('testvm', 'testsnapshot', vsphere)
-      end
-    end
-  end
-
-  describe '#_revert_vm_snapshot' do
-    let(:snapshot_manager) { 'snapshot_manager' }
-    let(:pool_helper) { double('snapshot_manager') }
-    let(:vsphere) { {snapshot_manager => pool_helper} }
-
-    before do
-      expect(subject).not_to be_nil
-      $vsphere = vsphere
-    end
-
-    context '(valid host)' do
-      let(:vm_host) { double('vmhost') }
-      let(:vm_snapshot) { double('vmsnapshot') }
-
-      it 'reverts a snapshot' do
-        expect(vsphere).to receive(:find_vm).and_return vm_host
-        expect(vsphere).to receive(:find_snapshot).and_return vm_snapshot
-        expect(logger).to receive(:log)
-        expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion)
-        expect(logger).to receive(:log)
-
-        subject._revert_vm_snapshot('testvm', 'testsnapshot', vsphere)
-      end
-    end
-  end
-
-  describe '#_check_snapshot_queue' do
-    let(:pool_helper) { double('pool') }
-    let(:vsphere) { {pool => pool_helper} }
-
-    before do
-      expect(subject).not_to be_nil
-      $vsphere = vsphere
-    end
-
-    it 'checks appropriate redis queues' do
-      expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot')
-      expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert')
-
-      subject._check_snapshot_queue(vsphere)
-    end
-  end
 end

vmpooler

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'rubygems' unless defined?(Gem)
 require 'lib/vmpooler'
 
-config = Vmpooler.config
+config = Vmpooler.config(ENV['VMPOOLER_CONFIG'] || 'vmpooler.yaml')
 redis_host = config[:redis]['server']
 logger_file = config[:config]['logfile']
 
@@ -26,4 +26,11 @@ manager = Thread.new {
   ).execute!
 }
 
+if ENV['VMPOOLER_DEBUG']
+  trap("INT") {
+    puts "Shutting down."
+    [api, manager].each { |t| t.exit }
+  }
+end
+
 [api, manager].each { |t| t.join }

vmpooler-new.yaml

@@ -0,0 +1,418 @@
+---
+# TODO KILL ME
+:vsphere:
+  server: '127.0.0.1'
+  username: 'vmpooler'
+  password: 'swimsw1msw!m'
+
+
+:backingservice:
+# :backingservice:
+#
+# This section contains the backing services for VMs and Pools
+# The currently supported backing services are:
+# - vsphere
+# - dummy
+
+# :vsphere:
+#
+# This section contains the server hostname and authentication credentials
+# needed for vmpooler to connect to VMware vSphere.
+#
+# Available configuration parameters:
+#
+#   - server
+#     The FQDN hostname of the VMware vSphere server.
+#     (required)
+#
+#   - username
+#     The username used to authenticate VMware vSphere.
+#     (required)
+#
+#   - password
+#     The password used to authenticate VMware vSphere.
+#     (required)
+
+# Example:
+
+  :vsphere:
+    server: '127.0.0.1'
+    username: 'vmpooler'
+    password: 'swimsw1msw!m'
+
+# :dummy:
+#
+# The dummy backing service is a simple text file service that can be used
+# to test vmpooler operations in a development or test environment
+#
+# Available configuration parameters:
+#
+#   - filename
+#     The filename used to store the backing text file
+
+# Example:
+
+  :dummy:
+    filename: 'C:/source/vmpooler/dummy-backing.yaml'
+
+    # createvm_max_time: 30
+    # createvm_fail_percent: 5
+    # migratevm_couldmove_percent: 25
+    # migratevm_max_time: 10
+    # migratevm_fail_percent: 5
+    # vmready_fail_percent: 5
+    # destroyvm_max_shutdown_time: 10
+    # destroyvm_fail_percent: 50
+    # destroyvm_max_time: 10
+    # getvm_poweroff_percent: 10
+    # getvm_rename_percent: 10
+
+# :redis:
+#
+# This section contains the server hostname and authentication credentials
+# needed for vmpooler to connect to Redis.
+#
+# Available configuration parameters:
+#
+#   - server
+#     The FQDN hostname of the Redis server.
+#     (optional; default: 'localhost')
+#
+#   - username
+#     The username used to authenticate Redis.
+#     (optional)
+#
+#   - password
+#     The password used to authenticate Redis.
+#     (optional)
+#
+#   - data_ttl
+#     How long (in hours) to retain metadata in Redis after VM destruction.
+#     (optional; default: '168')
+
+# Example:
+
+:redis:
+  server: 'localhost'
+
+
+  # :graphs:
+  #
+  # This section contains the server and prefix information for a graphite-
+  # compatible web front-end where graphs may be viewed. This is used by the
+  # vmpooler dashboard to retrieve statistics and graphs for a given instance.
+  #
+  # NOTE: This is not the endpoint for publishing metrics data. See `graphite:`
+  #       and `statsd:` below.
+  #
+  # NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be
+  #       consulted for `server`/`prefix` information to use in locating a
+  #       graph server for our dashboard.  `graphs:` is recommended over
+  #       `graphite:`
+  #
+  #
+  # Available configuration parameters:
+  #
+  #
+  #   - server
+  #     The FQDN hostname of the statsd daemon.
+  #     (required)
+  #
+  #   - prefix
+  #     The prefix to use while storing statsd data.
+  #     (optional; default: 'vmpooler')
+
+  # :statsd:
+  #
+  # This section contains the connection information required to store
+  # historical data via statsd.  This is mutually exclusive with graphite
+  # and takes precedence.
+  #
+  # Available configuration parameters:
+  #
+  #   - server
+  #     The FQDN hostname of the statsd daemon.
+  #     (required)
+  #
+  #   - prefix
+  #     The prefix to use while storing statsd data.
+  #     (optional; default: 'vmpooler')
+  #
+  #   - port
+  #     The UDP port to communicate with the statsd daemon.
+  #     (optional; default: 8125)
+
+  # Example:
+
+  :statsd:
+    server: 'localhost'
+    prefix: 'vmpooler'
+    port: 8125
+
+# :graphite:
+#
+# This section contains the connection information required to store
+# historical data in an external Graphite database.  This is mutually exclusive
+# with statsd.
+#
+# Available configuration parameters:
+#
+#   - server
+#     The FQDN hostname of the Graphite server.
+#     (required)
+#
+#   - prefix
+#     The prefix to use while storing Graphite data.
+#     (optional; default: 'vmpooler')
+#
+#   - port
+#     The TCP port to communicate with the graphite server.
+#     (optional; default: 2003)
+
+# Example:
+
+:graphite:
+  server: 'graphite.company.com'
+
+# :auth:
+#
+# This section contains information related to authenticating users
+# for token operations.
+#
+# Supported Auth Providers:
+#   - Dummy
+#   - LDAP
+#
+# - Dummy Auth Provider
+# The Dummy Authentication provider should only be used during development or testing
+# If the Username and Password are different then validation succeeds
+# If the Username and Password are the same then validation fails
+#
+# Example:
+# :auth:
+#   provider: 'dummy'
+#
+# - LDAP Auth Provider
+# The LDAP Authentication provider will validate usernames and passwords against an
+# existing LDAP service
+#
+# Available configuration parameters:
+#
+#   - host
+#     The FQDN hostname of the LDAP server.
+#
+#   - port
+#     The port used to connect to the LDAP service.
+#     (optional; default: '389')
+#
+#   - base
+#     The base DN used for LDAP searches.
+#
+#   - user_object
+#     The LDAP object-type used to designate a user object.
+#
+# Example:
+# :auth:
+#   provider: 'ldap'
+#   :ldap:
+#     host: 'localhost'
+#     port: 389
+#     base: 'ou=users,dc=company,dc=com'
+#     user_object: 'uid'
+
+:auth:
+  provider: 'dummy'
+
+# :tagfilter:
+#
+# Filter tags by regular expression.
+
+# Example:
+#
+# This example demonstrates discarding everything after a '/' character for
+# the 'url' tag, transforming 'foo.com/something.html' to 'foo.com'.
+
+:tagfilter:
+  url: '(.*)\/'
+
+# :config:
+#
+# This section contains global configuration information.
+#
+# Available configuration parameters:
+#
+#   - site_name
+#     The name of your deployment.
+#     (optional; default: 'vmpooler')
+#
+#   - logfile
+#     The path to vmpooler's log file.
+#     (optional; default: '/var/log/vmpooler.log')
+#
+#   - clone_target TODO
+#     The target cluster VMs are cloned into (host with least VMs chosen)
+#     (optional; default: same cluster/host as origin template)
+#
+#   - task_limit TODO
+#     The number of concurrent VMware vSphere tasks to perform.
+#     (optional; default: '10')
+#
+#   - timeout
+#     How long (in minutes) before marking a clone as 'failed' and retrying.
+#     (optional; default: '15')
+#
+#   - vm_checktime
+#     How often (in minutes) to check the sanity of VMs in 'ready' queues.
+#     (optional; default: '15')
+#
+#   - vm_lifetime
+#     How long (in hours) to keep VMs in 'running' queues before destroying.
+#     (optional; default: '24')
+#
+#   - vm_lifetime_auth
+#     Same as vm_lifetime, but applied if a valid authentication token is
+#     included during the request.
+#
+#   - allowed_tags
+#     If set, restricts tags to those specified in this array.
+#
+#   - domain
+#     If set, returns a top-level 'domain' JSON key in POST requests
+#
+#   - prefix
+#     If set, prefixes all created VMs with this string.  This should include
+#     a separator.
+#     (optional; default: '')
+#
+#   - migration_limit
+#     When set to any value greater than 0 enable VM migration at checkout.
+#     When enabled this capability will evaluate a VM for migration when it is requested
+#     in an effort to maintain a more even distribution of load across compute resources.
+#     The migration_limit ensures that no more than n migrations will be evaluated at any one time
+#     and greatly reduces the possibilty of VMs ending up bunched together on a particular host.
+
+# Example:
+
+:config:
+  site_name: 'vmpooler'
+  logfile: 'c:/temp/vmpooler.log'
+  task_limit: 10
+  timeout: 15
+  vm_checktime: 15
+  vm_lifetime: 12
+  vm_lifetime_auth: 24
+  # allowed_tags:
+  #   - 'created_by'
+  #   - 'project'
+  domain: 'company.com'
+  prefix: 'poolvm-'
+  migration_limit: 5
+
+# :pools:
+#
+# This section contains a list of virtual machine 'pools' for vmpooler to
+# create and maintain.
+#
+# Available configuration parameters (per-pool):
+#
+#   - name
+#     The name of the pool.
+#     (required)
+#
+#   - alias
+#     Other names this pool can be requested as.
+#     (optional)
+#
+#   - backingservice
+#     The backing service which will be used to provision this pool.  Default is vsphere
+#     (optional)
+#
+#   - timeout
+#     How long (in minutes) before marking a clone as 'failed' and retrying.
+#     This setting overrides any globally-configured timeout setting.
+#     (optional; default: '15')
+#
+#   - ready_ttl
+#     How long (in minutes) to keep VMs in 'ready' queues before destroying.
+#     (optional)
+#
+#   - template
+#     The template or virtual machine target to spawn clones from.
+#     (required)
+#
+#   - size
+#     The number of waiting VMs to keep in a pool.
+#     (required)
+
+#   VSphere Backing Service configuration Options
+#
+#   - folder
+#     The vSphere 'folder' destination for spawned clones.
+#     (required)
+#
+#   - datastore
+#     The vSphere 'datastore' destination for spawned clones.
+#     (required)
+#
+#   - clone_target
+#     Per-pool option to override the global 'clone_target' cluster.
+#     (optional)
+#
+
+# Example:
+
+:pools:
+  # VSPHERE
+  # - name: 'debian-7-i386'
+  #   alias: [ 'debian-7-32' ]
+  #   template: 'Templates/debian-7-i386'
+  #   folder: 'Pooled VMs/debian-7-i386'
+  #   datastore: 'vmstorage'
+  #   size: 5
+  #   timeout: 15
+  #   ready_ttl: 1440
+
+  # - name: 'debian-7-x86_64'
+  #   alias: [ 'debian-7-64', 'debian-7-amd64' ]
+  #   template: 'Templates/debian-7-x86_64'
+  #   folder: 'Pooled VMs/debian-7-x86_64'
+  #   datastore: 'vmstorage'
+  #   size: 2
+  #   timeout: 15
+  #   ready_ttl: 1440
+
+  # DUMMY
+  - name: 'debian-7-i386'
+    template: 'test1'
+    backingservice: dummy
+    size: 1
+    timeout: 15
+    ready_ttl: 1440
+  
+  - name: 'debian-7-x86_64'
+    template: 'test1'
+    backingservice: dummy
+    size: 1
+    timeout: 15
+    ready_ttl: 1440
+
+  # - name: 'win-2008r2-x86_64'
+  #   template: 'test1'
+  #   backingservice: dummy
+  #   size: 5
+  #   timeout: 15
+  #   ready_ttl: 1440
+
+  # - name: 'win-2016-x86_64'
+  #   template: 'test1'
+  #   backingservice: dummy
+  #   size: 5
+  #   timeout: 15
+  #   ready_ttl: 1440
+
+  # - name: 'win-2012r2-x86_64'
+  #   template: 'test1'
+  #   backingservice: dummy
+  #   size: 5
+  #   timeout: 15
+  #   ready_ttl: 1440

vmpooler.yaml.example

@@ -137,8 +137,22 @@
 # This section contains information related to authenticating users
 # for token operations.
 #
-# Currently the only supported provider is LDAP; the following parameters
-# will all be under an ':ldap:' subsection (see example below).
+# Supported Auth Providers:
+#   - Dummy
+#   - LDAP
+#
+# - Dummy Auth Provider
+# The Dummy Authentication provider should only be used during development or testing
+# If the Username and Password are different then validation succeeds
+# If the Username and Password are the same then validation fails
+#
+# Example:
+# :auth:
+#   provider: 'dummy'
+#
+# - LDAP Auth Provider
+# The LDAP Authentication provider will validate usernames and passwords against an
+# existing LDAP service
 #
 # Available configuration parameters:
 #
@@ -154,8 +168,15 @@
 #
 #   - user_object
 #     The LDAP object-type used to designate a user object.
-
+#
 # Example:
+# :auth:
+#   provider: 'ldap'
+#   :ldap:
+#     host: 'localhost'
+#     port: 389
+#     base: 'ou=users,dc=company,dc=com'
+#     user_object: 'uid'
 
 :auth:
   provider: 'ldap'