Add support for migrating VMs to pool_manager.

This commit adds a capability to pool_manager to migrate VMs placed in the migrating queue. When a VM is checked out an entry is created in vmpooler__migrating. The existing process for evaluating VM states executes the migrate_vm method for the provided VM, and removes it from the queue. The least used compatible host for the provided VM is selected and, if necessary, a migration to the lesser used host is performed. Migration time and time from the task being queued until completion are both tracked with the redis VM object in 'migration_time' and 'checkout_to_migration'. The migration time is logged in the vmpooler.log, or the VM is reported as not requiring migration. Without this change VMs are not evaluated for checkout at request time.

Add a method to wrap find_vm and find_vm_heavy in order to allow a
single operation to be performed that does both.

This commit also adds support for a configuration setting called
migration_limit that makes migration at checkout optional. Additionally,
logging is added to report a VM parent host when it is checked out. Without this
change vmpooler assumes that migration at checkout is always enabled.
If this setting is not present, or if the setting is 0, then migration
at checkout will be disabled. If the setting is greater than 0 then that
setting will be used to enforce a limit for the number of simultaneous
migrations that will be evaluated.

Documentation of this configuration option is added to the
vmpooler.yaml.example file.

lib/vmpooler/pool_manager.rb

@@ -455,6 +455,55 @@ module Vmpooler
       end
     end
 
+    def find_vsphere_pool_vm(pool, vm)
+      $vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm]
+    end
+
+    def migration_enabled?(migration_limit)
+      # Returns migration_limit setting when enabled
+      return false if migration_limit == 0 or not migration_limit
+      migration_limit if migration_limit >= 1
+    end
+
+    def migrate_vm(vm, pool)
+      Thread.new do
+        _migrate_vm(vm, pool)
+      end
+    end
+
+    def _migrate_vm(vm, pool)
+      $redis.srem('vmpooler__migrating__' + pool, vm)
+      vm_object = find_vsphere_pool_vm(pool, vm)
+      parent_host = vm_object.summary.runtime.host
+      parent_host_name = parent_host.name
+      migration_limit = migration_enabled? $config[:config]['migration_limit']
+
+      if not migration_limit
+        $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}")
+      else
+        migration_count = $redis.smembers('vmpooler__migration').size
+        if migration_count >= migration_limit
+          $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
+        else
+          $redis.sadd('vmpooler__migration', vm)
+          host = $vsphere[pool].find_least_used_compatible_host(vm_object)
+          if host == parent_host
+            $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
+          else
+            start = Time.now
+            $vsphere[pool].migrate_vm_host(vm_object, host)
+            finish = '%.2f' % (Time.now - start)
+            $metrics.timing("migrate.#{vm['template']}", finish)
+            checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout')))
+            $redis.hset('vmpooler__vm__' + vm, 'migration_time', finish)
+            $redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration)
+            $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds")
+          end
+          $redis.srem('vmpooler__migration', vm)
+        end
+      end
+    end
+
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
@@ -480,7 +529,8 @@ module Vmpooler
             (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) &&
             (! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) &&
             (! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) &&
-            (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']))
+            (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) &&
+            (! $redis.sismember('vmpooler__migrating__' + pool['name'], vm['name']))
 
             $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
 
@@ -555,6 +605,17 @@ module Vmpooler
         end
       end
 
+      # MIGRATIONS
+      $redis.smembers('vmpooler__migrating__' + pool['name']).each do |vm|
+        if inventory[vm]
+          begin
+            migrate_vm(vm, pool['name'])
+          rescue => err
+            $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
+          end
+        end
+      end
+
       # REPOPULATE
       ready = $redis.scard('vmpooler__ready__' + pool['name'])
       total = $redis.scard('vmpooler__pending__' + pool['name']) + ready