Add support for migrating VMs to pool_manager.

This commit adds a capability to pool_manager to migrate VMs placed in the migrating queue. When a VM is checked out an entry is created in vmpooler__migrating. The existing process for evaluating VM states executes the migrate_vm method for the provided VM, and removes it from the queue. The least used compatible host for the provided VM is selected and, if necessary, a migration to the lesser used host is performed. Migration time and time from the task being queued until completion are both tracked with the redis VM object in 'migration_time' and 'checkout_to_migration'. The migration time is logged in the vmpooler.log, or the VM is reported as not requiring migration. Without this change VMs are not evaluated for checkout at request time.

Add a method to wrap find_vm and find_vm_heavy in order to allow a
single operation to be performed that does both.

This commit also adds support for a configuration setting called
migration_limit that makes migration at checkout optional. Additionally,
logging is added to report a VM parent host when it is checked out. Without this
change vmpooler assumes that migration at checkout is always enabled.
If this setting is not present, or if the setting is 0, then migration
at checkout will be disabled. If the setting is greater than 0 then that
setting will be used to enforce a limit for the number of simultaneous
migrations that will be evaluated.

Documentation of this configuration option is added to the
vmpooler.yaml.example file.

lib/vmpooler/pool_manager.rb

@@ -455,6 +455,55 @@ module Vmpooler
       end
     end
 
+    def find_vsphere_pool_vm(pool, vm)
+      $vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm]
+    end
+
+    def migration_enabled?(migration_limit)
+      # Returns migration_limit setting when enabled
+      return false if migration_limit == 0 or not migration_limit
+      migration_limit if migration_limit >= 1
+    end
+
+    def migrate_vm(vm, pool)
+      Thread.new do
+        _migrate_vm(vm, pool)
+      end
+    end
+
+    def _migrate_vm(vm, pool)
+      $redis.srem('vmpooler__migrating__' + pool, vm)
+      vm_object = find_vsphere_pool_vm(pool, vm)
+      parent_host = vm_object.summary.runtime.host
+      parent_host_name = parent_host.name
+      migration_limit = migration_enabled? $config[:config]['migration_limit']
+
+      if not migration_limit
+        $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}")
+      else
+        migration_count = $redis.smembers('vmpooler__migration').size
+        if migration_count >= migration_limit
+          $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
+        else
+          $redis.sadd('vmpooler__migration', vm)
+          host = $vsphere[pool].find_least_used_compatible_host(vm_object)
+          if host == parent_host
+            $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
+          else
+            start = Time.now
+            $vsphere[pool].migrate_vm_host(vm_object, host)
+            finish = '%.2f' % (Time.now - start)
+            $metrics.timing("migrate.#{vm['template']}", finish)
+            checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout')))
+            $redis.hset('vmpooler__vm__' + vm, 'migration_time', finish)
+            $redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration)
+            $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds")
+          end
+          $redis.srem('vmpooler__migration', vm)
+        end
+      end
+    end
+
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
@@ -480,7 +529,8 @@ module Vmpooler
             (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) &&
             (! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) &&
             (! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) &&
-            (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']))
+            (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) &&
+            (! $redis.sismember('vmpooler__migrating__' + pool['name'], vm['name']))
 
             $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
 
@@ -555,6 +605,17 @@ module Vmpooler
         end
       end
 
+      # MIGRATIONS
+      $redis.smembers('vmpooler__migrating__' + pool['name']).each do |vm|
+        if inventory[vm]
+          begin
+            migrate_vm(vm, pool['name'])
+          rescue => err
+            $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
+          end
+        end
+      end
+
       # REPOPULATE
       ready = $redis.scard('vmpooler__ready__' + pool['name'])
       total = $redis.scard('vmpooler__pending__' + pool['name']) + ready

lib/vmpooler/vsphere_helper.rb

@@ -17,7 +17,7 @@ module Vmpooler
     end
 
     def add_disk(vm, size, datastore)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       return false unless size.to_i > 0
 
@@ -67,14 +67,14 @@ module Vmpooler
     end
 
     def find_datastore(datastorename)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       datacenter = @connection.serviceInstance.find_datacenter
       datacenter.find_datastore(datastorename)
     end
 
     def find_device(vm, deviceName)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       vm.config.hardware.device.each do |device|
         return device if device.deviceInfo.label == deviceName
@@ -84,7 +84,7 @@ module Vmpooler
     end
 
     def find_disk_controller(vm)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       devices = find_disk_devices(vm)
 
@@ -98,7 +98,7 @@ module Vmpooler
     end
 
     def find_disk_devices(vm)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       devices = {}
 
@@ -126,7 +126,7 @@ module Vmpooler
     end
 
     def find_disk_unit_number(vm, controller)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       used_unit_numbers = []
       available_unit_numbers = []
@@ -151,7 +151,7 @@ module Vmpooler
     end
 
     def find_folder(foldername)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       datacenter = @connection.serviceInstance.find_datacenter
       base = datacenter.vmFolder
@@ -211,7 +211,7 @@ module Vmpooler
       (memory_usage.to_f / memory_size.to_f) * 100
     end
 
-    def vsphere_connection_alive?(connection)
+    def ensure_connected(connection)
       begin
         connection.serviceInstance.CurrentTime
       rescue
@@ -220,7 +220,7 @@ module Vmpooler
     end
 
     def find_least_used_host(cluster)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       cluster_object = find_cluster(cluster)
       target_hosts = get_cluster_host_utilization(cluster_object)
@@ -243,7 +243,7 @@ module Vmpooler
     end
 
     def find_least_used_compatible_host(vm)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       source_host = vm.summary.runtime.host
       model = get_host_cpu_arch_version(source_host)
@@ -257,7 +257,7 @@ module Vmpooler
     end
 
     def find_pool(poolname)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       datacenter = @connection.serviceInstance.find_datacenter
       base = datacenter.hostFolder
@@ -286,13 +286,13 @@ module Vmpooler
     end
 
     def find_vm(vmname)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname)
     end
 
     def find_vm_heavy(vmname)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       vmname = vmname.is_a?(Array) ? vmname : [vmname]
       containerView = get_base_vm_container_from @connection
@@ -342,7 +342,7 @@ module Vmpooler
     end
 
     def find_vmdks(vmname, datastore)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       disks = []
 
@@ -361,7 +361,7 @@ module Vmpooler
     end
 
     def get_base_vm_container_from(connection)
-      vsphere_connection_alive? @connection
+      ensure_connected @connection
 
       viewManager = connection.serviceContent.viewManager
       viewManager.CreateContainerView(

vmpooler.yaml.example

@@ -225,6 +225,13 @@
 #     If set, prefixes all created VMs with this string.  This should include
 #     a separator.
 #     (optional; default: '')
+#
+#   - migration_limit
+#     When set to any value greater than 0 enable VM migration at checkout.
+#     When enabled this capability will evaluate a VM for migration when it is requested
+#     in an effort to maintain a more even distribution of load across compute resources.
+#     The migration_limit ensures that no more than n migrations will be evaluated at any one time
+#     and greatly reduces the possibilty of VMs ending up bunched together on a particular host.
 
 # Example: