From 58a548bc90cc30c761e0e6b233e74c8deca0501b Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Fri, 14 Oct 2016 12:55:36 -0700 Subject: [PATCH] Add support for migrating VMs to pool_manager. This commit adds a capability to pool_manager to migrate VMs placed in the migrating queue. When a VM is checked out an entry is created in vmpooler__migrating. The existing process for evaluating VM states executes the migrate_vm method for the provided VM, and removes it from the queue. The least used compatible host for the provided VM is selected and, if necessary, a migration to the lesser used host is performed. Migration time and time from the task being queued until completion are both tracked with the redis VM object in 'migration_time' and 'checkout_to_migration'. The migration time is logged in the vmpooler.log, or the VM is reported as not requiring migration. Without this change VMs are not evaluated for checkout at request time. Add a method to wrap find_vm and find_vm_heavy in order to allow a single operation to be performed that does both. This commit also adds support for a configuration setting called migration_limit that makes migration at checkout optional. Additionally, logging is added to report a VM parent host when it is checked out. Without this change vmpooler assumes that migration at checkout is always enabled. If this setting is not present, or if the setting is 0, then migration at checkout will be disabled. If the setting is greater than 0 then that setting will be used to enforce a limit for the number of simultaneous migrations that will be evaluated. Documentation of this configuration option is added to the vmpooler.yaml.example file. --- lib/vmpooler/pool_manager.rb | 63 +++++++++++++++++++++++++++++++++- lib/vmpooler/vsphere_helper.rb | 30 ++++++++-------- vmpooler.yaml.example | 7 ++++ 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 53895bb..7248be0 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -455,6 +455,55 @@ module Vmpooler end end + def find_vsphere_pool_vm(pool, vm) + $vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm] + end + + def migration_enabled?(migration_limit) + # Returns migration_limit setting when enabled + return false if migration_limit == 0 or not migration_limit + migration_limit if migration_limit >= 1 + end + + def migrate_vm(vm, pool) + Thread.new do + _migrate_vm(vm, pool) + end + end + + def _migrate_vm(vm, pool) + $redis.srem('vmpooler__migrating__' + pool, vm) + vm_object = find_vsphere_pool_vm(pool, vm) + parent_host = vm_object.summary.runtime.host + parent_host_name = parent_host.name + migration_limit = migration_enabled? $config[:config]['migration_limit'] + + if not migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + else + migration_count = $redis.smembers('vmpooler__migration').size + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") + else + $redis.sadd('vmpooler__migration', vm) + host = $vsphere[pool].find_least_used_compatible_host(vm_object) + if host == parent_host + $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") + else + start = Time.now + $vsphere[pool].migrate_vm_host(vm_object, host) + finish = '%.2f' % (Time.now - start) + $metrics.timing("migrate.#{vm['template']}", finish) + checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout'))) + $redis.hset('vmpooler__vm__' + vm, 'migration_time', finish) + $redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration) + $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds") + end + $redis.srem('vmpooler__migration', vm) + end + end + end + def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") @@ -480,7 +529,8 @@ module Vmpooler (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) && - (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) + (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) && + (! $redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])) $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name']) @@ -555,6 +605,17 @@ module Vmpooler end end + # MIGRATIONS + $redis.smembers('vmpooler__migrating__' + pool['name']).each do |vm| + if inventory[vm] + begin + migrate_vm(vm, pool['name']) + rescue => err + $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") + end + end + end + # REPOPULATE ready = $redis.scard('vmpooler__ready__' + pool['name']) total = $redis.scard('vmpooler__pending__' + pool['name']) + ready diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index 6b6f57d..6e65e30 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -17,7 +17,7 @@ module Vmpooler end def add_disk(vm, size, datastore) - vsphere_connection_alive? @connection + ensure_connected @connection return false unless size.to_i > 0 @@ -67,14 +67,14 @@ module Vmpooler end def find_datastore(datastorename) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter datacenter.find_datastore(datastorename) end def find_device(vm, deviceName) - vsphere_connection_alive? @connection + ensure_connected @connection vm.config.hardware.device.each do |device| return device if device.deviceInfo.label == deviceName @@ -84,7 +84,7 @@ module Vmpooler end def find_disk_controller(vm) - vsphere_connection_alive? @connection + ensure_connected @connection devices = find_disk_devices(vm) @@ -98,7 +98,7 @@ module Vmpooler end def find_disk_devices(vm) - vsphere_connection_alive? @connection + ensure_connected @connection devices = {} @@ -126,7 +126,7 @@ module Vmpooler end def find_disk_unit_number(vm, controller) - vsphere_connection_alive? @connection + ensure_connected @connection used_unit_numbers = [] available_unit_numbers = [] @@ -151,7 +151,7 @@ module Vmpooler end def find_folder(foldername) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter base = datacenter.vmFolder @@ -211,7 +211,7 @@ module Vmpooler (memory_usage.to_f / memory_size.to_f) * 100 end - def vsphere_connection_alive?(connection) + def ensure_connected(connection) begin connection.serviceInstance.CurrentTime rescue @@ -220,7 +220,7 @@ module Vmpooler end def find_least_used_host(cluster) - vsphere_connection_alive? @connection + ensure_connected @connection cluster_object = find_cluster(cluster) target_hosts = get_cluster_host_utilization(cluster_object) @@ -243,7 +243,7 @@ module Vmpooler end def find_least_used_compatible_host(vm) - vsphere_connection_alive? @connection + ensure_connected @connection source_host = vm.summary.runtime.host model = get_host_cpu_arch_version(source_host) @@ -257,7 +257,7 @@ module Vmpooler end def find_pool(poolname) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter base = datacenter.hostFolder @@ -286,13 +286,13 @@ module Vmpooler end def find_vm(vmname) - vsphere_connection_alive? @connection + ensure_connected @connection @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) end def find_vm_heavy(vmname) - vsphere_connection_alive? @connection + ensure_connected @connection vmname = vmname.is_a?(Array) ? vmname : [vmname] containerView = get_base_vm_container_from @connection @@ -342,7 +342,7 @@ module Vmpooler end def find_vmdks(vmname, datastore) - vsphere_connection_alive? @connection + ensure_connected @connection disks = [] @@ -361,7 +361,7 @@ module Vmpooler end def get_base_vm_container_from(connection) - vsphere_connection_alive? @connection + ensure_connected @connection viewManager = connection.serviceContent.viewManager viewManager.CreateContainerView( diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 4e54891..c83808c 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -225,6 +225,13 @@ # If set, prefixes all created VMs with this string. This should include # a separator. # (optional; default: '') +# +# - migration_limit +# When set to any value greater than 0 enable VM migration at checkout. +# When enabled this capability will evaluate a VM for migration when it is requested +# in an effort to maintain a more even distribution of load across compute resources. +# The migration_limit ensures that no more than n migrations will be evaluated at any one time +# and greatly reduces the possibilty of VMs ending up bunched together on a particular host. # Example: