diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 53895bb..7248be0 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -455,6 +455,55 @@ module Vmpooler end end + def find_vsphere_pool_vm(pool, vm) + $vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm] + end + + def migration_enabled?(migration_limit) + # Returns migration_limit setting when enabled + return false if migration_limit == 0 or not migration_limit + migration_limit if migration_limit >= 1 + end + + def migrate_vm(vm, pool) + Thread.new do + _migrate_vm(vm, pool) + end + end + + def _migrate_vm(vm, pool) + $redis.srem('vmpooler__migrating__' + pool, vm) + vm_object = find_vsphere_pool_vm(pool, vm) + parent_host = vm_object.summary.runtime.host + parent_host_name = parent_host.name + migration_limit = migration_enabled? $config[:config]['migration_limit'] + + if not migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + else + migration_count = $redis.smembers('vmpooler__migration').size + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") + else + $redis.sadd('vmpooler__migration', vm) + host = $vsphere[pool].find_least_used_compatible_host(vm_object) + if host == parent_host + $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") + else + start = Time.now + $vsphere[pool].migrate_vm_host(vm_object, host) + finish = '%.2f' % (Time.now - start) + $metrics.timing("migrate.#{vm['template']}", finish) + checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout'))) + $redis.hset('vmpooler__vm__' + vm, 'migration_time', finish) + $redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration) + $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds") + end + $redis.srem('vmpooler__migration', vm) + end + end + end + def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") @@ -480,7 +529,8 @@ module Vmpooler (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__pending__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__completed__' + pool['name'], vm['name'])) && - (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) + (! $redis.sismember('vmpooler__discovered__' + pool['name'], vm['name'])) && + (! $redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])) $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name']) @@ -555,6 +605,17 @@ module Vmpooler end end + # MIGRATIONS + $redis.smembers('vmpooler__migrating__' + pool['name']).each do |vm| + if inventory[vm] + begin + migrate_vm(vm, pool['name']) + rescue => err + $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") + end + end + end + # REPOPULATE ready = $redis.scard('vmpooler__ready__' + pool['name']) total = $redis.scard('vmpooler__pending__' + pool['name']) + ready diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index 6b6f57d..6e65e30 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -17,7 +17,7 @@ module Vmpooler end def add_disk(vm, size, datastore) - vsphere_connection_alive? @connection + ensure_connected @connection return false unless size.to_i > 0 @@ -67,14 +67,14 @@ module Vmpooler end def find_datastore(datastorename) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter datacenter.find_datastore(datastorename) end def find_device(vm, deviceName) - vsphere_connection_alive? @connection + ensure_connected @connection vm.config.hardware.device.each do |device| return device if device.deviceInfo.label == deviceName @@ -84,7 +84,7 @@ module Vmpooler end def find_disk_controller(vm) - vsphere_connection_alive? @connection + ensure_connected @connection devices = find_disk_devices(vm) @@ -98,7 +98,7 @@ module Vmpooler end def find_disk_devices(vm) - vsphere_connection_alive? @connection + ensure_connected @connection devices = {} @@ -126,7 +126,7 @@ module Vmpooler end def find_disk_unit_number(vm, controller) - vsphere_connection_alive? @connection + ensure_connected @connection used_unit_numbers = [] available_unit_numbers = [] @@ -151,7 +151,7 @@ module Vmpooler end def find_folder(foldername) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter base = datacenter.vmFolder @@ -211,7 +211,7 @@ module Vmpooler (memory_usage.to_f / memory_size.to_f) * 100 end - def vsphere_connection_alive?(connection) + def ensure_connected(connection) begin connection.serviceInstance.CurrentTime rescue @@ -220,7 +220,7 @@ module Vmpooler end def find_least_used_host(cluster) - vsphere_connection_alive? @connection + ensure_connected @connection cluster_object = find_cluster(cluster) target_hosts = get_cluster_host_utilization(cluster_object) @@ -243,7 +243,7 @@ module Vmpooler end def find_least_used_compatible_host(vm) - vsphere_connection_alive? @connection + ensure_connected @connection source_host = vm.summary.runtime.host model = get_host_cpu_arch_version(source_host) @@ -257,7 +257,7 @@ module Vmpooler end def find_pool(poolname) - vsphere_connection_alive? @connection + ensure_connected @connection datacenter = @connection.serviceInstance.find_datacenter base = datacenter.hostFolder @@ -286,13 +286,13 @@ module Vmpooler end def find_vm(vmname) - vsphere_connection_alive? @connection + ensure_connected @connection @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) end def find_vm_heavy(vmname) - vsphere_connection_alive? @connection + ensure_connected @connection vmname = vmname.is_a?(Array) ? vmname : [vmname] containerView = get_base_vm_container_from @connection @@ -342,7 +342,7 @@ module Vmpooler end def find_vmdks(vmname, datastore) - vsphere_connection_alive? @connection + ensure_connected @connection disks = [] @@ -361,7 +361,7 @@ module Vmpooler end def get_base_vm_container_from(connection) - vsphere_connection_alive? @connection + ensure_connected @connection viewManager = connection.serviceContent.viewManager viewManager.CreateContainerView( diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 4e54891..c83808c 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -225,6 +225,13 @@ # If set, prefixes all created VMs with this string. This should include # a separator. # (optional; default: '') +# +# - migration_limit +# When set to any value greater than 0 enable VM migration at checkout. +# When enabled this capability will evaluate a VM for migration when it is requested +# in an effort to maintain a more even distribution of load across compute resources. +# The migration_limit ensures that no more than n migrations will be evaluated at any one time +# and greatly reduces the possibilty of VMs ending up bunched together on a particular host. # Example: