diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 968b6bf..dd17385 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -20,9 +20,9 @@ module Vmpooler end # Check the state of a VM - def check_pending_vm(vm, pool, timeout) + def check_pending_vm(vm, pool, timeout, vsphere) Thread.new do - _check_pending_vm(vm, pool, timeout) + _check_pending_vm(vm, pool, timeout, vsphere) end end @@ -34,12 +34,14 @@ module Vmpooler end end - def _check_pending_vm(vm, pool, timeout) - host = $vsphere[pool].find_vm(vm) + def _check_pending_vm(vm, pool, timeout, vsphere) + host = vsphere.find_vm(vm) if host begin - open_socket vm, $config[:config]['domain'], timeout + Timeout.timeout(5) do + TCPSocket.new vm, 22 + end move_pending_vm_to_ready(vm, pool, host) rescue fail_pending_vm(vm, pool, timeout) @@ -87,7 +89,7 @@ module Vmpooler end end - def check_ready_vm(vm, pool, ttl) + def check_ready_vm(vm, pool, ttl, vsphere) Thread.new do if ttl > 0 if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl @@ -105,8 +107,8 @@ module Vmpooler $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) - host = $vsphere[pool].find_vm(vm) || - $vsphere[pool].find_vm_heavy(vm)[vm] + host = vsphere.find_vm(vm) || + vsphere.find_vm_heavy(vm)[vm] if host if @@ -147,14 +149,14 @@ module Vmpooler end end - def check_running_vm(vm, pool, ttl) + def check_running_vm(vm, pool, ttl, vsphere) Thread.new do - _check_running_vm(vm, pool, ttl) + _check_running_vm(vm, pool, ttl, vsphere) end end - def _check_running_vm(vm, pool, ttl) - host = $vsphere[pool].find_vm(vm) + def _check_running_vm(vm, pool, ttl, vsphere) + host = vsphere.find_vm(vm) if host queue_from, queue_to = 'running', 'completed' @@ -178,101 +180,105 @@ module Vmpooler end # Clone a VM - def clone_vm(template, folder, datastore, target) + def clone_vm(template, folder, datastore, target, vsphere) Thread.new do - vm = {} - - if template =~ /\// - templatefolders = template.split('/') - vm['template'] = templatefolders.pop - end - - if templatefolders - vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template']) - else - fail 'Please provide a full path to the template' - end - - if vm['template'].length == 0 - fail "Unable to find template '#{vm['template']}'!" - end - - # Generate a randomized hostname - o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join - - # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - - # Annotate with creation time, origin template, etc. - # Add extraconfig options that can be queried by vmtools - configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - annotation: JSON.pretty_generate( - name: vm['hostname'], - created_by: $config[:vsphere]['username'], - base_template: vm['template'], - creation_timestamp: Time.now.utc - ), - extraConfig: [ - { key: 'guestinfo.hostname', - value: vm['hostname'] - } - ] - ) - - # Choose a clone target - if target - $clone_target = $vsphere[vm['template']].find_least_used_host(target) - elsif $config[:config]['clone_target'] - $clone_target = $vsphere[vm['template']].find_least_used_host($config[:config]['clone_target']) - end - - # Put the VM in the specified folder and resource pool - relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - datastore: $vsphere[vm['template']].find_datastore(datastore), - host: $clone_target, - diskMoveType: :moveChildMostDiskBacking - ) - - # Create a clone spec - spec = RbVmomi::VIM.VirtualMachineCloneSpec( - location: relocateSpec, - config: configSpec, - powerOn: true, - template: false - ) - - # Clone the VM - $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") - begin - start = Time.now - vm[vm['template']].CloneVM_Task( - folder: $vsphere[vm['template']].find_folder(folder), - name: vm['hostname'], - spec: spec - ).wait_for_completion - finish = '%.2f' % (Time.now - start) + vm = {} - $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + if template =~ /\// + templatefolders = template.split('/') + vm['template'] = templatefolders.pop + end - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") - rescue - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed") - $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + if templatefolders + vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) + else + fail 'Please provide a full path to the template' + end + + if vm['template'].length == 0 + fail "Unable to find template '#{vm['template']}'!" + end + + # Generate a randomized hostname + o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + + # Add VM to Redis inventory ('pending' pool) + $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) + + # Annotate with creation time, origin template, etc. + # Add extraconfig options that can be queried by vmtools + configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( + annotation: JSON.pretty_generate( + name: vm['hostname'], + created_by: $config[:vsphere]['username'], + base_template: vm['template'], + creation_timestamp: Time.now.utc + ), + extraConfig: [ + { key: 'guestinfo.hostname', + value: vm['hostname'] + } + ] + ) + + # Choose a clone target + if target + $clone_target = vsphere.find_least_used_host(target) + elsif $config[:config]['clone_target'] + $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) + end + + # Put the VM in the specified folder and resource pool + relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( + datastore: vsphere.find_datastore(datastore), + host: $clone_target, + diskMoveType: :moveChildMostDiskBacking + ) + + # Create a clone spec + spec = RbVmomi::VIM.VirtualMachineCloneSpec( + location: relocateSpec, + config: configSpec, + powerOn: true, + template: false + ) + + # Clone the VM + $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + + begin + start = Time.now + vm[vm['template']].CloneVM_Task( + folder: vsphere.find_folder(folder), + name: vm['hostname'], + spec: spec + ).wait_for_completion + finish = '%.2f' % (Time.now - start) + + $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + + $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + rescue => err + $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") + $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + end + + $redis.decr('vmpooler__tasks__clone') + + $metrics.timing("clone.#{vm['template']}", finish) + rescue => err + $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") end - - $redis.decr('vmpooler__tasks__clone') - - $metrics.timing("clone.#{vm['template']}", finish) end end # Destroy a VM - def destroy_vm(vm, pool) + def destroy_vm(vm, pool, vsphere) Thread.new do $redis.srem('vmpooler__completed__' + pool, vm) $redis.hdel('vmpooler__active__' + pool, vm) @@ -281,8 +287,8 @@ module Vmpooler # Auto-expire metadata key $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) - host = $vsphere[pool].find_vm(vm) || - $vsphere[pool].find_vm_heavy(vm)[vm] + host = vsphere.find_vm(vm) || + vsphere.find_vm_heavy(vm)[vm] if host start = Time.now @@ -305,15 +311,15 @@ module Vmpooler end end - def create_vm_disk(vm, disk_size) + def create_vm_disk(vm, disk_size, vsphere) Thread.new do - _create_vm_disk(vm, disk_size) + _create_vm_disk(vm, disk_size, vsphere) end end - def _create_vm_disk(vm, disk_size) - host = $vsphere['disk_manager'].find_vm(vm) || - $vsphere['disk_manager'].find_vm_heavy(vm)[vm] + def _create_vm_disk(vm, disk_size, vsphere) + host = vsphere.find_vm(vm) || + vsphere.find_vm_heavy(vm)[vm] if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0)) $logger.log('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") @@ -330,7 +336,7 @@ module Vmpooler end if ((! datastore.nil?) && (! datastore.empty?)) - $vsphere['disk_manager'].add_disk(host, disk_size, datastore) + vsphere.add_disk(host, disk_size, datastore) rdisks = $redis.hget('vmpooler__vm__' + vm, 'disk') disks = rdisks ? rdisks.split(':') : [] @@ -346,15 +352,15 @@ module Vmpooler end end - def create_vm_snapshot(vm, snapshot_name) + def create_vm_snapshot(vm, snapshot_name, vsphere) Thread.new do - _create_vm_snapshot(vm, snapshot_name) + _create_vm_snapshot(vm, snapshot_name, vsphere) end end - def _create_vm_snapshot(vm, snapshot_name) - host = $vsphere['snapshot_manager'].find_vm(vm) || - $vsphere['snapshot_manager'].find_vm_heavy(vm)[vm] + def _create_vm_snapshot(vm, snapshot_name, vsphere) + host = vsphere.find_vm(vm) || + vsphere.find_vm_heavy(vm)[vm] if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?)) $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being snapshotted") @@ -376,18 +382,18 @@ module Vmpooler end end - def revert_vm_snapshot(vm, snapshot_name) + def revert_vm_snapshot(vm, snapshot_name, vsphere) Thread.new do - _revert_vm_snapshot(vm, snapshot_name) + _revert_vm_snapshot(vm, snapshot_name, vsphere) end end - def _revert_vm_snapshot(vm, snapshot_name) - host = $vsphere['snapshot_manager'].find_vm(vm) || - $vsphere['snapshot_manager'].find_vm_heavy(vm)[vm] + def _revert_vm_snapshot(vm, snapshot_name, vsphere) + host = vsphere.find_vm(vm) || + vsphere.find_vm_heavy(vm)[vm] if host - snapshot = $vsphere['snapshot_manager'].find_snapshot(host, snapshot_name) + snapshot = vsphere.find_snapshot(host, snapshot_name) if snapshot $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being reverted to snapshot '#{snapshot_name}'") @@ -410,19 +416,19 @@ module Vmpooler $threads['disk_manager'] = Thread.new do loop do - _check_disk_queue + _check_disk_queue $vsphere['disk_manager'] sleep(5) end end end - def _check_disk_queue + def _check_disk_queue(vsphere) vm = $redis.spop('vmpooler__tasks__disk') unless vm.nil? begin vm_name, disk_size = vm.split(':') - create_vm_disk(vm_name, disk_size) + create_vm_disk(vm_name, disk_size, vsphere) rescue $logger.log('s', "[!] [disk_manager] disk creation appears to have failed") end @@ -436,19 +442,19 @@ module Vmpooler $threads['snapshot_manager'] = Thread.new do loop do - _check_snapshot_queue + _check_snapshot_queue $vsphere['snapshot_manager'] sleep(5) end end end - def _check_snapshot_queue + def _check_snapshot_queue(vsphere) vm = $redis.spop('vmpooler__tasks__snapshot') unless vm.nil? begin vm_name, snapshot_name = vm.split(':') - create_vm_snapshot(vm_name, snapshot_name) + create_vm_snapshot(vm_name, snapshot_name, vsphere) rescue $logger.log('s', "[!] [snapshot_manager] snapshot appears to have failed") end @@ -459,15 +465,15 @@ module Vmpooler unless vm.nil? begin vm_name, snapshot_name = vm.split(':') - revert_vm_snapshot(vm_name, snapshot_name) + revert_vm_snapshot(vm_name, snapshot_name, vsphere) rescue $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed") end end end - def find_vsphere_pool_vm(pool, vm) - $vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm] + def find_vsphere_pool_vm(pool, vm, vsphere) + vsphere.find_vm(vm) || vsphere.find_vm_heavy(vm)[vm] end def migration_limit(migration_limit) @@ -476,45 +482,67 @@ module Vmpooler migration_limit if migration_limit >= 1 end - def migrate_vm(vm, pool) + def migrate_vm(vm, pool, vsphere) Thread.new do - _migrate_vm(vm, pool) + _migrate_vm(vm, pool, vsphere) end end - def _migrate_vm(vm, pool) - $redis.srem('vmpooler__migrating__' + pool, vm) - vm_object = find_vsphere_pool_vm(pool, vm) - parent_host = vm_object.summary.runtime.host - parent_host_name = parent_host.name - migration_limit = migration_limit $config[:config]['migration_limit'] + def _migrate_vm(vm, pool, vsphere) + begin + $redis.srem('vmpooler__migrating__' + pool, vm) + vm_object = find_vsphere_pool_vm(pool, vm, vsphere) + parent_host, parent_host_name = get_vm_host_info(vm_object) + migration_limit = migration_limit $config[:config]['migration_limit'] - if not migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") - else - migration_count = $redis.smembers('vmpooler__migration').size - if migration_count >= migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") + if not migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") else - $redis.sadd('vmpooler__migration', vm) - host = $vsphere[pool].find_least_used_compatible_host(vm_object) - if host == parent_host - $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") + migration_count = $redis.smembers('vmpooler__migration').size + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") else - start = Time.now - $vsphere[pool].migrate_vm_host(vm_object, host) - finish = '%.2f' % (Time.now - start) - $metrics.timing("migrate.#{vm['template']}", finish) - checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout'))) - $redis.hset('vmpooler__vm__' + vm, 'migration_time', finish) - $redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration) - $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds") + $redis.sadd('vmpooler__migration', vm) + host, host_name = vsphere.find_least_used_compatible_host(vm_object) + if host == parent_host + $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") + else + finish = migrate_vm_and_record_timing(vm_object, vm, host, vsphere) + $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") + end + remove_vmpooler_migration_vm(pool, vm) end - $redis.srem('vmpooler__migration', vm) end + rescue => err + $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") + remove_vmpooler_migration_vm(pool, vm) end end + def get_vm_host_info(vm_object) + parent_host = vm_object.summary.runtime.host + [parent_host, parent_host.name] + end + + def remove_vmpooler_migration_vm(pool, vm) + begin + $redis.srem('vmpooler__migration', vm) + rescue => err + $logger.log('s', "[x] [#{pool}] '#{vm}' removal from vmpooler__migration failed with an error: #{err}") + end + end + + def migrate_vm_and_record_timing(vm_object, vm_name, host, vsphere) + start = Time.now + vsphere.migrate_vm_host(vm_object, host) + finish = '%.2f' % (Time.now - start) + $metrics.timing("migrate.#{vm_name}", finish) + checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget("vmpooler__vm__#{vm_name}", 'checkout'))) + $redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish) + $redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration) + finish + end + def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") @@ -522,17 +550,17 @@ module Vmpooler $threads[pool['name']] = Thread.new do loop do - _check_pool(pool) + _check_pool(pool, $vsphere[pool['name']]) sleep(5) end end end - def _check_pool(pool) + def _check_pool(pool, vsphere) # INVENTORY inventory = {} begin - base = $vsphere[pool['name']].find_folder(pool['folder']) + base = vsphere.find_folder(pool['folder']) base.childEntity.each do |vm| if @@ -554,111 +582,93 @@ module Vmpooler end # RUNNING - running = $redis.smembers("vmpooler__running__#{pool['name']}") - if running - running.each do |vm| - if inventory[vm] - begin - check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12) - rescue - end + $redis.smembers("vmpooler__running__#{pool['name']}").each do |vm| + if inventory[vm] + begin + check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12, vsphere) + rescue end end end # READY - ready = $redis.smembers("vmpooler__ready__#{pool['name']}") - if ready - ready.each do |vm| if ready - if inventory[vm] - begin - check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) - rescue - end + $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm| + if inventory[vm] + begin + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere) + rescue end end end # PENDING - pending = $redis.smembers('vmpooler__pending__' + pool['name']) - if pending - pending.each do |vm| - if inventory[vm] - begin - check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15) - rescue - end + $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm| + if inventory[vm] + begin + check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15, vsphere) + rescue end end end # COMPLETED - completed = $redis.smembers('vmpooler__completed__' + pool['name']) - if completed - completed.each do |vm| - if inventory[vm] - begin - destroy_vm(vm, pool['name']) - rescue - $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") - $redis.srem('vmpooler__completed__' + pool['name'], vm) - $redis.hdel('vmpooler__active__' + pool['name'], vm) - $redis.del('vmpooler__vm__' + vm) - end - else - $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue") - $redis.srem('vmpooler__completed__' + pool['name'], vm) - $redis.hdel('vmpooler__active__' + pool['name'], vm) - $redis.del('vmpooler__vm__' + vm) + $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm| + if inventory[vm] + begin + destroy_vm(vm, pool['name'], vsphere) + rescue + $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") + $redis.srem("vmpooler__completed__#{pool['name']}", vm) + $redis.hdel("vmpooler__active__#{pool['name']}", vm) + $redis.del("vmpooler__vm__#{vm}") end + else + $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue") + $redis.srem("vmpooler__completed__#{pool['name']}", vm) + $redis.hdel("vmpooler__active__#{pool['name']}", vm) + $redis.del("vmpooler__vm__#{vm}") end end # DISCOVERED - discovered = $redis.smembers("vmpooler__discovered__#{pool['name']}") - if discovered - discovered.each do |vm| - %w(pending ready running completed).each do |queue| - if $redis.sismember('vmpooler__' + queue + '__' + pool['name'], vm) - $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue") - $redis.srem('vmpooler__discovered__' + pool['name'], vm) - end + $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm| + %w(pending ready running completed).each do |queue| + if $redis.sismember("vmpooler__#{queue}__#{pool['name']}", vm) + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue") + $redis.srem("vmpooler__discovered__#{pool['name']}", vm) end + end - if $redis.sismember('vmpooler__discovered__' + pool['name'], vm) - $redis.smove('vmpooler__discovered__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) - end + if $redis.sismember("vmpooler__discovered__#{pool['name']}", vm) + $redis.smove("vmpooler__discovered__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm) end end # MIGRATIONS - migrations = $redis.smembers('vmpooler__migrating__' + pool['name']) - if migrations - migrations.each do |vm| - if inventory[vm] - begin - migrate_vm(vm, pool['name']) - rescue => err - $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") - end + $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm| + if inventory[vm] + begin + migrate_vm(vm, pool['name'], vsphere) + rescue => err + $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") end end end # REPOPULATE - ready = $redis.scard('vmpooler__ready__' + pool['name']) - total = $redis.scard('vmpooler__pending__' + pool['name']) + ready + ready = $redis.scard("vmpooler__ready__#{pool['name']}") + total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready - $metrics.gauge('ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name'])) - $metrics.gauge('running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name'])) + $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}")) + $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}")) - if $redis.get('vmpooler__empty__' + pool['name']) + if $redis.get("vmpooler__empty__#{pool['name']}") unless ready == 0 - $redis.del('vmpooler__empty__' + pool['name']) + $redis.del("vmpooler__empty__#{pool['name']}") end else if ready == 0 - $redis.set('vmpooler__empty__' + pool['name'], 'true') + $redis.set("vmpooler__empty__#{pool['name']}", 'true') $logger.log('s', "[!] [#{pool['name']}] is empty") end end @@ -673,10 +683,11 @@ module Vmpooler pool['template'], pool['folder'], pool['datastore'], - pool['clone_target'] + pool['clone_target'], + vsphere ) - rescue - $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed") + rescue => err + $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}") $redis.decr('vmpooler__tasks__clone') end end diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index 66b387b..c49bce4 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -26,7 +26,7 @@ module Vmpooler end def add_disk(vm, size, datastore) - ensure_connected @connection + ensure_connected @connection, $credentials return false unless size.to_i > 0 @@ -76,14 +76,14 @@ module Vmpooler end def find_datastore(datastorename) - ensure_connected @connection + ensure_connected @connection, $credentials datacenter = @connection.serviceInstance.find_datacenter datacenter.find_datastore(datastorename) end def find_device(vm, deviceName) - ensure_connected @connection + ensure_connected @connection, $credentials vm.config.hardware.device.each do |device| return device if device.deviceInfo.label == deviceName @@ -93,7 +93,7 @@ module Vmpooler end def find_disk_controller(vm) - ensure_connected @connection + ensure_connected @connection, $credentials devices = find_disk_devices(vm) @@ -107,7 +107,7 @@ module Vmpooler end def find_disk_devices(vm) - ensure_connected @connection + ensure_connected @connection, $credentials devices = {} @@ -135,7 +135,7 @@ module Vmpooler end def find_disk_unit_number(vm, controller) - ensure_connected @connection + ensure_connected @connection, $credentials used_unit_numbers = [] available_unit_numbers = [] @@ -160,7 +160,7 @@ module Vmpooler end def find_folder(foldername) - ensure_connected @connection + ensure_connected @connection, $credentials datacenter = @connection.serviceInstance.find_datacenter base = datacenter.vmFolder @@ -221,7 +221,7 @@ module Vmpooler end def find_least_used_host(cluster) - ensure_connected @connection + ensure_connected @connection, $credentials cluster_object = find_cluster(cluster) target_hosts = get_cluster_host_utilization(cluster_object) @@ -244,7 +244,7 @@ module Vmpooler end def find_least_used_compatible_host(vm) - ensure_connected @connection + ensure_connected @connection, $credentials source_host = vm.summary.runtime.host model = get_host_cpu_arch_version(source_host) @@ -259,7 +259,7 @@ module Vmpooler end def find_pool(poolname) - ensure_connected @connection + ensure_connected @connection, $credentials datacenter = @connection.serviceInstance.find_datacenter base = datacenter.hostFolder @@ -288,13 +288,13 @@ module Vmpooler end def find_vm(vmname) - ensure_connected @connection + ensure_connected @connection, $credentials @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) end def find_vm_heavy(vmname) - ensure_connected @connection + ensure_connected @connection, $credentials vmname = vmname.is_a?(Array) ? vmname : [vmname] containerView = get_base_vm_container_from @connection @@ -344,7 +344,7 @@ module Vmpooler end def find_vmdks(vmname, datastore) - ensure_connected @connection + ensure_connected @connection, $credentials disks = [] @@ -363,7 +363,7 @@ module Vmpooler end def get_base_vm_container_from(connection) - ensure_connected @connection + ensure_connected @connection, $credentials viewManager = connection.serviceContent.viewManager viewManager.CreateContainerView(