From 087be6014dfb38741ae2130532306738f42ecba1 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 4 Feb 2014 15:05:24 -0800 Subject: [PATCH 1/2] Store 'clone' timestamps in Redis database Using redis keys to store clone timestamps will allow for a global TTL for VMs in the 'pending' pool; failed clones should be cleaned up and retried after a set period of time (eg. 15m). --- lib/vsphere_helper.rb | 56 ++++++++++++++++++++++++++++++++++++++++ vmware-host-pooler | 59 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/lib/vsphere_helper.rb b/lib/vsphere_helper.rb index 85d765b..afb52e3 100755 --- a/lib/vsphere_helper.rb +++ b/lib/vsphere_helper.rb @@ -100,6 +100,62 @@ class VsphereHelper + def find_vm_heavy vmname + begin + @connection.serviceInstance.CurrentTime + rescue + initialize() + end + + vmname = vmname.is_a?(Array) ? vmname : [ vmname ] + containerView = get_base_vm_container_from @connection + propertyCollector = @connection.propertyCollector + + objectSet = [{ + :obj => containerView, + :skip => true, + :selectSet => [ RbVmomi::VIM::TraversalSpec.new({ + :name => 'gettingTheVMs', + :path => 'view', + :skip => false, + :type => 'ContainerView' + }) ] + }] + + propSet = [{ + :pathSet => [ 'name' ], + :type => 'VirtualMachine' + }] + + results = propertyCollector.RetrievePropertiesEx({ + :specSet => [{ + :objectSet => objectSet, + :propSet => propSet + }], + :options => { :maxObjects => nil } + }) + + vms = {} + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + + while results.token do + results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token}) + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + end + + vms + end + + + def get_base_vm_container_from connection begin connection.serviceInstance.CurrentTime diff --git a/vmware-host-pooler b/vmware-host-pooler index 0b83114..70f9511 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -62,16 +62,17 @@ def check_pending_vm vm, pool, timeout $redis.smove('vmware_host_pool__pending__'+pool, 'vmware_host_pool__ready__'+pool, vm) $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") - else - if ( - (host.runtime) and - (host.runtime.bootTime) and - ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > timeout) - ) - $redis.smove('vmware_host_pool__pending__'+pool, 'vmware_host_pool__completed__'+pool, vm) + end + else + clone_stamp = $redis.hget('vmware_host_pool__vm__'+vm, 'clone') - $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") - end + if ( + (clone_stamp) and + (((Time.now - Time.parse(clone_stamp))/60) > timeout) + ) + $redis.smove('vmware_host_pool__pending__'+pool, 'vmware_host_pool__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") end end } @@ -158,6 +159,7 @@ def clone_vm template, pool, folder, datastore # Add VM to Redis inventory ('pending' pool) $redis.sadd('vmware_host_pool__pending__'+vm['template'], vm['hostname']) + $redis.hset('vmware_host_pool__vm__'+vm['hostname'], 'clone', Time.now) # Annotate with creation time, origin template, etc. configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( @@ -213,6 +215,7 @@ def destroy_vm vm, pool Thread.new { $redis.srem('vmware_host_pool__completed__'+pool, vm) $redis.hdel('vmware_host_pool__active__'+pool, vm) + $redis.del('vmware_host_pool__vm__'+vm) host = $vsphere[pool].find_vm(vm) @@ -234,6 +237,26 @@ def destroy_vm vm, pool $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") $graphite.log("vcloud.destroy.#{pool}", finish) if defined? $graphite + else + host = $vsphere[pool].find_vm_heavy(vm)[vm] + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState == 'poweredOn') + ) + $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") + host.PowerOffVM_Task.wait_for_completion + end + + host.Destroy_Task.wait_for_completion + finish = '%.2f' % (Time.now-start) + + $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") + + $graphite.log("vcloud.destroy.#{pool}", finish) if defined? $graphite + end end } end @@ -299,7 +322,7 @@ def check_pool pool # PENDING $redis.smembers('vmware_host_pool__pending__'+pool['name']).each do |vm| - pool['timeout'] ||= 10 + pool['timeout'] ||= 15 if (inventory[vm]) begin @@ -318,11 +341,27 @@ def check_pool pool $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") $redis.srem('vmware_host_pool__completed__'+pool['name'], vm) $redis.hdel('vmware_host_pool__active__'+pool['name'], vm) + $redis.del('vmware_host_pool__vm__'+vm) end else $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removing from 'completed' queue") $redis.srem('vmware_host_pool__completed__'+pool['name'], vm) $redis.hdel('vmware_host_pool__active__'+pool['name'], vm) + $redis.del('vmware_host_pool__vm__'+vm) + end + end + + # DISCOVERED + $redis.smembers('vmware_host_pool__discovered__'+pool['name']).each do |vm| + ['pending', 'ready', 'running', 'completed'].each do |queue| + if ($redis.sismember('vmware_host_pool__'+queue+'__'+pool['name'], vm)) + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removing from 'discovered'") + $redis.srem('vmware_host_pool__discovered__'+pool['name'], vm) + end + end + + if ($redis.sismember('vmware_host_pool__discovered__'+pool['name'], vm)) + $redis.smove('vmware_host_pool__discovered__'+pool['name'], 'vmware_host_pool__completed__'+pool['name'], vm) end end From b6e15080ded3c78338f371b016b8a82b1dde94cb Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 4 Feb 2014 15:13:38 -0800 Subject: [PATCH 2/2] Adding missing 'start' variable --- vmware-host-pooler | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vmware-host-pooler b/vmware-host-pooler index 70f9511..331a9ea 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -241,6 +241,8 @@ def destroy_vm vm, pool host = $vsphere[pool].find_vm_heavy(vm)[vm] if (host) + start = Time.now + if ( (host.runtime) and (host.runtime.powerState) and