From 1e6ca05d147fe5fd152b4ed94066dec88cc04888 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 18 Feb 2014 14:05:29 -0800 Subject: [PATCH 1/3] Add additional checks to VMs in 'ready' queues --- vmware-host-pooler | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 234833f..70c25a3 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -4,6 +4,7 @@ require 'json' require 'rbvmomi' require 'redis' require 'time' +require 'timeout' require 'yaml' $:.unshift(File.dirname(__FILE__)) @@ -87,6 +88,14 @@ def check_ready_vm vm, pool, ttl Thread.new { host = $vsphere[pool].find_vm(vm) + if (ttl > 0) + if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) + $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + end + end + if (host) if ( (host.runtime) and @@ -95,18 +104,22 @@ def check_ready_vm vm, pool, ttl ) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - else - if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) - $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes") - end + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") end else $redis.srem('vmware_host_pool__ready__'+pool, vm) - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory") + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") + end + + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + end end } end @@ -300,11 +313,9 @@ def check_pool pool # READY $redis.smembers('vmware_host_pool__ready__'+pool['name']).each do |vm| if (inventory[vm]) - if (pool['ready_ttl']) - begin - check_ready_vm(vm, pool['name'], pool['ready_ttl']) - rescue - end + begin + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) + rescue end end end From 62cbc5a6ff737d1bb428b62544ec3b4b0765ca1e Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 18 Feb 2014 22:48:22 -0800 Subject: [PATCH 2/3] 'vm_checktime' sets interval for check_ready_vm --- vmware-host-pooler | 54 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 70c25a3..0f17fcb 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -32,8 +32,9 @@ if (defined? $config[:config]['graphite']) end # Set some defaults -$config[:config]['task_limit'] ||= 10 -$config[:config]['vm_lifetime'] ||= 24 +$config[:config]['task_limit'] ||= 10 +$config[:config]['vm_checktime'] ||= 15 +$config[:config]['vm_lifetime'] ||= 24 $config[:redis] ||= Hash.new $config[:redis]['server'] ||= 'localhost' @@ -96,29 +97,38 @@ def check_ready_vm vm, pool, ttl end end - if (host) - if ( - (host.runtime) and - (host.runtime.powerState) and - (host.runtime.powerState != 'poweredOn') - ) - $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + check_stamp = $redis.hget('vmware_host_pool__vm__'+vm, 'check') - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + if ( + (! check_stamp) or + ((Time.now - Time.parse(clone_stamp)) > $config[:config]['vm_checktime']) + ) + $redis.hset('vmware_host_pool__vm__'+vm, 'check', Time.now) + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState != 'poweredOn') + ) + $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end + else + $redis.srem('vmware_host_pool__ready__'+pool, vm) + + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") end - else - $redis.srem('vmware_host_pool__ready__'+pool, vm) - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - Timeout::timeout(5) { - TCPSocket.new vm, 22 - } - rescue - if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + end end end } From e7559c515ef22f40b3915ca1001cc7e08d3ad21c Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 19 Feb 2014 09:38:05 -0800 Subject: [PATCH 3/3] check_ready_vm fixups - Correct key name ('check', not 'clone') - Correct timestamp math (minutes, not seconds) - Only search for the host in vSphere if we're actually doing a check --- vmware-host-pooler | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 0f17fcb..35da677 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -87,8 +87,6 @@ end def check_ready_vm vm, pool, ttl Thread.new { - host = $vsphere[pool].find_vm(vm) - if (ttl > 0) if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) @@ -101,10 +99,12 @@ def check_ready_vm vm, pool, ttl if ( (! check_stamp) or - ((Time.now - Time.parse(clone_stamp)) > $config[:config]['vm_checktime']) + (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime']) ) $redis.hset('vmware_host_pool__vm__'+vm, 'check', Time.now) + host = $vsphere[pool].find_vm(vm) + if (host) if ( (host.runtime) and