diff --git a/vmware-host-pooler b/vmware-host-pooler index 234833f..35da677 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -4,6 +4,7 @@ require 'json' require 'rbvmomi' require 'redis' require 'time' +require 'timeout' require 'yaml' $:.unshift(File.dirname(__FILE__)) @@ -31,8 +32,9 @@ if (defined? $config[:config]['graphite']) end # Set some defaults -$config[:config]['task_limit'] ||= 10 -$config[:config]['vm_lifetime'] ||= 24 +$config[:config]['task_limit'] ||= 10 +$config[:config]['vm_checktime'] ||= 15 +$config[:config]['vm_lifetime'] ||= 24 $config[:redis] ||= Hash.new $config[:redis]['server'] ||= 'localhost' @@ -85,28 +87,49 @@ end def check_ready_vm vm, pool, ttl Thread.new { - host = $vsphere[pool].find_vm(vm) - - if (host) - if ( - (host.runtime) and - (host.runtime.powerState) and - (host.runtime.powerState != 'poweredOn') - ) + if (ttl > 0) + if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - else - if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + end + end + + check_stamp = $redis.hget('vmware_host_pool__vm__'+vm, 'check') + + if ( + (! check_stamp) or + (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime']) + ) + $redis.hset('vmware_host_pool__vm__'+vm, 'check', Time.now) + + host = $vsphere[pool].find_vm(vm) + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState != 'poweredOn') + ) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes") + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end + else + $redis.srem('vmware_host_pool__ready__'+pool, vm) + + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") + end + + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") end end - else - $redis.srem('vmware_host_pool__ready__'+pool, vm) - - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory") end } end @@ -300,11 +323,9 @@ def check_pool pool # READY $redis.smembers('vmware_host_pool__ready__'+pool['name']).each do |vm| if (inventory[vm]) - if (pool['ready_ttl']) - begin - check_ready_vm(vm, pool['name'], pool['ready_ttl']) - rescue - end + begin + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) + rescue end end end