From fba2c17446a5a23c573091ba02f63ce315be3096 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 18 Feb 2014 14:05:29 -0800 Subject: [PATCH 1/4] Add additional checks to VMs in 'ready' queues --- vmware-host-pooler | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 234833f..70c25a3 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -4,6 +4,7 @@ require 'json' require 'rbvmomi' require 'redis' require 'time' +require 'timeout' require 'yaml' $:.unshift(File.dirname(__FILE__)) @@ -87,6 +88,14 @@ def check_ready_vm vm, pool, ttl Thread.new { host = $vsphere[pool].find_vm(vm) + if (ttl > 0) + if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) + $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + end + end + if (host) if ( (host.runtime) and @@ -95,18 +104,22 @@ def check_ready_vm vm, pool, ttl ) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - else - if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) - $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes") - end + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") end else $redis.srem('vmware_host_pool__ready__'+pool, vm) - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory") + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") + end + + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + end end } end @@ -300,11 +313,9 @@ def check_pool pool # READY $redis.smembers('vmware_host_pool__ready__'+pool['name']).each do |vm| if (inventory[vm]) - if (pool['ready_ttl']) - begin - check_ready_vm(vm, pool['name'], pool['ready_ttl']) - rescue - end + begin + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0) + rescue end end end From bf2658cd7d194fdf9f64874524bee85e0131b488 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Tue, 18 Feb 2014 22:48:22 -0800 Subject: [PATCH 2/4] 'vm_checktime' sets interval for check_ready_vm --- vmware-host-pooler | 54 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 70c25a3..0f17fcb 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -32,8 +32,9 @@ if (defined? $config[:config]['graphite']) end # Set some defaults -$config[:config]['task_limit'] ||= 10 -$config[:config]['vm_lifetime'] ||= 24 +$config[:config]['task_limit'] ||= 10 +$config[:config]['vm_checktime'] ||= 15 +$config[:config]['vm_lifetime'] ||= 24 $config[:redis] ||= Hash.new $config[:redis]['server'] ||= 'localhost' @@ -96,29 +97,38 @@ def check_ready_vm vm, pool, ttl end end - if (host) - if ( - (host.runtime) and - (host.runtime.powerState) and - (host.runtime.powerState != 'poweredOn') - ) - $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + check_stamp = $redis.hget('vmware_host_pool__vm__'+vm, 'check') - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + if ( + (! check_stamp) or + ((Time.now - Time.parse(clone_stamp)) > $config[:config]['vm_checktime']) + ) + $redis.hset('vmware_host_pool__vm__'+vm, 'check', Time.now) + + if (host) + if ( + (host.runtime) and + (host.runtime.powerState) and + (host.runtime.powerState != 'poweredOn') + ) + $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end + else + $redis.srem('vmware_host_pool__ready__'+pool, vm) + + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") end - else - $redis.srem('vmware_host_pool__ready__'+pool, vm) - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - Timeout::timeout(5) { - TCPSocket.new vm, 22 - } - rescue - if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + begin + Timeout::timeout(5) { + TCPSocket.new vm, 22 + } + rescue + if ($redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm)) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + end end end } From bb805d71f5c76005597c6fe58206daa1b3d12766 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 19 Feb 2014 09:38:05 -0800 Subject: [PATCH 3/4] check_ready_vm fixups - Correct key name ('check', not 'clone') - Correct timestamp math (minutes, not seconds) - Only search for the host in vSphere if we're actually doing a check --- vmware-host-pooler | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vmware-host-pooler b/vmware-host-pooler index 0f17fcb..35da677 100755 --- a/vmware-host-pooler +++ b/vmware-host-pooler @@ -87,8 +87,6 @@ end def check_ready_vm vm, pool, ttl Thread.new { - host = $vsphere[pool].find_vm(vm) - if (ttl > 0) if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl) $redis.smove('vmware_host_pool__ready__'+pool, 'vmware_host_pool__completed__'+pool, vm) @@ -101,10 +99,12 @@ def check_ready_vm vm, pool, ttl if ( (! check_stamp) or - ((Time.now - Time.parse(clone_stamp)) > $config[:config]['vm_checktime']) + (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime']) ) $redis.hset('vmware_host_pool__vm__'+vm, 'check', Time.now) + host = $vsphere[pool].find_vm(vm) + if (host) if ( (host.runtime) and From 26fbb2d85b174b61fa2bcc920854ed6d9e2b7c7d Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 19 Feb 2014 11:53:19 -0800 Subject: [PATCH 4/4] Update TODO --- TODO.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/TODO.md b/TODO.md index db103b4..42aa592 100644 --- a/TODO.md +++ b/TODO.md @@ -16,8 +16,3 @@ Enhancements * Namespace the whole app * Require 'authorization key' (returned in POST JSON) to DELETE via the API - -Fixes ------ - -* VMs in 'ready' pool should be checked even if no ready_ttl is set