Merge branch 'master' into host_snapshots

lib/vmpooler/pool_manager.rb

@@ -142,14 +142,20 @@ module Vmpooler
       if host
         queue_from, queue_to = 'running', 'completed'
 
+        # Check that VM is powered on
         if (host.runtime) &&
             (host.runtime.powerState != 'poweredOn')
           move_vm_queue(pool, vm, queue_from, queue_to, 'appears to be powered off or dead')
-        else
+        end
-          if (host.runtime) &&
+
-              (host.runtime.bootTime)
+        # Check that VM is within defined lifetime
-            ((((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
+        checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
-            move_vm_queue(pool, vm, queue_from, queue_to, "reached end of TTL after #{ttl} minutes")
+        if checkouttime
+          running = (Time.now - Time.parse(checkouttime)) / 60 / 60
+
+          if (ttl.to_i > 0) &&
+              (running.to_i >= ttl.to_i)
+            move_vm_queue(pool, vm, queue_from, queue_to, "reached end of TTL after #{ttl} hours")
           end
         end
       end
@@ -383,10 +389,17 @@ module Vmpooler
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $threads[pool['name']] = Thread.new do
       $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
 
+      $threads[pool['name']] = Thread.new do
         loop do
+          _check_pool(pool)
+          sleep(5)
+        end
+      end
+    end
+
+    def _check_pool(pool)
       # INVENTORY
       inventory = {}
       begin
@@ -413,17 +426,10 @@ module Vmpooler
       # RUNNING
       $redis.smembers('vmpooler__running__' + pool['name']).each do |vm|
         if inventory[vm]
-              if pool['running_ttl']
           begin
-                  check_running_vm(vm, pool['name'], pool['running_ttl'])
+            check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12)
           rescue
           end
-              else
-                begin
-                  check_running_vm(vm, pool['name'], '720')
-                rescue
-                end
-              end
         end
       end
 
@@ -439,17 +445,9 @@ module Vmpooler
 
       # PENDING
       $redis.smembers('vmpooler__pending__' + pool['name']).each do |vm|
-            unless pool['timeout']
-              if $config[:config]['timeout']
-                pool['timeout'] = $config[:config]['timeout']
-              else
-                pool['timeout'] = 15
-              end
-            end
-
         if inventory[vm]
           begin
-                check_pending_vm(vm, pool['name'], pool['timeout'])
+            check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15)
           rescue
           end
         end
@@ -488,26 +486,9 @@ module Vmpooler
         end
       end
 
-          # LONG-RUNNING
-          $redis.smembers('vmpooler__running__' + pool['name']).each do |vm|
-            if $redis.hget('vmpooler__active__' + pool['name'], vm)
-              running = (Time.now - Time.parse($redis.hget('vmpooler__active__' + pool['name'], vm))) / 60 / 60
-              lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime']
-
-              if
-                (lifetime.to_i > 0) &&
-                (running.to_i >= lifetime.to_i)
-
-                $redis.smove('vmpooler__running__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
-
-                $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{lifetime} hours")
-              end
-            end
-          end
-
       # REPOPULATE
-          total = $redis.scard('vmpooler__ready__' + pool['name']) +
+      ready = $redis.scard('vmpooler__ready__' + pool['name'])
-                  $redis.scard('vmpooler__pending__' + pool['name'])
+      total = $redis.scard('vmpooler__pending__' + pool['name']) + ready
 
       begin
         if defined? $graphite
@@ -517,9 +498,20 @@ module Vmpooler
       rescue
       end
 
+      if $redis.get('vmpooler__empty__' + pool['name'])
+        unless ready == 0
+          $redis.del('vmpooler__empty__' + pool['name'])
+        end
+      else
+        if ready == 0
+          $redis.set('vmpooler__empty__' + pool['name'], 'true')
+          $logger.log('s', "[!] [#{pool['name']}] is empty")
+        end
+      end
+
       if total < pool['size']
         (1..(pool['size'] - total)).each do |_i|
-              if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit']
+          if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
             begin
               $redis.incr('vmpooler__tasks__clone')
 
@@ -536,10 +528,6 @@ module Vmpooler
           end
         end
       end
-
-          sleep(5)
-        end
-      end
     end
 
     def execute!

spec/vmpooler/pool_manager_spec.rb

@@ -169,6 +169,7 @@ describe 'Pool Manager' do
         allow(vm_host).to receive(:runtime).and_return true
         allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff'
 
+        expect(redis).to receive(:hget)
         expect(redis).to receive(:smove)
         expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
 
@@ -179,10 +180,10 @@ describe 'Pool Manager' do
         allow(pool_helper).to receive(:find_vm).and_return vm_host
         allow(vm_host).to receive(:runtime).and_return true
         allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn'
-        allow(vm_host).to receive_message_chain(:runtime, :bootTime).and_return Time.parse('2005-01-01')
 
+        expect(redis).to receive(:hget).with('vmpooler__active__pool1', 'vm1').and_return((Time.now - timeout*60*60).to_s)
         expect(redis).to receive(:smove)
-        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} minutes")
+        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} hours")
 
         subject._check_running_vm(vm, pool, timeout)
       end
@@ -211,6 +212,42 @@ describe 'Pool Manager' do
     end
   end
 
+  describe '#_check_pool' do
+    let(:pool_helper) { double('pool') }
+    let(:vsphere) { {pool => pool_helper} }
+    let(:config) { {
+      config: { task_limit: 10 },
+      pools: [ {'name' => 'pool1', 'size' => 5} ]
+    } }
+
+    before do
+      expect(subject).not_to be_nil
+      $vsphere = vsphere
+      allow(logger).to receive(:log)
+      allow(pool_helper).to receive(:find_folder)
+      allow(redis).to receive(:smembers).with('vmpooler__pending__pool1').and_return([])
+      allow(redis).to receive(:smembers).with('vmpooler__ready__pool1').and_return([])
+      allow(redis).to receive(:smembers).with('vmpooler__running__pool1').and_return([])
+      allow(redis).to receive(:smembers).with('vmpooler__completed__pool1').and_return([])
+      allow(redis).to receive(:smembers).with('vmpooler__discovered__pool1').and_return([])
+      allow(redis).to receive(:set)
+      allow(redis).to receive(:get).with('vmpooler__tasks__clone').and_return(0)
+      allow(redis).to receive(:get).with('vmpooler__empty__pool1').and_return(nil)
+    end
+
+    context 'logging' do
+
+      it 'logs empty pool' do
+        allow(redis).to receive(:scard).with('vmpooler__pending__pool1').and_return(0)
+        allow(redis).to receive(:scard).with('vmpooler__ready__pool1').and_return(0)
+
+        expect(logger).to receive(:log).with('s', "[!] [pool1] is empty")
+        subject._check_pool(config[:pools][0])
+      end
+
+    end
+  end
+
   describe '#_create_vm_snapshot' do
     let(:snapshot_manager) { 'snapshot_manager' }
     let(:pool_helper) { double('snapshot_manager') }