Move to using redis to track the ondemand clone task count so it can be displayed by dashboard and API

lib/vmpooler/api/helpers.rb

@@ -238,7 +238,7 @@ module Vmpooler
         queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
         queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
         queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
         queue[:booting] = 0 if queue[:booting] < 0
         queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i

lib/vmpooler/pool_manager.rb

@@ -35,9 +35,6 @@ module Vmpooler
       # Name generator for generating host names
       @name_generator = Spicy::Proton.new
 
-      @tasks = Concurrent::Hash.new
-      @tasks['ondemand_clone_count'] = 0
-
       # load specified providers from config file
       load_used_providers
     end
@@ -403,12 +400,9 @@ module Vmpooler
           end
           raise
         ensure
-          if request_id
+          @redis.with_metrics do |redis|
-            @tasks['ondemand_clone_count'] -= 1
+            redis.decr('vmpooler__tasks__ondemandclone') if request_id
-          else
+            redis.decr('vmpooler__tasks__clone') unless request_id
-            @redis.with_metrics do |redis|
-              redis.decr('vmpooler__tasks__clone')
-            end
           end
         end
       end
@@ -1443,26 +1437,26 @@ module Vmpooler
       queue_key = 'vmpooler__odcreate__task'
       queue = redis.zrange(queue_key, 0, -1, with_scores: true)
       ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
-      @tasks['ondemand_clone_count'] = 0 unless @tasks['ondemand_clone_count']
       queue.each do |request, score|
-        break unless @tasks['ondemand_clone_count'] < ondemand_clone_limit
+        clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
+        break unless clone_count < ondemand_clone_limit
 
         pool_alias, pool, count, request_id = request.split(':')
         count = count.to_i
         provider = get_provider_for_pool(pool)
-        slots = ondemand_clone_limit - @tasks['ondemand_clone_count']
+        slots = ondemand_clone_limit - clone_count
         break if slots == 0
 
         if slots >= count
           count.times do
-            @tasks['ondemand_clone_count'] += 1
+            redis.incr('vmpooler__tasks__ondemandclone')
             clone_vm(pool, provider, request_id, pool_alias)
           end
           redis.zrem(queue_key, request)
         else
           remaining_count = count - slots
           slots.times do
-            @tasks['ondemand_clone_count'] += 1
+            redis.incr('vmpooler__tasks__ondemandclone')
             clone_vm(pool, provider, request_id, pool_alias)
           end
           redis.pipelined do
@@ -1542,6 +1536,7 @@ module Vmpooler
       @redis.with_metrics do |redis|
         # Clear out the tasks manager, as we don't know about any tasks at this point
         redis.set('vmpooler__tasks__clone', 0)
+        redis.set('vmpooler__tasks__ondemandclone', 0)
         # Clear out vmpooler__migrations since stale entries may be left after a restart
         redis.del('vmpooler__migration')
       end

spec/unit/api/helpers_spec.rb

@@ -128,7 +128,7 @@ describe Vmpooler::API::Helpers do
       ]
 
       allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return 1
+      allow(redis).to receive(:get).and_return(1,0)
 
       expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8})
     end
@@ -140,7 +140,7 @@ describe Vmpooler::API::Helpers do
       ]
 
       allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:get).and_return 5
+      allow(redis).to receive(:get).and_return(5,0)
 
       expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8})
     end

spec/unit/pool_manager_spec.rb

@@ -867,14 +867,11 @@ EOT
         end
       end
 
-      it 'should reduce the ondemand clone count' do
+      it 'should reduce the clone count' do
-        count = { 'ondemand_clone_count' => 1 }
-        subject.instance_variable_set(:@tasks, count)
         redis_connection_pool.with do |redis|
+          expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone')
           subject._clone_vm(pool,provider,request_id,pool)
         end
-        count = subject.instance_variable_get(:@tasks)
-        expect(count['ondemand_clone_count']).to eq(0)
       end
     end
   end
@@ -3167,6 +3164,7 @@ EOT
       it 'should run startup tasks only once' do
         redis_connection_pool.with do |redis|
           expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once
+          expect(redis).to receive(:set).with('vmpooler__tasks__ondemandclone', 0).once
           expect(redis).to receive(:del).with('vmpooler__migration').once
         end