From 8459e2ec2138aefb71b8d7374da4c07a2c226c5c Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Mon, 11 May 2020 18:22:54 -0700 Subject: [PATCH] Move to using redis to track the ondemand clone task count so it can be displayed by dashboard and API --- lib/vmpooler/api/helpers.rb | 2 +- lib/vmpooler/pool_manager.rb | 23 +++++++++-------------- spec/unit/api/helpers_spec.rb | 4 ++-- spec/unit/pool_manager_spec.rb | 8 +++----- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 0b98143..f696e52 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -238,7 +238,7 @@ module Vmpooler queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend) queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend) - queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i queue[:booting] = 0 if queue[:booting] < 0 queue[:total] = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index aea6fc5..e68343f 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -35,9 +35,6 @@ module Vmpooler # Name generator for generating host names @name_generator = Spicy::Proton.new - @tasks = Concurrent::Hash.new - @tasks['ondemand_clone_count'] = 0 - # load specified providers from config file load_used_providers end @@ -403,12 +400,9 @@ module Vmpooler end raise ensure - if request_id - @tasks['ondemand_clone_count'] -= 1 - else - @redis.with_metrics do |redis| - redis.decr('vmpooler__tasks__clone') - end + @redis.with_metrics do |redis| + redis.decr('vmpooler__tasks__ondemandclone') if request_id + redis.decr('vmpooler__tasks__clone') unless request_id end end end @@ -1443,26 +1437,26 @@ module Vmpooler queue_key = 'vmpooler__odcreate__task' queue = redis.zrange(queue_key, 0, -1, with_scores: true) ondemand_clone_limit = $config[:config]['ondemand_clone_limit'] - @tasks['ondemand_clone_count'] = 0 unless @tasks['ondemand_clone_count'] queue.each do |request, score| - break unless @tasks['ondemand_clone_count'] < ondemand_clone_limit + clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i + break unless clone_count < ondemand_clone_limit pool_alias, pool, count, request_id = request.split(':') count = count.to_i provider = get_provider_for_pool(pool) - slots = ondemand_clone_limit - @tasks['ondemand_clone_count'] + slots = ondemand_clone_limit - clone_count break if slots == 0 if slots >= count count.times do - @tasks['ondemand_clone_count'] += 1 + redis.incr('vmpooler__tasks__ondemandclone') clone_vm(pool, provider, request_id, pool_alias) end redis.zrem(queue_key, request) else remaining_count = count - slots slots.times do - @tasks['ondemand_clone_count'] += 1 + redis.incr('vmpooler__tasks__ondemandclone') clone_vm(pool, provider, request_id, pool_alias) end redis.pipelined do @@ -1542,6 +1536,7 @@ module Vmpooler @redis.with_metrics do |redis| # Clear out the tasks manager, as we don't know about any tasks at this point redis.set('vmpooler__tasks__clone', 0) + redis.set('vmpooler__tasks__ondemandclone', 0) # Clear out vmpooler__migrations since stale entries may be left after a restart redis.del('vmpooler__migration') end diff --git a/spec/unit/api/helpers_spec.rb b/spec/unit/api/helpers_spec.rb index 62fed34..e6ce9a7 100644 --- a/spec/unit/api/helpers_spec.rb +++ b/spec/unit/api/helpers_spec.rb @@ -128,7 +128,7 @@ describe Vmpooler::API::Helpers do ] allow(redis).to receive(:pipelined).with(no_args).and_return [1,1] - allow(redis).to receive(:get).and_return 1 + allow(redis).to receive(:get).and_return(1,0) expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8}) end @@ -140,7 +140,7 @@ describe Vmpooler::API::Helpers do ] allow(redis).to receive(:pipelined).with(no_args).and_return [1,1] - allow(redis).to receive(:get).and_return 5 + allow(redis).to receive(:get).and_return(5,0) expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8}) end diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 547c496..baa1df8 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -867,14 +867,11 @@ EOT end end - it 'should reduce the ondemand clone count' do - count = { 'ondemand_clone_count' => 1 } - subject.instance_variable_set(:@tasks, count) + it 'should reduce the clone count' do redis_connection_pool.with do |redis| + expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone') subject._clone_vm(pool,provider,request_id,pool) end - count = subject.instance_variable_get(:@tasks) - expect(count['ondemand_clone_count']).to eq(0) end end end @@ -3167,6 +3164,7 @@ EOT it 'should run startup tasks only once' do redis_connection_pool.with do |redis| expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once + expect(redis).to receive(:set).with('vmpooler__tasks__ondemandclone', 0).once expect(redis).to receive(:del).with('vmpooler__migration').once end