diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 55cbf53..43f1f9a 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -359,8 +359,7 @@ module Vmpooler status 201 platforms_with_aliases = [] - payload.delete('request_id') - payload.each do |poolname, count| + payload.reject { |k,v| k == 'request_id' }.each do |poolname, count| selection = evaluate_template_aliases(poolname, count) selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } end @@ -800,22 +799,30 @@ module Vmpooler result = { 'ok' => false } - payload = JSON.parse(request.body.read) + begin + payload = JSON.parse(request.body.read) - if payload - invalid = invalid_templates(payload.reject { |k,v| k == 'request_id' }) - if invalid.empty? - result = generate_ondemand_request(payload) - else - result[:bad_templates] = invalid - invalid.each do |bad_template| - metrics.increment('ondemandrequest.invalid.' + bad_template) + if payload + invalid = invalid_templates(payload.reject { |k,v| k == 'request_id' }) + if invalid.empty? + result = generate_ondemand_request(payload) + else + result[:bad_templates] = invalid + invalid.each do |bad_template| + metrics.increment('ondemandrequest.invalid.' + bad_template) + end + status 404 end + else + metrics.increment('ondemandrequest.invalid.unknown') status 404 end - else - metrics.increment('ondemandrequest.invalid.unknown') - status 404 + rescue JSON::ParserError + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } end JSON.pretty_generate(result) @@ -908,16 +915,16 @@ module Vmpooler def check_ondemand_request(request_id) result = { 'ok' => false } - result['request_id'] = request_id - result['ready'] = false request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") if request_hash.empty? result['message'] = "no request found for request_id '#{request_id}'" return result end - status 202 + result['request_id'] = request_id + result['ready'] = false result['ok'] = true + status 202 if request_hash['status'] == 'ready' result['ready'] = true diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 46dbe6e..fc384f1 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -36,6 +36,7 @@ module Vmpooler @name_generator = Spicy::Proton.new @tasks = Concurrent::Hash.new + @tasks['ondemand_clone_count'] = 0 # load specified providers from config file load_used_providers @@ -113,17 +114,13 @@ module Vmpooler clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone') return true unless clone_stamp - # if clone_stamp == 'QUEUED' - # $logger.log('s', "Waiting for clone_stamp. Got 'QUEUED'.") - # return true - # end time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60 if time_since_clone > timeout if exists - pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if $request_id + pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id redis.multi redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm) - redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}") if request_id + result = redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id redis.exec $metrics.increment("errors.markedasfailed.#{pool}") $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") @@ -271,12 +268,7 @@ module Vmpooler @redis.with_metrics do |redis| # Check that VM is within defined lifetime checkouttime = redis.hget('vmpooler__active__' + pool, vm) - # return if checkouttime == 'QUEUED' if checkouttime - # if checkouttime == 'QUEUED' - # $logger.log('s', "checkouttime is #{checkouttime}") - # return - # end time_since_checkout = Time.now - Time.parse(checkouttime) running = time_since_checkout / 60 / 60 @@ -376,6 +368,8 @@ module Vmpooler redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias redis.exec + vm_hash = redis.hgetall("vmpooler__vm__#{new_vmname}") + begin $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'") start = Time.now @@ -425,11 +419,13 @@ module Vmpooler mutex.synchronize do @redis.with_metrics do |redis| - redis.hdel('vmpooler__active__' + pool, vm) - redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + redis.pipelined do + redis.hdel('vmpooler__active__' + pool, vm) + redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) - # Auto-expire metadata key - redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + # Auto-expire metadata key + redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + end start = Time.now @@ -516,7 +512,7 @@ module Vmpooler if provider_purge Thread.new do begin - purge_vms_and_folders(provider.to_s) + purge_vms_and_folders($providers[provider.to_s]) rescue StandardError => e $logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}") end @@ -527,13 +523,14 @@ module Vmpooler end # Return a list of pool folders - def pool_folders(provider_name) + def pool_folders(provider) + provider_name = provider.name folders = {} $config[:pools].each do |pool| next unless pool['provider'] == provider_name folder_parts = pool['folder'].split('/') - datacenter = $providers[provider_name].get_target_datacenter_from_config(pool['name']) + datacenter = provider.get_target_datacenter_from_config(pool['name']) folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}" end folders @@ -550,8 +547,8 @@ module Vmpooler def purge_vms_and_folders(provider) configured_folders = pool_folders(provider) base_folders = get_base_folders(configured_folders) - whitelist = $providers[provider].provider_config['folder_whitelist'] - $providers[provider].purge_unconfigured_folders(base_folders, configured_folders, whitelist) + whitelist = provider.provider_config['folder_whitelist'] + provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist) end def create_vm_disk(pool_name, vm, disk_size, provider) @@ -855,20 +852,25 @@ module Vmpooler end if options[:pool_reset] - break if redis.sismember('vmpooler__poolreset', options[:poolname]) - end - - if options[:pending_vm] - pending = redis.scard("vmpooler__pending__#{options[:poolname]}") + pending = redis.sismember('vmpooler__poolreset', options[:poolname]) break if pending end - if options[:ondemand_request] - break if redis.zcard('vmpooler__provisioning__request') - break if redis.zcard('vmpooler__provisioning__processing') - break if redis.zcard('vmpooler__odcreate__task') + if options[:pending_vm] + pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}") + break unless pending_vm_count == 0 end + if options[:ondemand_request] + redis.multi + redis.zcard('vmpooler__provisioning__request') + redis.zcard('vmpooler__provisioning__processing') + redis.zcard('vmpooler__odcreate__task') + od_request, od_processing, od_createtask = redis.exec + break unless od_request == 0 + break unless od_processing == 0 + break unless od_createtask == 0 + end end break if time_passed?(:exit_by, exit_by) @@ -1462,6 +1464,8 @@ module Vmpooler in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1) in_progress_requests&.each do |request_id| next unless vms_ready?(request_id, redis) + $logger.log('s', 'vms are ready') + redis.multi redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'ready') redis.zrem('vmpooler__provisioning__processing', request_id) diff --git a/spec/helpers.rb b/spec/helpers.rb index 917ccc0..daf3dc8 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -129,3 +129,16 @@ end def pool_has_ready_vm?(pool, vm, redis) !!redis.sismember('vmpooler__ready__' + pool, vm) end + +def create_ondemand_request_for_test(request_id, score, platforms_string, redis) + redis.zadd('vmpooler__provisioning__request', score, request_id) + redis.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) +end + +def set_ondemand_request_ready(request_id, redis) + redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'ready') +end + +def create_ondemand_vm(vmname, request_id, pool, pool_alias, redis) + redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vmname) +end diff --git a/spec/integration/api/v1/ondemandvm_spec.rb b/spec/integration/api/v1/ondemandvm_spec.rb index 48ced51..34f5f49 100644 --- a/spec/integration/api/v1/ondemandvm_spec.rb +++ b/spec/integration/api/v1/ondemandvm_spec.rb @@ -5,30 +5,34 @@ describe Vmpooler::API::V1 do include Rack::Test::Methods def app() - Vmpooler::API - end + Vmpooler::API end - describe '/vm' do + describe '/ondemandvm' do let(:prefix) { '/api/v1' } let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { 'site_name' => 'test pooler', - 'vm_lifetime_auth' => 2 + 'vm_lifetime_auth' => 2, + 'backend_weight' => { + 'compute1' => 5 + } }, pools: [ {'name' => 'pool1', 'size' => 0}, - {'name' => 'pool2', 'size' => 0}, - {'name' => 'pool3', 'size' => 0} + {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute1'}, + {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1'} ], alias: { 'poolone' => ['pool1'] }, - pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone', 'genericpool' ] + pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ] } } let(:current_time) { Time.now } let(:vmname) { 'abcdefghijkl' } let(:checkoutlock) { Mutex.new } + let(:redis) { MockRedis.new } + let(:uuid) { SecureRandom.uuid } before(:each) do app.settings.set :config, config @@ -37,16 +41,18 @@ describe Vmpooler::API::V1 do app.settings.set :config, auth: false app.settings.set :checkoutlock, checkoutlock create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + config[:pools].each do |pool| + redis.sadd('vmpooler__pools', pool['name']) + end end describe 'POST /ondemandvm' do - let(:uuid) { SecureRandom.uuid } context 'with a configured pool' do it 'generates a request_id when none is provided' do expect(SecureRandom).to receive(:uuid).and_return(uuid) post "#{prefix}/ondemandvm", '{"pool1":"1"}' - expect_json(ok = true, http = 201) + expect_json(true, 201) expected = { "ok": true, @@ -57,7 +63,7 @@ describe Vmpooler::API::V1 do it 'uses the given request_id when provided' do post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' - expect_json(ok = true, http = 201) + expect_json(true, 201) expected = { "ok": true, @@ -66,10 +72,10 @@ describe Vmpooler::API::V1 do expect(last_response.body).to eq(JSON.pretty_generate(expected)) end - it 'returns 404 when the request_id has been used' do + it 'returns 409 conflict error when the request_id has been used' do post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}' - expect_json(ok = false, http = 409) + expect_json(false, 409) expected = { "ok": false, @@ -78,13 +84,50 @@ describe Vmpooler::API::V1 do } expect(last_response.body).to eq(JSON.pretty_generate(expected)) end + + it 'uses a configured platform to fulfill a ondemand request' do + expect(SecureRandom).to receive(:uuid).and_return(uuid) + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + expect_json(true, 201) + expected = { + "ok": true, + "request_id": uuid + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'creates a provisioning request in redis' do + expect(SecureRandom).to receive(:uuid).and_return(uuid) + expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1) + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + end + + it 'sets a platform string in redis for the request to indicate selected platforms' do + expect(SecureRandom).to receive(:uuid).and_return(uuid) + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1') + post "#{prefix}/ondemandvm", '{"poolone":"1"}' + end + + context 'with auth configured' do + + it 'sets the token and user' do + app.settings.set :config, auth: true + expect(SecureRandom).to receive(:uuid).and_return(uuid) + allow(redis).to receive(:hset) + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345') + expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe') + post "#{prefix}/ondemandvm", '{"pool1":"1"}', { + 'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345' + } + end + end end context 'with a pool that is not configured' do let(:badpool) { 'pool4' } it 'returns the bad template' do post "#{prefix}/ondemandvm", '{"pool4":"1"}' - expect_json(ok = false, http = 404) + expect_json(false, 404) expected = { "ok": false, @@ -93,6 +136,74 @@ describe Vmpooler::API::V1 do expect(last_response.body).to eq(JSON.pretty_generate(expected)) end end + + it 'returns 400 and a message when JSON is invalid' do + post "#{prefix}/ondemandvm", '{"pool1":"1}' + expect_json(false, 400) + expected = { + "ok": false, + "message": "JSON payload could not be parsed" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + describe 'GET /ondemandvm' do + it 'returns 404 with message when request is not found' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(false, 404) + expected = { + "ok": false, + "message": "no request found for request_id '#{uuid}'" + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'when the request is found' do + let(:score) { current_time } + let(:platforms_string) { 'pool1:pool1:1' } + before(:each) do + create_ondemand_request_for_test(uuid, score, platforms_string, redis) + end + + it 'returns 202 while the request is waiting' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 202) + expected = { + "ok": true, + "request_id": uuid, + "ready": false, + "pool1": { + "ready": "0", + "pending": "1" + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with ready instances' do + before(:each) do + create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis) + set_ondemand_request_ready(uuid, redis) + end + + it 'returns 200 with hostnames when the request is ready' do + get "#{prefix}/ondemandvm/#{uuid}" + expect_json(true, 200) + expected = { + "ok": true, + "request_id": uuid, + "ready": true, + "pool1": { + "hostname": [ + vmname + ] + } + } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end end end end diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 055544d..e46de2a 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -15,7 +15,9 @@ describe 'Pool Manager' do let(:vm) { 'vm1' } let(:timeout) { 5 } let(:host) { double('host') } - let(:token) { 'token1234'} + let(:token) { 'token1234' } + let(:request_id) { '1234' } + let(:current_time) { Time.now } let(:provider_options) { {} } let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new( @@ -25,6 +27,7 @@ describe 'Pool Manager' do timeout: 5 ) { MockRedis.new } } + let(:redis) { MockRedis.new } let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'mock_provider', provider_options) } @@ -217,6 +220,18 @@ EOT subject.fail_pending_vm(vm, pool, timeout, redis, true) end end + + context 'with request_id' do + + it 'creates a new odcreate task' do + redis_connection_pool.with do |redis| + redis.hset("vmpooler__vm__#{vm}", 'clone',(Time.now - 900).to_s) + redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool) + subject.fail_pending_vm(vm, pool, timeout, redis, true, request_id) + expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to eq(["#{pool}:#{pool}:1:#{request_id}"]) + end + end + end end describe '#move_pending_vm_to_ready' do @@ -294,6 +309,28 @@ EOT end end end + + context 'with request_id' do + it 'sets the vm as active' do + redis_connection_pool.with do |redis| + expect(Time).to receive(:now).and_return(current_time).at_least(:once) + redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool) + subject.move_pending_vm_to_ready(vm, pool, redis, request_id) + expect(redis.hget("vmpooler__active__#{pool}", vm)).to eq(current_time.to_s) + expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to eq(current_time.to_s) + expect(redis.sismember("vmpooler__#{request_id}__#{pool}__#{pool}", vm)).to be true + end + end + + it 'logs that the vm is ready for the request' do + redis_connection_pool.with do |redis| + redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool) + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") + + subject.move_pending_vm_to_ready(vm, pool, redis, request_id) + end + end + end end describe '#check_ready_vm' do @@ -711,12 +748,10 @@ EOT subject._clone_vm(pool,provider) expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1) - # Get the new VM Name from the pending pool queue as it should be the only entry - vm_name = redis.smembers("vmpooler__pending__#{pool}")[0] - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(pool) - expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm_name}")).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil + expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil + expect(redis.hget("vmpooler__vm__#{vm}", 'template')).to eq(pool) + expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm}")).to_not be_nil + expect(redis.hget("vmpooler__vm__#{vm}", 'clone_time')).to_not be_nil end end @@ -785,6 +820,37 @@ EOT it 'should raise the error' do expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/) end + + end + + context 'with request_id' do + before(:each) do + allow(metrics).to receive(:timing) + expect(metrics).to receive(:timing).with(/clone\./,/0/) + expect(provider).to receive(:create_vm).with(pool, String) + allow(logger).to receive(:log) + redis_connection_pool.with do |redis| + expect(subject).to receive(:find_unique_hostname).with(pool, redis).and_return(vm) + end + end + + it 'should set request_id and pool_alias on the vm data' do + redis_connection_pool.with do |redis| + subject._clone_vm(pool,provider,request_id,pool) + expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool) + expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id) + end + end + + it 'should reduce the ondemand clone count' do + count = { 'ondemand_clone_count' => 1 } + subject.instance_variable_set(:@tasks, count) + redis_connection_pool.with do |redis| + subject._clone_vm(pool,provider,request_id,pool) + end + count = subject.instance_variable_get(:@tasks) + expect(count['ondemand_clone_count']).to eq(0) + end end end @@ -831,9 +897,8 @@ EOT it 'should call redis expire with 0' do redis_connection_pool.with do |redis| - expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be_nil + expect(redis).to receive(:expire).with("vmpooler__vm__#{vm}", 0) subject._destroy_vm(vm,pool,provider) - expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be_nil end end end @@ -1238,15 +1303,15 @@ EOT } it 'should return a list of pool folders' do - expect($providers[provider_name]).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter) + expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter) - expect(subject.pool_folders(provider_name)).to eq(expected_response) + expect(subject.pool_folders(provider)).to eq(expected_response) end it 'should raise an error when the provider fails to get the datacenter' do - expect($providers[provider_name]).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror') + expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror') - expect{ subject.pool_folders(provider_name) }.to raise_error(RuntimeError, 'mockerror') + expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror') end end @@ -1277,16 +1342,16 @@ EOT it 'should run purge_unconfigured_folders' do expect(subject).to receive(:pool_folders).and_return(configured_folders) - expect($providers[provider_name]).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist) - expect($providers[provider_name]).to receive(:provider_config).and_return({}) + expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist) + expect(provider).to receive(:provider_config).and_return({}) subject.purge_vms_and_folders(provider) end it 'should raise any errors' do expect(subject).to receive(:pool_folders).and_return(configured_folders) - expect($providers[provider_name]).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror') - expect($providers[provider_name]).to receive(:provider_config).and_return({}) + expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror') + expect(provider).to receive(:provider_config).and_return({}) expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror') end @@ -3196,11 +3261,6 @@ EOT let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur context 'when a pool reset is requested' do - before(:each) do - redis_connection_pool.with do |redis| - redis.sadd('vmpooler__poolreset', pool) - end - end it 'should sleep until the reset request is detected' do redis_connection_pool.with do |redis| @@ -3212,6 +3272,62 @@ EOT end end end + + describe 'with the pending_vm wakeup option' do + let(:wakeup_option) {{ + :pending_vm => true, + :poolname => pool + }} + + let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur + + context 'when a pending_vm is detected' do + + it 'should sleep until the pending instance' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(0,0,1) + end + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + end + end + + describe 'with the ondemand_request wakeup option' do + let(:wakeup_option) {{ :ondemand_request => true }} + + let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur + + it 'should sleep until the provisioning request is detected' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:multi).and_return('OK').exactly(3).times + expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[1,0,0]) + end + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + + it 'should sleep until provisioning processing is detected' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:multi).and_return('OK').exactly(3).times + expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,1,0]) + end + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + + it 'should sleep until ondemand creation task is detected' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:multi).and_return('OK').exactly(3).times + expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,0,1]) + end + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + end end describe "#check_pool" do @@ -4478,9 +4594,96 @@ EOT subject._check_pool(config[:pools][0],provider) end end + end - # + describe 'process_ondemand_requests' do + context 'with no requests' do + it 'returns 0' do + result = subject.process_ondemand_requests + expect(result).to eq(0) + end + it 'runs process_ondemand_vms' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:process_ondemand_vms).with(redis).and_return(0) + subject.process_ondemand_requests + end + end + it 'checks ready requests' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:check_ondemand_requests_ready).with(redis).and_return(0) + subject.process_ondemand_requests + end + end + end + + context 'with provisioning requests' do + before(:each) do + redis_connection_pool.with do |redis| + redis.zadd('vmpooler__provisioning__request', current_time, request_id) + end + end + + it 'returns the number of requests processed' do + result = subject.process_ondemand_requests + expect(result).to eq(1) + end + + it 'runs create_ondemand_vms for each request' do + redis_connection_pool.with do |redis| + expect(subject).to receive(:create_ondemand_vms).with(request_id, redis) + subject.process_ondemand_requests + end + end + end + end + + describe '#create_ondemand_vms' do + context 'when requested does not have corresponding data' do + #end + it 'logs an error' do + redis_connection_pool.with do |redis| + expect(logger).to receive(:log).with('s', "Failed to find odrequest for request_id '1111'") + #expect(redis).to receive(:zrem).with('vmpooler__provisioning__request', '1111') + subject.create_ondemand_vms('1111', redis) + end + end + end + + context 'with a request that has data' do + let(:request_string) { "#{pool}:#{pool}:1" } + before(:each) do + expect(Time).to receive(:now).and_return(current_time).at_least(:once) + redis_connection_pool.with do |redis| + create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis) + end + end + + it 'creates tasks for instances to be provisioned' do + redis_connection_pool.with do |redis| + allow(redis).to receive(:zadd) + expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}") + subject.create_ondemand_vms(request_id, redis) + end + end + + it 'adds a member to provisioning__processing' do + redis_connection_pool.with do |redis| + allow(redis).to receive(:zadd) + expect(redis).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id) + subject.create_ondemand_vms(request_id, redis) + end + end + end + end + + describe '#process_ondemand_vms' do + it 'returns the length of the queue' do + redis_connection_pool.with do |redis| + result = subject.process_ondemand_vms(redis) + expect(result).to eq(0) + end + end end end