diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 4f551eb..0b14668 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -235,180 +235,6 @@ EOT end end - describe '#move_pending_vm_to_ready' do - let(:host) { { 'hostname' => vm }} - - before do - expect(subject).not_to be_nil - end - - before(:each) do - redis_connection_pool.with do |redis| - create_pending_vm(pool,vm,redis) - end - end - - context 'when hostname matches VM name' do - it 'should move the VM from pending to ready pool' do - redis_connection_pool.with do |redis| - expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true) - expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) - subject.move_pending_vm_to_ready(vm, pool, redis) - expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false) - expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) - end - end - - it 'should log a message' do - expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") - - redis_connection_pool.with do |redis| - subject.move_pending_vm_to_ready(vm, pool, redis) - end - end - - it 'should receive time_to_ready_state metric' do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s) - expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/) - - subject.move_pending_vm_to_ready(vm, pool, redis) - end - end - - - it 'should set the boot time in redis' do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s) - expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil - subject.move_pending_vm_to_ready(vm, pool, redis) - expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil - # TODO Should we inspect the value to see if it's valid? - end - end - - it 'should not determine boot timespan if clone start time not set' do - redis_connection_pool.with do |redis| - expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil - subject.move_pending_vm_to_ready(vm, pool, redis) - expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to match(/\d\.\d{2}/) - end - end - - it 'should raise error if clone start time is not parsable' do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate') - expect{subject.move_pending_vm_to_ready(vm, pool, redis)}.to raise_error(/iamnotparsable_asdate/) - end - end - - it 'should save the last boot time' do - redis_connection_pool.with do |redis| - expect(redis.hget('vmpooler__lastboot', pool)).to be(nil) - subject.move_pending_vm_to_ready(vm, pool, redis) - expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil) - end - end - end - - context 'with request_id' do - context 'with a pending request' do - before(:each) do - allow(subject).to receive(:check_ondemand_request_ready) - config[:config]['ondemand_request_ttl'] = 20 - end - - it 'sets the vm as active' do - redis_connection_pool.with do |redis| - expect(Time).to receive(:now).and_return(current_time).at_least(:once) - redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool) - subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - expect(redis.hget("vmpooler__active__#{pool}", vm)).to eq(current_time.to_s) - expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to eq(current_time.to_s) - expect(redis.sismember("vmpooler__#{request_id}__#{pool}__#{pool}", vm)).to be true - end - end - - it 'logs that the vm is ready for the request' do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool) - expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") - - subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - end - end - end - - context 'when the request has been marked as failed' do - before(:each) do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed') - end - end - - it 'moves the vm to completed' do - redis_connection_pool.with do |redis| - expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time") - subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - end - end - - it 'returns nil' do - redis_connection_pool.with do |redis| - result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - expect(result).to be nil - end - end - end - - context 'when the request has been marked as deleted' do - before(:each) do - redis_connection_pool.with do |redis| - redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') - end - end - - it 'moves the vm to completed' do - redis_connection_pool.with do |redis| - expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted") - subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - end - end - - it 'returns nil' do - redis_connection_pool.with do |redis| - result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - expect(result).to be nil - end - end - end - - context 'with auth on the request' do - let(:user) { 'vmpuser' } - let(:platform_alias) { pool } - let(:platforms_string) { "#{platform_alias}:#{pool}:1" } - let(:score) { current_time.to_i } - before(:each) do - config[:config]['ondemand_request_ttl'] = 20 - allow(subject).to receive(:check_ondemand_request_ready) - redis_connection_pool.with do |redis| - create_ondemand_request_for_test(request_id, score, platforms_string, redis, user, token) - end - end - - it 'should specify auth data on the vm' do - redis_connection_pool.with do |redis| - allow(redis).to receive(:hset) - expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'token:token', token) - expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'token:user', user) - - subject.move_pending_vm_to_ready(vm, pool, redis, request_id) - end - end - end - end - end - describe '#check_ready_vm' do let(:ttl) { 5 } let(:poolconfig) { config[:pools][0] } @@ -771,168 +597,6 @@ EOT end end - describe '#_clone_vm' do - let (:pool_object) { { 'name' => pool } } - let (:redis_ttl) { 1 } - - before do - expect(subject).not_to be_nil - end - - let(:config) { - YAML.load(<<-EOT ---- -:config: - prefix: "prefix" -:redis: - ttl: #{redis_ttl} -EOT - ) - } - - context 'with no errors during cloning' do - before(:each) do - allow(metrics).to receive(:timing) - expect(metrics).to receive(:timing).with(/clone\./,/0/) - expect(provider).to receive(:create_vm).with(pool, String) - allow(logger).to receive(:log) - redis_connection_pool.with do |redis| - expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm) - end - end - - it 'should create a cloning VM' do - redis_connection_pool.with do |redis| - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - - subject._clone_vm(pool,provider) - - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1) - expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm}", 'template')).to eq(pool) - expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm}")).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm}", 'clone_time')).to_not be_nil - end - end - - it 'should decrement the clone tasks counter' do - redis_connection_pool.with do |redis| - redis.incr('vmpooler__tasks__clone') - redis.incr('vmpooler__tasks__clone') - expect(redis.get('vmpooler__tasks__clone')).to eq('2') - subject._clone_vm(pool,provider) - expect(redis.get('vmpooler__tasks__clone')).to eq('1') - end - end - - it 'should log a message that is being cloned from a template' do - expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] Starting to clone '(.+)'/) - - subject._clone_vm(pool,provider) - end - - it 'should log a message that it completed being cloned' do - expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned in [0-9.]+ seconds/) - - subject._clone_vm(pool,provider) - end - end - - context 'with an error during cloning' do - before(:each) do - expect(provider).to receive(:create_vm).with(pool, String).and_raise('MockError') - allow(logger).to receive(:log) - redis_connection_pool.with do |redis| - expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm) - end - end - - it 'should not create a cloning VM' do - redis_connection_pool.with do |redis| - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - - expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/) - - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - # Get the new VM Name from the pending pool queue as it should be the only entry - vm_name = redis.smembers("vmpooler__pending__#{pool}")[0] - expect(vm_name).to be_nil - end - end - - it 'should decrement the clone tasks counter' do - redis_connection_pool.with do |redis| - redis.incr('vmpooler__tasks__clone') - redis.incr('vmpooler__tasks__clone') - expect(redis.get('vmpooler__tasks__clone')).to eq('2') - expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/) - expect(redis.get('vmpooler__tasks__clone')).to eq('1') - end - end - - it 'should expire the vm metadata' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:expire) - expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/) - end - end - - it 'should raise the error' do - expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/) - end - - end - - context 'with request_id' do - before(:each) do - allow(metrics).to receive(:timing) - expect(metrics).to receive(:timing).with(/clone\./,/0/) - expect(provider).to receive(:create_vm).with(pool, String) - allow(logger).to receive(:log) - redis_connection_pool.with do |redis| - expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm) - end - end - - it 'should set request_id and pool_alias on the vm data' do - redis_connection_pool.with do |redis| - subject._clone_vm(pool,provider,request_id,pool) - expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool) - expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id) - end - end - - it 'should reduce the clone count' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone') - subject._clone_vm(pool,provider,request_id,pool) - end - end - end - - context 'with #check_dns_available' do - before(:each) do - allow(logger).to receive(:log) - end - it 'should error out if DNS already exists' do - vm_name = "foo" - resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true) - expect(subject).to receive(:generate_and_check_hostname).exactly(3).times.and_return([vm_name, true]) #skip this, make it available all times - expect(resolv).to receive(:getaddress).exactly(3).times.and_return("1.2.3.4") - expect(metrics).to receive(:increment).with("errors.staledns.#{pool}").exactly(3).times - expect{subject._clone_vm(pool,provider)}.to raise_error(/Unable to generate a unique hostname after/) - end - it 'should be successful if DNS does not exist' do - vm_name = "foo" - resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true) - expect(subject).to receive(:generate_and_check_hostname).and_return([vm_name, true]) - expect(resolv).to receive(:getaddress).exactly(1).times.and_raise(Resolv::ResolvError) - expect(provider).to receive(:create_vm).with(pool, String) - subject._clone_vm(pool,provider) - end - end - end - describe '#destroy_vm' do before do expect(subject).not_to be_nil @@ -954,112 +618,6 @@ EOT end end - describe "#_destroy_vm" do - before(:each) do - expect(subject).not_to be_nil - - redis_connection_pool.with do |redis| - create_completed_vm(vm,pool,redis,true) - end - - allow(provider).to receive(:destroy_vm).with(pool,vm).and_return(true) - - # Set redis configuration - config[:redis] = {} - config[:redis]['data_ttl'] = 168 - end - - context 'when redis data_ttl is not specified in the configuration' do - before(:each) do - config[:redis]['data_ttl'] = nil - end - - it 'should call redis expire with 0' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:expire).with("vmpooler__vm__#{vm}", 0) - subject._destroy_vm(vm,pool,provider) - end - end - end - - context 'when there is no redis section in the configuration' do - before(:each) do - config[:redis] = nil - end - - it 'should raise an error' do - expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError) - end - end - - context 'when a VM does not exist' do - before(:each) do - # As per base_spec, destroy_vm will return true if the VM does not exist - expect(provider).to receive(:destroy_vm).with(pool,vm).and_return(true) - end - - it 'should not raise an error' do - subject._destroy_vm(vm,pool,provider) - end - end - - context 'when the VM is destroyed without error' do - it 'should log a message the VM was destroyed' do - expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/) - allow(logger).to receive(:log) - - subject._destroy_vm(vm,pool,provider) - end - - it 'should emit a timing metric' do - allow(subject).to receive(:get_vm_usage_labels) - allow(metrics).to receive(:timing) - expect(metrics).to receive(:timing).with("destroy.#{pool}", String) - - subject._destroy_vm(vm,pool,provider) - end - - it 'should dereference the mutex' do - expect(subject).to receive(:dereference_mutex) - - subject._destroy_vm(vm,pool,provider) - end - end - - context 'when the VM destruction raises an eror' do - before(:each) do - # As per base_spec, destroy_vm will return true if the VM does not exist - expect(provider).to receive(:destroy_vm).with(pool,vm).and_raise('MockError') - end - - it 'should not log a message the VM was destroyed' do - expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/).exactly(0).times - allow(logger).to receive(:log) - - expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/) - end - - it 'should not emit a timing metric' do - expect(metrics).to receive(:timing).with("destroy.#{pool}", String).exactly(0).times - - expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/) - end - end - - context 'when the VM mutex is locked' do - let(:mutex) { Mutex.new } - before(:each) do - mutex.lock - end - - it 'should return' do - expect(subject).to receive(:vm_mutex).with(vm).and_return(mutex) - - expect(subject._destroy_vm(vm,pool,provider)).to eq(nil) - end - end - end - describe '#purge_unused_vms_and_resources' do let(:config) { YAML.load(<<-EOT --- @@ -4537,120 +4095,6 @@ EOT end end - describe '#create_ondemand_vms' do - context 'when requested does not have corresponding data' do - it 'logs an error' do - redis_connection_pool.with do |redis| - expect(logger).to receive(:log).with('s', "Failed to find odrequest for request_id '1111'") - subject.create_ondemand_vms('1111', redis) - end - end - end - - context 'with a request that has data' do - let(:request_string) { "#{pool}:#{pool}:1" } - before(:each) do - expect(Time).to receive(:now).and_return(current_time).at_least(:once) - redis_connection_pool.with do |redis| - create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis) - end - end - - it 'creates tasks for instances to be provisioned' do - redis_connection_pool.with do |redis| - allow(redis).to receive(:zadd) - expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}") - subject.create_ondemand_vms(request_id, redis) - end - end - - it 'adds a member to provisioning__processing' do - redis_connection_pool.with do |redis| - allow(redis).to receive(:zadd) - expect(redis).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id) - subject.create_ondemand_vms(request_id, redis) - end - end - end - end - - describe '#process_ondemand_vms' do - it 'returns the length of the queue' do - redis_connection_pool.with do |redis| - result = subject.process_ondemand_vms(redis) - expect(result).to eq(0) - end - end - - context 'with a request to create a single vm' do - let(:request_string) { "#{pool}:#{pool}:1" } - let(:pool_alias) { pool } - before(:each) do - config[:config]['ondemand_clone_limit'] = 10 - expect(subject).to receive(:get_provider_for_pool).and_return(provider) - redis_connection_pool.with do |redis| - create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis) - end - end - - it 'creates the vm' do - redis_connection_pool.with do |redis| - expect(subject).to receive(:clone_vm).with(pool, provider, request_id, pool_alias) - subject.process_ondemand_vms(redis) - end - end - end - - context 'with a request to create more instances than the limit' do - let(:request_string) { "#{pool}:#{pool}:5" } - let(:request_string_remaining) { "#{pool}:#{pool}:2" } - let(:pool_alias) { pool } - before(:each) do - config[:config]['ondemand_clone_limit'] = 3 - expect(subject).to receive(:get_provider_for_pool).and_return(provider) - redis_connection_pool.with do |redis| - create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis) - end - end - - it 'should create the maximum number of vms' do - redis_connection_pool.with do |redis| - expect(subject).to receive(:clone_vm).with(pool, provider, request_id, pool_alias).exactly(3).times - subject.process_ondemand_vms(redis) - end - end - - it 'should add the remaining number back as a new create task with the same score' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string_remaining}:#{request_id}") - subject.process_ondemand_vms(redis) - end - end - - it 'should return the number of requests processed' do - redis_connection_pool.with do |redis| - result = subject.process_ondemand_vms(redis) - expect(result).to eq(1) - end - end - end - - context 'when the limit has been reached' do - let(:clone_count) { { 'ondemand_clone_count' => 3 } } - before(:each) do - config[:config]['ondemand_clone_limit'] = 3 - subject.instance_variable_set(:@tasks, clone_count) - end - - it 'does not create any instances' do - redis_connection_pool.with do |redis| - expect(subject).to_not receive(:clone_vm) - subject.process_ondemand_vms(redis) - end - end - end - end - describe '#vms_ready?' do let(:request_string) { "#{pool}:#{pool}:5" } let(:platform_alias) { pool } @@ -4799,106 +4243,6 @@ EOT end end - describe '#request_expired?' do - let(:ondemand_request_ttl) { 5 } - let(:expiration_ttl) { 10 } - before(:each) do - config[:config]['ondemand_request_ttl'] = ondemand_request_ttl - config[:redis]['data_ttl'] = expiration_ttl - end - - context 'with a request that has taken too long to be filled' do - let(:expired_time) { (Time.now - 960).to_i } - before(:each) do - redis_connection_pool.with do |redis| - expect(subject).to receive(:remove_vms_for_failed_request) - create_ondemand_processing(request_id, expired_time, redis) - end - end - - it 'returns true when the request is expired' do - redis_connection_pool.with do |redis| - result = subject.request_expired?(request_id, expired_time, redis) - expect(result).to be true - end - end - - it 'logs a message that the request has expired' do - redis_connection_pool.with do |redis| - expect(logger).to receive(:log).with('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'") - subject.request_expired?(request_id, expired_time, redis) - end - end - - it 'removes the request from processing requests' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id) - subject.request_expired?(request_id, expired_time, redis) - end - end - - it 'sets the status as failed on the request hash' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'failed') - subject.request_expired?(request_id, expired_time, redis) - end - end - - it 'marks the request hash for expiration' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:expire).with("vmpooler__odrequest__#{request_id}", expiration_ttl * 60 * 60) - subject.request_expired?(request_id, expired_time, redis) - end - end - end - - context 'with a request that has been made within the ttl' do - before(:each) do - redis_connection_pool.with do |redis| - create_ondemand_processing(request_id, current_time, redis) - end - end - - it 'should return false' do - redis_connection_pool.with do |redis| - result = subject.request_expired?(request_id, current_time, redis) - expect(result).to be false - end - end - end - end - - describe '#remove_vms_for_failed_request)' do - let(:expiration_ttl) { 100 * 60 * 60 } - let(:platform_alias) { pool } - let(:platforms_string) { "#{platform_alias}:#{pool}:3" } - context 'with two vms marked as ready for the request' do - before(:each) do - redis_connection_pool.with do |redis| - create_ondemand_request_for_test(request_id, current_time, platforms_string, redis) - [vm,"#{vm}2"].each do |v| - create_running_vm(pool, v, redis) - redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v) - end - end - end - - it 'should remove the associated vms' do - redis_connection_pool.with do |redis| - expect(subject).to receive(:move_vm_queue).with(pool, String, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time").twice - subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis) - end - end - - it 'should mark the ready set for expiration' do - redis_connection_pool.with do |redis| - expect(redis).to receive(:expire).with("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl) - subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis) - end - end - end - end - describe 'check_ondemand_requests' do let(:threads) {[]} let(:maxloop) { 0 }