From 98a547b807051a7373edb82caaf58f1a7bc1d43e Mon Sep 17 00:00:00 2001 From: Spencer McElmurry Date: Fri, 26 Jul 2019 12:28:16 -0700 Subject: [PATCH] (POOLER-143) Add clone_target config change to API This allows the user to change the cluster in which the targeted pool will clone to. Upon configuration change, the thread will wake up and execute the change within 1 second. --- lib/vmpooler/api/v1.rb | 73 ++++++++++++++++++++++++++ lib/vmpooler/pool_manager.rb | 32 ++++++++++- spec/integration/api/v1/config_spec.rb | 65 ++++++++++++++++++++++- spec/unit/pool_manager_spec.rb | 55 ++++++++++++++++++- 4 files changed, 222 insertions(+), 3 deletions(-) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 3b3a98d..048e8a7 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -194,6 +194,26 @@ module Vmpooler result end + def update_clone_target(payload) + result = { 'ok' => false } + + pool_index = pool_index(pools) + pools_updated = 0 + sync_clone_targets + + payload.each do |poolname, clone_target| + unless pools[pool_index[poolname]]['clone_target'] == clone_target + pools[pool_index[poolname]]['clone_target'] == clone_target + backend.hset('vmpooler__config__clone_target', poolname, clone_target) + pools_updated += 1 + status 201 + end + end + status 200 unless pools_updated > 0 + result['ok'] = true + result + end + def sync_pool_templates pool_index = pool_index(pools) template_configs = backend.hgetall('vmpooler__config__template') @@ -222,6 +242,20 @@ module Vmpooler end end + def sync_clone_targets + pool_index = pool_index(pools) + clone_target_configs = backend.hgetall('vmpooler__config__clone_target') + unless clone_target_configs.nil? + clone_target_configs.each do |poolname, clone_target| + if pool_index.include? poolname + unless pools[pool_index[poolname]]['clone_target'] == clone_target + pools[pool_index[poolname]]['clone_target'] == clone_target + end + end + end + end + end + get '/' do sync_pool_sizes redirect to('/dashboard/') @@ -700,6 +734,14 @@ module Vmpooler invalid end + def invalid_pool(payload) + invalid = [] + payload.each do |pool, clone_target| + invalid << pool unless pool_exists?(pool) + end + invalid + end + post "#{api_prefix}/vm/:template/?" do content_type :json result = { 'ok' => false } @@ -1011,6 +1053,37 @@ module Vmpooler JSON.pretty_generate(result) end + post "#{api_prefix}/config/clonetarget/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + payload = JSON.parse(request.body.read) + + if payload + invalid = invalid_pool(payload) + if invalid.empty? + result = update_clone_target(payload) + else + invalid.each do |bad_template| + metrics.increment("config.invalid.#{bad_template}") + end + result[:bad_templates] = invalid + status 400 + end + else + metrics.increment('config.invalid.unknown') + status 404 + end + else + status 405 + end + + JSON.pretty_generate(result) + end + get "#{api_prefix}/config/?" do content_type :json result = { 'ok' => false } diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 23c4f52..f2242ff 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -680,6 +680,10 @@ module Vmpooler initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}") end + if options[:clone_target_change] + initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) + end + if options[:pool_template_change] initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname]) end @@ -698,6 +702,13 @@ module Vmpooler break unless ready_size == initial_ready_size end + if options[:clone_target_change] + clone_target = $redis.hget("vmpooler__config__clone_target}", options[:poolname]) + if clone_target + break unless clone_target == initial_clone_target + end + end + if options[:pool_template_change] configured_template = $redis.hget('vmpooler__config__template', options[:poolname]) if configured_template @@ -742,7 +753,7 @@ module Vmpooler loop_delay = (loop_delay * loop_delay_decay).to_i loop_delay = loop_delay_max if loop_delay > loop_delay_max end - sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true) + sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true) unless maxloop.zero? break if loop_count >= maxloop @@ -847,6 +858,21 @@ module Vmpooler $logger.log('s', "[*] [#{pool['name']}] is ready for use") end + def update_clone_target(pool) + mutex = pool_mutex(pool['name']) + return if mutex.locked? + clone_target = $redis.hget('vmpooler__config__clone_target', pool['name']) + return if clone_target.nil? + return if clone_target == pool['clone_target'] + $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}") + mutex.synchronize do + pool['clone_target'] = clone_target + # Remove all ready and pending VMs so new instances are created for the new clone_target + drain_pool(pool['name']) + end + $logger.log('s', "[*] [#{pool['name']}] is ready for use") + end + def remove_excess_vms(pool) ready = $redis.scard("vmpooler__ready__#{pool['name']}") total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready @@ -1080,6 +1106,10 @@ module Vmpooler # otherwise identify this change when running update_pool_size(pool) + # Check to see if a pool size change has been made via the configuration API + # Additionally, a pool will drain ready and pending instances + update_clone_target(pool) + repopulate_pool_vms(pool['name'], provider, pool_check_response, pool['size']) # Remove VMs in excess of the configured pool size diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v1/config_spec.rb index 5c3e0ed..0b73d54 100644 --- a/spec/integration/api/v1/config_spec.rb +++ b/spec/integration/api/v1/config_spec.rb @@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do 'experimental_features' => true }, pools: [ - {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1'}, + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, {'name' => 'pool2', 'size' => 10} ], statsd: { 'prefix' => 'stats_prefix'}, @@ -223,6 +223,69 @@ describe Vmpooler::API::V1 do end end + describe 'POST /config/clonetarget' do + it 'changes the clone target' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'changes a pool size for multiple pools' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1","pool2":"cluster2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when a specified pool does not exist' do + post "#{prefix}/config/clonetarget", '{"pool10":"cluster1"}' + expect_json(ok = false, http = 400) + expected = { + ok: false, + bad_templates: ['pool10'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 200 when no change is required' do + post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster"}' + expect_json(ok = true, http = 200) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'succeeds with 201 when at least one pool changes' do + post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster","pool2":"cluster2"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end + describe 'GET /config' do let(:prefix) { '/api/v1' } diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 7cef5a4..3b41e62 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -2390,6 +2390,59 @@ EOT end end + describe 'update_clone_target' do + let(:newtarget) { 'cluster2' } + let(:config) { + YAML.load(<<-EOT +--- +:pools: + - name: #{pool} + clone_target: 'cluster1' +EOT + ) + } + let(:poolconfig) { config[:pools][0] } + + context 'with a locked mutex' do + + let(:mutex) { Mutex.new } + before(:each) do + mutex.lock + expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex) + end + + it 'should return nil' do + expect(subject.update_clone_target(poolconfig)).to be_nil + end + end + + it 'should get the pool clone target configuration from redis' do + expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool) + + subject.update_clone_target(poolconfig) + end + + it 'should return when clone_target is not set in redis' do + expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(nil) + + expect(subject.update_clone_target(poolconfig)).to be_nil + end + + it 'should return when no change in configuration is required' do + expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return('cluster1') + + expect(subject.update_clone_target(poolconfig)).to be_nil + end + + it 'should update the clone target' do + expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(newtarget) + + subject.update_clone_target(poolconfig) + + expect(poolconfig['clone_target']).to eq(newtarget) + end + end + describe "#execute!" do let(:config) { YAML.load(<<-EOT @@ -2759,7 +2812,7 @@ EOT expect(subject).to receive(:sleep).exactly(2).times expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, false, false, true) - + subject.sleep_with_wakeup_events(loop_delay) end