From 52b60b074cba9c76b15d297a09d964d40429d850 Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Tue, 11 Feb 2020 16:34:19 -0800 Subject: [PATCH 1/2] (POOLER-153) Add endpoint for resetting a pool This commit adds a capability to vmpooler to reset a pool, deleting its ready and pending instances and replacing them with fresh ones. Without this change vmpooler does not offer a mechanism to reset a pool without also changing its template. --- docs/API.md | 26 ++++++ lib/vmpooler/api/v1.rb | 49 +++++++++++ lib/vmpooler/pool_manager.rb | 24 +++++- spec/integration/api/v1/poolreset.rb | 117 +++++++++++++++++++++++++++ spec/unit/pool_manager_spec.rb | 22 +++++ 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 spec/integration/api/v1/poolreset.rb diff --git a/docs/API.md b/docs/API.md index 560d4b1..d30b329 100644 --- a/docs/API.md +++ b/docs/API.md @@ -773,3 +773,29 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templat "ok": true } ``` + +##### POST /poolreset + +Clear all pending and ready instances in a pool, and deploy replacements + +All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned. + +When a pool reset is requested a 201 status will be returned. + +A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances. + +For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured. + +Responses: +* 201 - Pool reset requested received +* 400 - An invalid configuration was provided causing requested changes to fail +* 404 - An unknown error occurred +* 405 - The endpoint is disabled because experimental features are disabled +``` +$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v1/poolreset +``` +```json +{ + "ok": true +} +``` diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 1f63fc7..bff7c19 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -200,6 +200,17 @@ module Vmpooler result end + def reset_pool(payload) + result = { 'ok' => false } + + payload.each do |poolname, count| + backend.sadd('vmpooler__poolreset', poolname) + end + status 201 + result['ok'] = true + result + end + def update_clone_target(payload) result = { 'ok' => false } @@ -1063,6 +1074,44 @@ module Vmpooler JSON.pretty_generate(result) end + post "#{api_prefix}/poolreset/?" do + content_type :json + result = { 'ok' => false } + + if config['experimental_features'] + need_token! if Vmpooler::API.settings.config[:auth] + + begin + payload = JSON.parse(request.body.read) + if payload + invalid = invalid_templates(payload) + if invalid.empty? + result = reset_pool(payload) + else + invalid.each do |bad_pool| + metrics.increment("poolreset.invalid.#{bad_pool}") + end + result[:bad_pools] = invalid + status 400 + end + else + metrics.increment('poolreset.invalid.unknown') + status 404 + end + rescue JSON::ParserError + status 400 + result = { + 'ok' => false, + 'message' => 'JSON payload could not be parsed' + } + end + else + status 405 + end + + JSON.pretty_generate(result) + end + post "#{api_prefix}/config/clonetarget/?" do content_type :json result = { 'ok' => false } diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 6c07bde..80d7898 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -680,6 +680,10 @@ module Vmpooler # - Fires when a template configuration update is requested # - Additional options # :poolname + # :pool_reset + # - Fires when a pool reset is requested + # - Additional options + # :poolname # def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {}) exit_by = Time.now + loop_delay @@ -726,6 +730,10 @@ module Vmpooler end end + if options[:pool_reset] + break if $redis.sismember('vmpooler__poolreset', options[:poolname]) + end + end break if time_passed?(:exit_by, exit_by) @@ -763,7 +771,7 @@ module Vmpooler loop_delay = (loop_delay * loop_delay_decay).to_i loop_delay = loop_delay_max if loop_delay > loop_delay_max end - sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true) + sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true) unless maxloop.zero? break if loop_count >= maxloop @@ -917,6 +925,17 @@ module Vmpooler end end + def reset_pool(pool) + poolname = pool['name'] + return unless $redis.sismember('vmpooler__poolreset', poolname) + $redis.srem('vmpooler__poolreset', poolname) + mutex = pool_mutex(poolname) + mutex.synchronize do + drain_pool(poolname) + $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances") + end + end + def create_inventory(pool, provider, pool_check_response) inventory = {} begin @@ -1125,6 +1144,9 @@ module Vmpooler # Remove VMs in excess of the configured pool size remove_excess_vms(pool) + # Reset a pool when poolreset is requested from the API + reset_pool(pool) + pool_check_response end diff --git a/spec/integration/api/v1/poolreset.rb b/spec/integration/api/v1/poolreset.rb new file mode 100644 index 0000000..9c87c6c --- /dev/null +++ b/spec/integration/api/v1/poolreset.rb @@ -0,0 +1,117 @@ +require 'spec_helper' +require 'rack/test' + +describe Vmpooler::API::V1 do + include Rack::Test::Methods + + def app() + Vmpooler::API + end + + let(:config) { + { + config: { + 'site_name' => 'test pooler', + 'vm_lifetime_auth' => 2, + 'experimental_features' => true + }, + pools: [ + {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'}, + {'name' => 'pool2', 'size' => 10} + ], + statsd: { 'prefix' => 'stats_prefix'}, + alias: { 'poolone' => 'pool1' }, + pool_names: [ 'pool1', 'pool2', 'poolone' ] + } + } + + describe '/poolreset' do + let(:prefix) { '/api/v1' } + let(:metrics) { Vmpooler::DummyStatsd.new } + + let(:current_time) { Time.now } + + before(:each) do + app.settings.set :config, config + app.settings.set :redis, redis + app.settings.set :metrics, metrics + app.settings.set :config, auth: false + create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time) + end + + describe 'POST /poolreset' do + it 'refreshes ready and pending instances from a pool' do + post "#{prefix}/poolreset", '{"pool1":"1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails on nonexistent pools' do + post "#{prefix}/poolreset", '{"poolpoolpool":"1"}' + expect_json(ok = false, http = 400) + end + + it 'resets multiple pools' do + post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}' + expect_json(ok = true, http = 201) + + expected = { ok: true } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'fails when not all pools exist' do + post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}' + expect_json(ok = false, http = 400) + + expected = { + ok: false, + bad_pools: ['pool3'] + } + + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + context 'with experimental features disabled' do + before(:each) do + config[:config]['experimental_features'] = false + end + + it 'should return 405' do + post "#{prefix}/poolreset", '{"pool1":"1"}' + expect_json(ok = false, http = 405) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + + it 'should return 400 for invalid json' do + post "#{prefix}/poolreset", '{"pool1":"1}' + expect_json(ok = false, http = 400) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'should return 400 with a bad pool name' do + post "#{prefix}/poolreset", '{"pool11":"1"}' + expect_json(ok = false, http = 400) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + + it 'should return 404 when there is no payload' do + post "#{prefix}/poolreset" + expect_json(ok = false, http = 404) + + expected = { ok: false } + expect(last_response.body).to eq(JSON.pretty_generate(expected)) + end + end + end +end diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 3b41e62..8cc9d05 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -2874,6 +2874,28 @@ EOT end end end + + describe 'with the pool_reset wakeup option' do + let(:wakeup_option) {{ + :pool_reset => true, + :poolname => pool + }} + + let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur + + context 'when a pool reset is requested' do + before(:each) do + redis.sadd('vmpooler__poolreset', pool) + end + + it 'should sleep until the reset request is detected' do + expect(subject).to receive(:sleep).exactly(3).times + expect(redis).to receive(:sismember).with('vmpooler__poolreset', pool).and_return(false,false,true) + + subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option) + end + end + end end describe "#check_pool" do From 0a21ac563d3619c55da3795b4c3b0623da3b5279 Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Wed, 12 Feb 2020 20:53:08 -0800 Subject: [PATCH 2/2] Update travis tests to use latest ruby versions This commit updates travis to use ruby 2.4.9, 2.5.7, and jruby 9.2.9.0 for tests. Test with latest z releases of ruby versions --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 978a549..cd54d84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,22 +4,22 @@ language: ruby matrix: include: - - rvm: 2.4.5 + - rvm: 2.4.9 env: "CHECK=rubocop" - - rvm: 2.4.5 + - rvm: 2.4.9 env: "CHECK=test" - - rvm: 2.5.3 + - rvm: 2.5.7 env: "CHECK=test" - - rvm: jruby-9.2.5.0 + - rvm: jruby-9.2.9.0 env: "CHECK=test" # Remove the allow_failures section once # Rubocop is required for Travis to pass a build allow_failures: - - rvm: 2.4.5 + - rvm: 2.4.9 env: "CHECK=rubocop" install: