(POOLER-153) Add endpoint for resetting a pool

This commit adds a capability to vmpooler to reset a pool, deleting its ready and pending instances and replacing them with fresh ones. Without this change vmpooler does not offer a mechanism to reset a pool without also changing its template.

docs/API.md

@@ -773,3 +773,29 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templat
   "ok": true
 }
 ```
+
+##### POST /poolreset
+
+Clear all pending and ready instances in a pool, and deploy replacements
+
+All pool reset requests must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned.
+
+When a pool reset is requested a 201 status will be returned.
+
+A pool reset will cause vmpooler manager to log that it has cleared ready and pending instances.
+
+For poolreset to be available it is necessary to enable experimental features. Additionally, the request must be performed with an authentication token when authentication is configured.
+
+Responses:
+* 201 - Pool reset requested received
+* 400 - An invalid configuration was provided causing requested changes to fail
+* 404 - An unknown error occurred
+* 405 - The endpoint is disabled because experimental features are disabled
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"1"}' --url https://vmpooler.example.com/api/v1/poolreset
+```
+```json
+{
+  "ok": true
+}
+```

lib/vmpooler/api/v1.rb

@@ -200,6 +200,17 @@ module Vmpooler
       result
     end
 
+    def reset_pool(payload)
+      result = { 'ok' => false }
+
+      payload.each do |poolname, count|
+        backend.sadd('vmpooler__poolreset', poolname)
+      end
+      status 201
+      result['ok'] = true
+      result
+    end
+
     def update_clone_target(payload)
       result = { 'ok' => false }
 
@@ -1063,6 +1074,44 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/poolreset/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      if config['experimental_features']
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        begin
+          payload = JSON.parse(request.body.read)
+          if payload
+            invalid = invalid_templates(payload)
+            if invalid.empty?
+              result = reset_pool(payload)
+            else
+              invalid.each do |bad_pool|
+                metrics.increment("poolreset.invalid.#{bad_pool}")
+              end
+              result[:bad_pools] = invalid
+              status 400
+            end
+          else
+            metrics.increment('poolreset.invalid.unknown')
+            status 404
+          end
+        rescue JSON::ParserError
+          status 400
+          result = {
+            'ok' => false,
+            'message' => 'JSON payload could not be parsed'
+          }
+        end
+      else
+        status 405
+      end
+
+      JSON.pretty_generate(result)
+    end
+
     post "#{api_prefix}/config/clonetarget/?" do
       content_type :json
       result = { 'ok' => false }

lib/vmpooler/pool_manager.rb

@@ -680,6 +680,10 @@ module Vmpooler
     #   - Fires when a template configuration update is requested
     #   - Additional options
     #       :poolname
+    # :pool_reset
+    #   - Fires when a pool reset is requested
+    #   - Additional options
+    #       :poolname
     #
     def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {})
       exit_by = Time.now + loop_delay
@@ -726,6 +730,10 @@ module Vmpooler
             end
           end
 
+          if options[:pool_reset]
+            break if $redis.sismember('vmpooler__poolreset', options[:poolname])
+          end
+
         end
 
         break if time_passed?(:exit_by, exit_by)
@@ -763,7 +771,7 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true)
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
 
             unless maxloop.zero?
               break if loop_count >= maxloop
@@ -917,6 +925,17 @@ module Vmpooler
       end
     end
 
+    def reset_pool(pool)
+      poolname = pool['name']
+      return unless $redis.sismember('vmpooler__poolreset', poolname)
+      $redis.srem('vmpooler__poolreset', poolname)
+      mutex = pool_mutex(poolname)
+      mutex.synchronize do
+        drain_pool(poolname)
+        $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
+      end
+    end
+
     def create_inventory(pool, provider, pool_check_response)
       inventory = {}
       begin
@@ -1125,6 +1144,9 @@ module Vmpooler
       # Remove VMs in excess of the configured pool size
       remove_excess_vms(pool)
 
+      # Reset a pool when poolreset is requested from the API
+      reset_pool(pool)
+
       pool_check_response
     end
 

spec/integration/api/v1/poolreset.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+require 'rack/test'
+
+describe Vmpooler::API::V1 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  let(:config) {
+    {
+      config: {
+        'site_name' => 'test pooler',
+        'vm_lifetime_auth' => 2,
+        'experimental_features' => true
+      },
+      pools: [
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
+        {'name' => 'pool2', 'size' => 10}
+      ],
+      statsd: { 'prefix' => 'stats_prefix'},
+      alias: { 'poolone' => 'pool1' },
+      pool_names: [ 'pool1', 'pool2', 'poolone' ]
+    }
+  }
+
+  describe '/poolreset' do
+    let(:prefix) { '/api/v1' }
+    let(:metrics) { Vmpooler::DummyStatsd.new }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      app.settings.set :config, config
+      app.settings.set :redis, redis
+      app.settings.set :metrics, metrics
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /poolreset' do
+      it 'refreshes ready and pending instances from a pool' do
+        post "#{prefix}/poolreset", '{"pool1":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/poolreset", '{"poolpoolpool":"1"}'
+        expect_json(ok = false, http = 400)
+      end
+
+      it 'resets multiple pools' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool2":"1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/poolreset", '{"pool1":"1","pool3":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = {
+          ok: false,
+          bad_pools: ['pool3']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/poolreset", '{"pool1":"1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+
+      it 'should return 400 for invalid json' do
+        post "#{prefix}/poolreset", '{"pool1":"1}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 400 with a bad pool name' do
+        post "#{prefix}/poolreset", '{"pool11":"1"}'
+        expect_json(ok = false, http = 400)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'should return 404 when there is no payload' do
+        post "#{prefix}/poolreset"
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+  end
+end

spec/unit/pool_manager_spec.rb

@@ -2874,6 +2874,28 @@ EOT
         end
       end
     end
+
+    describe 'with the pool_reset wakeup option' do
+      let(:wakeup_option) {{
+        :pool_reset => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a pool reset is requested' do
+        before(:each) do
+          redis.sadd('vmpooler__poolreset', pool)
+        end
+
+        it 'should sleep until the reset request is detected' do
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:sismember).with('vmpooler__poolreset', pool).and_return(false,false,true)
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
   end
 
   describe "#check_pool" do