(POOLER-143) Add clone_target config change to API

This allows the user to change the cluster in which the targeted pool
will clone to. Upon configuration change, the thread will wake up and
execute the change within 1 second.

lib/vmpooler/api/v1.rb

@@ -194,6 +194,26 @@ module Vmpooler
       result
     end
 
+    def update_clone_target(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+      sync_clone_targets
+
+      payload.each do |poolname, clone_target|
+        unless pools[pool_index[poolname]]['clone_target'] == clone_target
+          pools[pool_index[poolname]]['clone_target'] == clone_target
+          backend.hset('vmpooler__config__clone_target', poolname, clone_target)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
     def sync_pool_templates
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
@@ -222,6 +242,20 @@ module Vmpooler
       end
     end
 
+    def sync_clone_targets
+      pool_index = pool_index(pools)
+      clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
+      unless clone_target_configs.nil?
+        clone_target_configs.each do |poolname, clone_target|
+          if pool_index.include? poolname
+            unless pools[pool_index[poolname]]['clone_target'] == clone_target
+              pools[pool_index[poolname]]['clone_target'] == clone_target
+              end
+          end
+        end
+      end
+    end
+
     get '/' do
       sync_pool_sizes
       redirect to('/dashboard/')
@@ -700,6 +734,14 @@ module Vmpooler
       invalid
     end
 
+    def invalid_pool(payload)
+      invalid = []
+      payload.each do |pool, clone_target|
+        invalid << pool unless pool_exists?(pool)
+      end
+      invalid
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -1011,6 +1053,37 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/config/clonetarget/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      if config['experimental_features']
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_pool(payload)
+          if invalid.empty?
+            result = update_clone_target(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("config.invalid.#{bad_template}")
+            end
+            result[:bad_templates] = invalid
+            status 400
+          end
+        else
+          metrics.increment('config.invalid.unknown')
+          status 404
+        end
+      else
+        status 405
+      end
+
+      JSON.pretty_generate(result)
+    end
+
     get "#{api_prefix}/config/?" do
       content_type :json
       result = { 'ok' => false }

lib/vmpooler/pool_manager.rb

@@ -680,6 +680,10 @@ module Vmpooler
         initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
       end
 
+      if options[:clone_target_change]
+        initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target])
+      end
+
       if options[:pool_template_change]
         initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
       end
@@ -698,6 +702,13 @@ module Vmpooler
             break unless ready_size == initial_ready_size
           end
 
+          if options[:clone_target_change]
+            clone_target = $redis.hget("vmpooler__config__clone_target}", options[:poolname])
+            if clone_target
+              break unless clone_target == initial_clone_target
+            end
+          end
+
           if options[:pool_template_change]
             configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
             if configured_template
@@ -742,7 +753,7 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true)
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true)
 
             unless maxloop.zero?
               break if loop_count >= maxloop
@@ -847,6 +858,21 @@ module Vmpooler
       $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
+    def update_clone_target(pool)
+      mutex = pool_mutex(pool['name'])
+      return if mutex.locked?
+      clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
+      return if clone_target.nil?
+      return if clone_target == pool['clone_target']
+      $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
+      mutex.synchronize do
+        pool['clone_target'] = clone_target
+        # Remove all ready and pending VMs so new instances are created for the new clone_target
+        drain_pool(pool['name'])
+      end
+      $logger.log('s', "[*] [#{pool['name']}] is ready for use")
+    end
+
     def remove_excess_vms(pool)
       ready = $redis.scard("vmpooler__ready__#{pool['name']}")
       total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
@@ -1080,6 +1106,10 @@ module Vmpooler
       # otherwise identify this change when running
       update_pool_size(pool)
 
+      # Check to see if a pool size change has been made via the configuration API
+      # Additionally, a pool will drain ready and pending instances
+      update_clone_target(pool)
+
       repopulate_pool_vms(pool['name'], provider, pool_check_response, pool['size'])
 
       # Remove VMs in excess of the configured pool size

spec/integration/api/v1/config_spec.rb

@@ -16,7 +16,7 @@ describe Vmpooler::API::V1 do
         'experimental_features' => true
       },
       pools: [
-        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1'},
+        {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1', 'clone_target' => 'default_cluster'},
         {'name' => 'pool2', 'size' => 10}
       ],
       statsd: { 'prefix' => 'stats_prefix'},
@@ -223,6 +223,69 @@ describe Vmpooler::API::V1 do
       end
     end
 
+    describe 'POST /config/clonetarget' do
+      it 'changes the clone target' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'changes a pool size for multiple pools' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"cluster1","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        post "#{prefix}/config/clonetarget", '{"pool10":"cluster1"}'
+        expect_json(ok = false, http = 400)
+        expected = {
+          ok: false,
+          bad_templates: ['pool10']
+        }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 200 when no change is required' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 201 when at least one pool changes' do
+        post "#{prefix}/config/clonetarget", '{"pool1":"default_cluster","pool2":"cluster2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'with experimental features disabled' do
+        before(:each) do
+          config[:config]['experimental_features'] = false
+        end
+
+        it 'should return 405' do
+          post "#{prefix}/config/clonetarget", '{"pool1":"cluster1"}'
+          expect_json(ok = false, http = 405)
+
+          expected = { ok: false }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+      end
+    end
+
     describe 'GET /config' do
       let(:prefix) { '/api/v1' }
 

spec/unit/pool_manager_spec.rb

@@ -2390,6 +2390,59 @@ EOT
     end
   end
 
+  describe 'update_clone_target' do
+    let(:newtarget) { 'cluster2' }
+    let(:config) {
+      YAML.load(<<-EOT
+---
+:pools:
+  - name: #{pool}
+    clone_target: 'cluster1'
+EOT
+      )
+    }
+    let(:poolconfig) { config[:pools][0] }
+
+    context 'with a locked mutex' do
+
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+        expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
+      end
+
+      it 'should return nil' do
+        expect(subject.update_clone_target(poolconfig)).to be_nil
+      end
+    end
+
+    it 'should get the pool clone target configuration from redis' do
+      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool)
+
+      subject.update_clone_target(poolconfig)
+    end
+
+    it 'should return when clone_target is not set in redis' do
+      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(nil)
+
+      expect(subject.update_clone_target(poolconfig)).to be_nil
+    end
+
+    it 'should return when no change in configuration is required' do
+      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return('cluster1')
+
+      expect(subject.update_clone_target(poolconfig)).to be_nil
+    end
+
+    it 'should update the clone target' do
+      expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(newtarget)
+
+      subject.update_clone_target(poolconfig)
+
+      expect(poolconfig['clone_target']).to eq(newtarget)
+    end
+  end
+
   describe "#execute!" do
     let(:config) {
       YAML.load(<<-EOT