(POOLER-143) Add clone_target config change to API

This allows the user to change the cluster in which the targeted pool
will clone to. Upon configuration change, the thread will wake up and
execute the change within 1 second.

lib/vmpooler/api/v1.rb

@@ -194,6 +194,26 @@ module Vmpooler
       result
     end
 
+    def update_clone_target(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+      sync_clone_targets
+
+      payload.each do |poolname, clone_target|
+        unless pools[pool_index[poolname]]['clone_target'] == clone_target
+          pools[pool_index[poolname]]['clone_target'] == clone_target
+          backend.hset('vmpooler__config__clone_target', poolname, clone_target)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
     def sync_pool_templates
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
@@ -222,6 +242,20 @@ module Vmpooler
       end
     end
 
+    def sync_clone_targets
+      pool_index = pool_index(pools)
+      clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
+      unless clone_target_configs.nil?
+        clone_target_configs.each do |poolname, clone_target|
+          if pool_index.include? poolname
+            unless pools[pool_index[poolname]]['clone_target'] == clone_target
+              pools[pool_index[poolname]]['clone_target'] == clone_target
+              end
+          end
+        end
+      end
+    end
+
     get '/' do
       sync_pool_sizes
       redirect to('/dashboard/')
@@ -700,6 +734,14 @@ module Vmpooler
       invalid
     end
 
+    def invalid_pool(payload)
+      invalid = []
+      payload.each do |pool, clone_target|
+        invalid << pool unless pool_exists?(pool)
+      end
+      invalid
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -1011,6 +1053,37 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/config/clonetarget/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      if config['experimental_features']
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_pool(payload)
+          if invalid.empty?
+            result = update_clone_target(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("config.invalid.#{bad_template}")
+            end
+            result[:bad_templates] = invalid
+            status 400
+          end
+        else
+          metrics.increment('config.invalid.unknown')
+          status 404
+        end
+      else
+        status 405
+      end
+
+      JSON.pretty_generate(result)
+    end
+
     get "#{api_prefix}/config/?" do
       content_type :json
       result = { 'ok' => false }

lib/vmpooler/pool_manager.rb

@@ -680,6 +680,10 @@ module Vmpooler
         initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
       end
 
+      if options[:clone_target_change]
+        initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target])
+      end
+
       if options[:pool_template_change]
         initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
       end
@@ -698,6 +702,13 @@ module Vmpooler
             break unless ready_size == initial_ready_size
           end
 
+          if options[:clone_target_change]
+            clone_target = $redis.hget("vmpooler__config__clone_target}", options[:poolname])
+            if clone_target
+              break unless clone_target == initial_clone_target
+            end
+          end
+
           if options[:pool_template_change]
             configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
             if configured_template
@@ -742,7 +753,7 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true)
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true)
 
             unless maxloop.zero?
               break if loop_count >= maxloop
@@ -847,6 +858,21 @@ module Vmpooler
       $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
+    def update_clone_target(pool)
+      mutex = pool_mutex(pool['name'])
+      return if mutex.locked?
+      clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
+      return if clone_target.nil?
+      return if clone_target == pool['clone_target']
+      $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
+      mutex.synchronize do
+        pool['clone_target'] = clone_target
+        # Remove all ready and pending VMs so new instances are created for the new clone_target
+        drain_pool(pool['name'])
+      end
+      $logger.log('s', "[*] [#{pool['name']}] is ready for use")
+    end
+
     def remove_excess_vms(pool)
       ready = $redis.scard("vmpooler__ready__#{pool['name']}")
       total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
@@ -1080,6 +1106,10 @@ module Vmpooler
       # otherwise identify this change when running
       update_pool_size(pool)
 
+      # Check to see if a pool size change has been made via the configuration API
+      # Additionally, a pool will drain ready and pending instances
+      update_clone_target(pool)
+
       repopulate_pool_vms(pool['name'], provider, pool_check_response, pool['size'])
 
       # Remove VMs in excess of the configured pool size