(POOLER-107) Add configuration API endpoint

This commit adds a configuration endpoint to the vmpooler API. Pool
size, and pool template, can be adjusted for pools that are configured
at vmpooler application start time. Pool template changes trigger a pool
refresh, and the new template has delta disks created automatically by
vmpooler.

Additionally, the capability to create template delta disks is added to
the vsphere provider, and this is implemented to ensure that templates
have delta disks created at application start time.

The mechanism used to find template VM objects is simplified to make the flow of logic easier to understand. As an additional benefit, performance of this lookup is improved by using FindByInventoryPath.

A table of contents is added to API.md to ease navigation. Without this change API.md has no table of contents and is difficult to navigate.

Add mutex object for managing pool configuration updates

This commit adds a mutex object for ensuring that pool configuration changes are synchronized across multiple running threads, removing the possibility of two threads attempting to update something at once, without relying on redis data. Without this change this is managed crudely by specifying in redis that a configuration update is taking place. This redis data is left so the REPOPULATE section of _check_pool can still identify when a configuration change is in progress, and prevent a pool from repopulating at that time.

Add wake up event for pool template changes

This commit adds a wake up event to detect pool template changes.
Additionally, GET /config has a template_ready section added to the
output for each pool, which makes clear when a pool is ready to populate
itself.

lib/vmpooler/api/helpers.rb

@@ -378,6 +378,29 @@ module Vmpooler
         result
       end
 
+      def pool_index(pools)
+        pools_hash = {}
+        index = 0
+        for pool in pools
+          pools_hash[pool['name']] = index
+          index += 1
+        end
+        pools_hash
+      end
+
+      def template_ready?(pool, backend)
+        prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
+        return false if prepared_template.nil?
+        return true if pool['template'] == prepared_template
+        return false
+      end
+
+      def is_integer?(x)
+        Integer(x)
+        true
+      rescue
+        false
+      end
     end
   end
 end

lib/vmpooler/api/v1.rb

@@ -120,6 +120,74 @@ module Vmpooler
       result
     end
 
+    def update_pool_size(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+      sync_pool_sizes
+
+      payload.each do |poolname, size|
+        unless pools[pool_index[poolname]]['size'] == size.to_i
+          pools[pool_index[poolname]]['size'] = size.to_i
+          backend.hset('vmpooler__config__poolsize', poolname, size)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
+    def update_pool_template(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+      sync_pool_templates
+
+      payload.each do |poolname, template|
+        unless pools[pool_index[poolname]]['template'] == template
+          pools[pool_index[poolname]]['template'] = template
+          backend.hset('vmpooler__config__template', poolname, template)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
+    def sync_pool_templates
+      pool_index = pool_index(pools)
+      template_configs = backend.hgetall('vmpooler__config__template')
+      unless template_configs.nil?
+        template_configs.each do |poolname, template|
+          if pool_index.include? poolname
+            unless pools[pool_index[poolname]]['template'] == template
+              pools[pool_index[poolname]]['template'] = template
+            end
+          end
+        end
+      end
+    end
+
+    def sync_pool_sizes
+      pool_index = pool_index(pools)
+      poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
+      unless poolsize_configs.nil?
+        poolsize_configs.each do |poolname, size|
+          if pool_index.include? poolname
+            unless pools[pool_index[poolname]]['size'] == size.to_i
+              pools[pool_index[poolname]]['size'] == size.to_i
+            end
+          end
+        end
+      end
+    end
+
     # Provide run-time statistics
     #
     # Example:
@@ -196,6 +264,8 @@ module Vmpooler
         }
       }
 
+      sync_pool_sizes
+
       result[:capacity] = get_capacity_metrics(pools, backend) unless views and not views.include?("capacity")
       result[:queue] = get_queue_metrics(pools, backend) unless views and not views.include?("queue")
       result[:clone] = get_task_metrics(backend, 'clone', Date.today.to_s) unless views and not views.include?("clone")
@@ -502,6 +572,30 @@ module Vmpooler
       invalid
     end
 
+    def invalid_template_or_size(payload)
+      invalid = []
+      payload.each do |pool, size|
+        invalid << pool unless pool_exists?(pool)
+        unless is_integer?(size)
+          invalid << pool
+          next
+        end
+        invalid << pool unless Integer(size) >= 0
+      end
+      invalid
+    end
+
+    def invalid_template_or_path(payload)
+      invalid = []
+      payload.each do |pool, template|
+        invalid << pool unless pool_exists?(pool)
+        invalid << pool unless template.include? '/'
+        invalid << pool if template[0] == '/'
+        invalid << pool if template[-1] == '/'
+      end
+      invalid
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -747,6 +841,95 @@ module Vmpooler
 
       JSON.pretty_generate(result)
     end
+
+    post "#{api_prefix}/config/poolsize/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      if config['experimental_features']
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_template_or_size(payload)
+          if invalid.empty?
+            result = update_pool_size(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("config.invalid.#{bad_template}")
+            end
+            result[:bad_templates] = invalid
+            status 400
+          end
+        else
+          metrics.increment('config.invalid.unknown')
+          status 404
+        end
+      else
+        status 405
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/config/pooltemplate/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      if config['experimental_features']
+        need_token! if Vmpooler::API.settings.config[:auth]
+
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_template_or_path(payload)
+          if invalid.empty?
+            result = update_pool_template(payload)
+          else
+            invalid.each do |bad_template|
+              metrics.increment("config.invalid.#{bad_template}")
+            end
+            result[:bad_templates] = invalid
+            status 400
+          end
+        else
+          metrics.increment('config.invalid.unknown')
+          status 404
+        end
+      else
+        status 405
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    get "#{api_prefix}/config/?" do
+      content_type :json
+      result = { 'ok' => false }
+      status 404
+
+      if pools
+        sync_pool_sizes
+        sync_pool_templates
+
+        pool_configuration = []
+        pools.each do |pool|
+          pool['template_ready'] = template_ready?(pool, backend)
+          pool_configuration << pool
+        end
+
+        result = {
+          pool_configuration: pool_configuration,
+          status: {
+            ok: true
+          }
+        }
+
+        status 200
+      end
+      JSON.pretty_generate(result)
+    end
   end
   end
 end

lib/vmpooler/pool_manager.rb

@@ -21,6 +21,9 @@ module Vmpooler
 
       # Our thread-tracker object
       $threads = {}
+
+      # Pool mutex
+      @reconfigure_pool = {}
     end
 
     def config
@@ -187,9 +190,9 @@ module Vmpooler
       end
     end
 
-    def move_vm_queue(pool, vm, queue_from, queue_to, msg)
+    def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
       $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
-      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}")
+      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
     end
 
     # Clone a VM
@@ -482,6 +485,10 @@ module Vmpooler
     #   - Fires when the number of ready VMs changes due to being consumed.
     #   - Additional options
     #       :poolname
+    # :pool_template_change
+    #   - Fires when a template configuration update is requested
+    #   - Additional options
+    #       :poolname
     #
     def sleep_with_wakeup_events(loop_delay, wakeup_period = 5, options = {})
       exit_by = Time.now + loop_delay
@@ -492,6 +499,10 @@ module Vmpooler
         initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
       end
 
+      if options[:pool_template_change]
+        initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
+      end
+
       loop do
         sleep(1)
         break if time_passed?(:exit_by, exit_by)
@@ -505,6 +516,14 @@ module Vmpooler
             ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
             break unless ready_size == initial_ready_size
           end
+
+          if options[:pool_template_change]
+            configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
+            if configured_template
+              break unless initial_template == configured_template
+            end
+          end
+
         end
 
         break if time_passed?(:exit_by, exit_by)
@@ -532,6 +551,7 @@ module Vmpooler
           loop_delay = loop_delay_min
           provider = get_provider_for_pool(pool['name'])
           raise("Could not find provider '#{pool['provider']}") if provider.nil?
+          sync_pool_template(pool)
           loop do
             result = _check_pool(pool, provider)
 
@@ -541,7 +561,7 @@ module Vmpooler
               loop_delay = (loop_delay * loop_delay_decay).to_i
               loop_delay = loop_delay_max if loop_delay > loop_delay_max
             end
-            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'])
+            sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true)
 
             unless maxloop.zero?
               break if loop_count >= maxloop
@@ -555,6 +575,101 @@ module Vmpooler
       end
     end
 
+    def pool_mutex(poolname)
+      @reconfigure_pool[poolname] || @reconfigure_pool[poolname] = Mutex.new
+    end
+
+    def sync_pool_template(pool)
+      pool_template = $redis.hget('vmpooler__config__template', pool['name'])
+      if pool_template
+        unless pool['template'] == pool_template
+          pool['template'] = pool_template
+        end
+      end
+    end
+
+    def prepare_template(pool, provider)
+      provider.create_template_delta_disks(pool) if $config[:config]['create_template_delta_disks']
+      $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
+    end
+
+    def evaluate_template(pool, provider)
+      mutex = pool_mutex(pool['name'])
+      prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
+      configured_template = $redis.hget('vmpooler__config__template', pool['name'])
+      return if mutex.locked?
+      if prepared_template.nil?
+        mutex.synchronize do
+          prepare_template(pool, provider)
+          prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
+        end
+      end
+      return if configured_template.nil?
+      return if configured_template == prepared_template
+      mutex.synchronize do
+        update_pool_template(pool, provider, configured_template, prepared_template)
+      end
+    end
+
+    def drain_pool(poolname)
+      # Clear a pool of ready and pending instances
+      if $redis.scard("vmpooler__ready__#{poolname}") > 0
+        $logger.log('s', "[*] [#{poolname}] removing ready instances")
+        $redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
+          move_vm_queue(poolname, vm, 'ready', 'completed')
+        end
+      end
+      if $redis.scard("vmpooler__pending__#{poolname}") > 0
+        $logger.log('s', "[*] [#{poolname}] removing pending instances")
+        $redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
+          move_vm_queue(poolname, vm, 'pending', 'completed')
+        end
+      end
+    end
+
+    def update_pool_template(pool, provider, configured_template, prepared_template)
+      pool['template'] = configured_template
+      $logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
+      # Remove all ready and pending VMs so new instances are created from the new template
+      drain_pool(pool['name'])
+      # Prepare template for deployment
+      $logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
+      prepare_template(pool, provider)
+      $logger.log('s', "[*] [#{pool['name']}] is ready for use")
+    end
+
+    def remove_excess_vms(pool, provider, ready, total)
+      return if total.nil?
+      return if total == 0
+      mutex = pool_mutex(pool['name'])
+      return if mutex.locked?
+      return unless ready > pool['size']
+      mutex.synchronize do
+        difference = ready - pool['size']
+        difference.times do
+          next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
+          move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
+        end
+        if total > ready
+          $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+            move_vm_queue(pool['name'], vm, 'pending', 'completed')
+          end
+        end
+      end
+    end
+
+    def update_pool_size(pool)
+      mutex = pool_mutex(pool['name'])
+      return if mutex.locked?
+      poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
+      return if poolsize.nil?
+      poolsize = Integer(poolsize)
+      return if poolsize == pool['size']
+      mutex.synchronize do
+        pool['size'] = poolsize
+      end
+    end
+
     def _check_pool(pool, provider)
       pool_check_response = {
         discovered_vms: 0,
@@ -683,36 +798,53 @@ module Vmpooler
         end
       end
 
+      # UPDATE TEMPLATE
+      # Evaluates a pool template to ensure templates are prepared adequately for the configured provider
+      # If a pool template configuration change is detected then template preparation is repeated for the new template
+      # Additionally, a pool will drain ready and pending instances
+      evaluate_template(pool, provider)
+
       # REPOPULATE
-      ready = $redis.scard("vmpooler__ready__#{pool['name']}")
-      total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
+      # Do not attempt to repopulate a pool while a template is updating
+      unless pool_mutex(pool['name']).locked?
+        ready = $redis.scard("vmpooler__ready__#{pool['name']}")
+        total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
 
-      $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
-      $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
+        $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
+        $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
 
-      if $redis.get("vmpooler__empty__#{pool['name']}")
-        $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
-      elsif ready.zero?
-        $redis.set("vmpooler__empty__#{pool['name']}", 'true')
-        $logger.log('s', "[!] [#{pool['name']}] is empty")
-      end
+        if $redis.get("vmpooler__empty__#{pool['name']}")
+          $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
+        elsif ready.zero?
+          $redis.set("vmpooler__empty__#{pool['name']}", 'true')
+          $logger.log('s', "[!] [#{pool['name']}] is empty")
+        end
 
-      if total < pool['size']
-        (1..(pool['size'] - total)).each do |_i|
-          if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
-            begin
-              $redis.incr('vmpooler__tasks__clone')
-              pool_check_response[:cloned_vms] += 1
-              clone_vm(pool, provider)
-            rescue => err
-              $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
-              $redis.decr('vmpooler__tasks__clone')
-              raise
+        # Check to see if a pool size change has been made via the configuration API
+        # Since check_pool runs in a loop it does not
+        # otherwise identify this change when running
+        update_pool_size(pool)
+
+        if total < pool['size']
+          (1..(pool['size'] - total)).each do |_i|
+            if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
+              begin
+                $redis.incr('vmpooler__tasks__clone')
+                pool_check_response[:cloned_vms] += 1
+                clone_vm(pool, provider)
+              rescue => err
+                $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
+                $redis.decr('vmpooler__tasks__clone')
+                raise
+              end
             end
           end
         end
       end
 
+      # Remove VMs in excess of the configured pool size
+      remove_excess_vms(pool, provider, ready, total)
+
       pool_check_response
     end
 
@@ -739,6 +871,8 @@ module Vmpooler
       $redis.set('vmpooler__tasks__clone', 0)
       # Clear out vmpooler__migrations since stale entries may be left after a restart
       $redis.del('vmpooler__migration')
+      # Ensure template deltas are created on each startup
+      $redis.del('vmpooler__template__prepared')
 
       # Copy vSphere settings to correct location.  This happens with older configuration files
       if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)

lib/vmpooler/providers/base.rb

@@ -217,6 +217,14 @@ module Vmpooler
         def vm_exists?(pool_name, vm_name)
           !get_vm(pool_name, vm_name).nil?
         end
+
+        # inputs
+        #   [Hash] pool : Configuration for the pool
+        # returns
+        #   nil when successful. Raises error when encountered
+        def create_template_delta_disks(pool)
+          raise("#{self.class.name} does not implement create_template_delta_disks")
+        end
       end
     end
   end

lib/vmpooler/providers/vsphere.rb

@@ -197,17 +197,10 @@ module Vmpooler
             target_cluster_name = get_target_cluster_from_config(pool_name)
             target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
-            # Extract the template VM name from the full path
-            raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless template_path =~ /\//
-            templatefolders = template_path.split('/')
-            template_name = templatefolders.pop
+            # Get the template VM object
+            raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless valid_template_path? template_path
 
-            # Get the actual objects from vSphere
-            template_folder_object = find_folder(templatefolders.join('/'), connection, target_datacenter_name)
-            raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil?
-
-            template_vm_object = template_folder_object.find(template_name)
-            raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil?
+            template_vm_object = find_template_vm(pool, connection)
 
             # Annotate with creation time, origin template, etc.
             # Add extraconfig options that can be queried by vmtools
@@ -933,6 +926,37 @@ module Vmpooler
           raise("Cannot create folder #{new_folder}") if folder_object.nil?
           folder_object
         end
+
+        def find_template_vm(pool, connection)
+          datacenter = get_target_datacenter_from_config(pool['name'])
+          raise('cannot find datacenter') if datacenter.nil?
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => "#{datacenter}/vm/#{pool['template']}"
+          }
+
+          template_vm_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+          raise("Pool #{pool['name']} specifies a template VM of #{pool['template']} which does not exist for the provider #{name}") if template_vm_object.nil?
+
+          template_vm_object
+        end
+
+        def create_template_delta_disks(pool)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+            template_vm_object = find_template_vm(pool, connection)
+
+            template_vm_object.add_delta_disk_layer_on_all_disks
+          end
+        end
+
+        def valid_template_path?(template)
+          return false unless template.include?('/')
+          return false if template[0] == '/'
+          return false if template[-1] == '/'
+          return true
+        end
       end
     end
   end