(POOLER-107) Add configuration API endpoint

This commit adds a configuration endpoint to the vmpooler API. Pool
size, and pool template, can be adjusted for pools that are configured
at vmpooler application start time. Pool template changes trigger a pool
refresh, and the new template has delta disks created automatically by
vmpooler.

Additionally, the capability to create template delta disks is added to
the vsphere provider, and this is implemented to ensure that templates
have delta disks created at application start time.

The mechanism used to find template VM objects is simplified to make the flow of logic easier to understand. As an additional benefit, performance of this lookup is improved by using FindByInventoryPath.

lib/vmpooler/api/helpers.rb

@@ -378,6 +378,16 @@ module Vmpooler
         result
       end
 
+      def pool_index(pools)
+        pools_hash = {}
+        index = 0
+        for pool in pools
+          pools_hash[pool['name']] = index
+          index += 1
+        end
+        pools_hash
+      end
+
     end
   end
 end

lib/vmpooler/api/v1.rb

@@ -120,6 +120,44 @@ module Vmpooler
       result
     end
 
+    def update_pool_size(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+
+      payload.each do |poolname, size|
+        unless pools[pool_index[poolname]]['size'] == size.to_i
+          pools[pool_index[poolname]]['size'] = size.to_i
+          backend.hset('vmpooler__config__poolsize', poolname, size)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
+    def update_pool_template(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+
+      payload.each do |poolname, template|
+        unless pools[pool_index[poolname]]['template'] == template
+          pools[pool_index[poolname]]['template'] = template
+          backend.hset('vmpooler__config__template', poolname, template)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
     # Provide run-time statistics
     #
     # Example:
@@ -502,6 +540,24 @@ module Vmpooler
       invalid
     end
 
+    def invalid_template_or_size(payload)
+      invalid = []
+      payload.each do |pool, size|
+        invalid << pool unless pool_exists?(pool)
+        Integer(size) rescue invalid << pool
+      end
+      invalid
+    end
+
+    def invalid_template_or_path(payload)
+      invalid = []
+      payload.each do |pool, template|
+        invalid << pool unless pool_exists?(pool)
+        invalid << pool unless template.include? '/'
+      end
+      invalid
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -747,6 +803,58 @@ module Vmpooler
 
       JSON.pretty_generate(result)
     end
+
+    post "#{api_prefix}/config/poolsize/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = JSON.parse(request.body.read)
+
+      if payload
+        invalid = invalid_template_or_size(payload)
+        if invalid.empty?
+          result = update_pool_size(payload)
+        else
+          invalid.each do |bad_template|
+            metrics.increment("config.invalid.#{bad_template}")
+          end
+          status 404
+        end
+      else
+        metrics.increment('config.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/config/pooltemplate/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = JSON.parse(request.body.read)
+
+      if payload
+        invalid = invalid_template_or_path(payload)
+        if invalid.empty?
+          result = update_pool_template(payload)
+        else
+          invalid.each do |bad_template|
+            metrics.increment("config.invalid.#{bad_template}")
+          end
+          status 404
+        end
+      else
+        metrics.increment('config.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
   end
   end
 end

lib/vmpooler/pool_manager.rb

@@ -187,9 +187,9 @@ module Vmpooler
       end
     end
 
-    def move_vm_queue(pool, vm, queue_from, queue_to, msg)
+    def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
       $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
-      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}")
+      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
     end
 
     # Clone a VM
@@ -555,6 +555,75 @@ module Vmpooler
       end
     end
 
+    def prepare_template(pool, provider)
+      if $config[:config]['create_template_delta_disks']
+        # Ensure templates are evaluated for delta disk creation on startup
+        return if $redis.hget('vmpooler__config__updating', pool['name'])
+        unless $redis.hget('vmpooler__template__prepared', pool['name'])
+          begin
+            $redis.hset('vmpooler__config__updating', pool['name'], 1)
+            provider.create_template_delta_disks(pool)
+            $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
+          ensure
+            $redis.hdel('vmpooler__config__updating', pool['name'])
+          end
+        end
+      end
+    end
+
+    def update_pool_template(pool, provider)
+      $redis.hset('vmpooler__template', pool['name'], pool['template']) unless $redis.hget('vmpooler__template', pool['name'])
+      return unless $redis.hget('vmpooler__config__template', pool['name'])
+      unless $redis.hget('vmpooler__config__template', pool['name']) == $redis.hget('vmpooler__template', pool['name'])
+        # Ensure we are only updating a template once
+        return if $redis.hget('vmpooler__config__updating', pool['name'])
+        begin
+          $redis.hset('vmpooler__config__updating', pool['name'], 1)
+
+          old_template_name = $redis.hget('vmpooler__template', pool['name'])
+          new_template_name = $redis.hget('vmpooler__config__template', pool['name'])
+          pool['template'] = new_template_name
+          $redis.hset('vmpooler__template', pool['name'], new_template_name)
+          $logger.log('s', "[*] [#{pool['name']}] template updated from #{old_template_name} to #{new_template_name}")
+          # Remove all ready and pending VMs so new instances are created from the new template
+          if $redis.scard("vmpooler__ready__#{pool['name']}") > 0
+            $logger.log('s', "[*] [#{pool['name']}] removing ready and pending instances")
+            $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
+              $redis.smove("vmpooler__ready__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+            end
+          end
+          if $redis.scard("vmpooler__pending__#{pool['name']}") > 0
+            $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+              $redis.smove("vmpooler__pending__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+            end
+          end
+          # Prepare template for deployment
+          $logger.log('s', "[*] [#{pool['name']}] creating template deltas")
+          provider.create_template_delta_disks(pool)
+          $logger.log('s', "[*] [#{pool['name']}] template deltas have been created")
+        ensure
+          $redis.hdel('vmpooler__config__updating', pool['name'])
+        end
+      end
+    end
+
+    def remove_excess_vms(pool, provider, ready, total)
+      unless ready.nil?
+        if total > pool['size']
+          difference = ready - pool['size']
+          difference.times do
+            next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
+            move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
+          end
+          if total > ready
+            $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+              move_vm_queue(pool['name'], vm, 'pending', 'completed')
+            end
+          end
+        end
+      end
+    end
+
     def _check_pool(pool, provider)
       pool_check_response = {
         discovered_vms: 0,
@@ -683,36 +752,59 @@ module Vmpooler
         end
       end
 
+      # Create template delta disks
+      prepare_template(pool, provider)
+
+      # UPDATE TEMPLATE
+      # Check to see if a pool template change has been made via the configuration API
+      # Since check_pool runs in a loop it does not otherwise identify this change when running
+      update_pool_template(pool, provider)
+
       # REPOPULATE
-      ready = $redis.scard("vmpooler__ready__#{pool['name']}")
-      total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
+      # Do not attempt to repopulate a pool while a template is updating
+      unless $redis.hget('vmpooler__config__updating', pool['name'])
+        ready = $redis.scard("vmpooler__ready__#{pool['name']}")
+        total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
 
-      $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
-      $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
+        $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
+        $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
 
-      if $redis.get("vmpooler__empty__#{pool['name']}")
-        $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
-      elsif ready.zero?
-        $redis.set("vmpooler__empty__#{pool['name']}", 'true')
-        $logger.log('s', "[!] [#{pool['name']}] is empty")
-      end
+        if $redis.get("vmpooler__empty__#{pool['name']}")
+          $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
+        elsif ready.zero?
+          $redis.set("vmpooler__empty__#{pool['name']}", 'true')
+          $logger.log('s', "[!] [#{pool['name']}] is empty")
+        end
 
-      if total < pool['size']
-        (1..(pool['size'] - total)).each do |_i|
-          if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
-            begin
-              $redis.incr('vmpooler__tasks__clone')
-              pool_check_response[:cloned_vms] += 1
-              clone_vm(pool, provider)
-            rescue => err
-              $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
-              $redis.decr('vmpooler__tasks__clone')
-              raise
+        # Check to see if a pool size change has been made via the configuration API
+        # Since check_pool runs in a loop it does not
+        # otherwise identify this change when running
+        if $redis.hget('vmpooler__config__poolsize', pool['name'])
+          unless $redis.hget('vmpooler__config__poolsize', pool['name']).to_i == pool['size']
+            pool['size'] = Integer($redis.hget('vmpooler__config__poolsize', pool['name']))
+          end
+        end
+
+        if total < pool['size']
+          (1..(pool['size'] - total)).each do |_i|
+            if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
+              begin
+                $redis.incr('vmpooler__tasks__clone')
+                pool_check_response[:cloned_vms] += 1
+                clone_vm(pool, provider)
+              rescue => err
+                $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
+                $redis.decr('vmpooler__tasks__clone')
+                raise
+              end
             end
           end
         end
       end
 
+      # Remove VMs in excess of the configured pool size
+      remove_excess_vms(pool, provider, ready, total)
+
       pool_check_response
     end
 
@@ -739,6 +831,10 @@ module Vmpooler
       $redis.set('vmpooler__tasks__clone', 0)
       # Clear out vmpooler__migrations since stale entries may be left after a restart
       $redis.del('vmpooler__migration')
+      # Clear out any configuration changes in flight that were interrupted
+      $redis.del('vmpooler__config__updating')
+      # Ensure template deltas are created on each startup
+      $redis.del('vmpooler__template__prepared')
 
       # Copy vSphere settings to correct location.  This happens with older configuration files
       if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)

lib/vmpooler/providers/base.rb

@@ -217,6 +217,14 @@ module Vmpooler
         def vm_exists?(pool_name, vm_name)
           !get_vm(pool_name, vm_name).nil?
         end
+
+        # inputs
+        #   [Hash] pool : Configuration for the pool
+        # returns
+        #   nil when successful. Raises error when encountered
+        def create_template_delta_disks(pool)
+          raise("#{self.class.name} does not implement create_template_delta_disks")
+        end
       end
     end
   end

lib/vmpooler/providers/vsphere.rb

@@ -197,17 +197,10 @@ module Vmpooler
             target_cluster_name = get_target_cluster_from_config(pool_name)
             target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
-            # Extract the template VM name from the full path
+            # Get the template VM object
             raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless template_path =~ /\//
-            templatefolders = template_path.split('/')
-            template_name = templatefolders.pop
 
-            # Get the actual objects from vSphere
-            template_folder_object = find_folder(templatefolders.join('/'), connection, target_datacenter_name)
-            raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil?
-
-            template_vm_object = template_folder_object.find(template_name)
-            raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil?
+            template_vm_object = find_template_vm(pool, connection)
 
             # Annotate with creation time, origin template, etc.
             # Add extraconfig options that can be queried by vmtools
@@ -964,6 +957,30 @@ module Vmpooler
           raise("Cannot create folder #{new_folder}") if folder_object.nil?
           folder_object
         end
+
+        def find_template_vm(pool, connection)
+          datacenter = get_target_datacenter_from_config(pool['name'])
+          raise('cannot find datacenter') if datacenter.nil?
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => "#{datacenter}/vm/#{pool['template']}"
+          }
+
+          template_vm_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+          raise("Pool #{pool['name']} specifies a template VM of #{pool['template']} which does not exist for the provider #{name}") if template_vm_object.nil?
+
+          template_vm_object
+        end
+
+        def create_template_delta_disks(pool)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+            template_vm_object = find_template_vm(pool, connection)
+
+            template_vm_object.add_delta_disk_layer_on_all_disks
+          end
+        end
       end
     end
   end