From 9758adccfebdac87be3b5ca541c1451930dc589d Mon Sep 17 00:00:00 2001
From: "kirby@puppetlabs.com" <mk.kirby@gmail.com>
Date: Fri, 11 May 2018 13:42:21 -0700
Subject: [PATCH] (POOLER-107) Add configuration API endpoint

This commit adds a configuration endpoint to the vmpooler API. Pool
size, and pool template, can be adjusted for pools that are configured
at vmpooler application start time. Pool template changes trigger a pool
refresh, and the new template has delta disks created automatically by
vmpooler.

Additionally, the capability to create template delta disks is added to
the vsphere provider, and this is implemented to ensure that templates
have delta disks created at application start time.

The mechanism used to find template VM objects is simplified to make the flow of logic easier to understand. As an additional benefit, performance of this lookup is improved by using FindByInventoryPath.
---
 docs/API.md                            |  56 ++++++
 lib/vmpooler/api/helpers.rb            |  10 +
 lib/vmpooler/api/v1.rb                 | 108 ++++++++++
 lib/vmpooler/pool_manager.rb           | 142 ++++++++++---
 lib/vmpooler/providers/base.rb         |   8 +
 lib/vmpooler/providers/vsphere.rb      |  35 +++-
 spec/integration/api/v1/config_spec.rb | 160 +++++++++++++++
 spec/unit/pool_manager_spec.rb         | 264 +++++++++++++++++++++++++
 spec/unit/providers/vsphere_spec.rb    |  64 ++++--
 9 files changed, 803 insertions(+), 44 deletions(-)
 create mode 100644 spec/integration/api/v1/config_spec.rb

diff --git a/docs/API.md b/docs/API.md
index bbab501..7ec0155 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -540,3 +540,59 @@ $ curl -G -d 'from=2015-03-10' -d 'to=2015-03-11' --url vmpooler.company.com/api
   ]
 }
 ```
+
+#### Changing configuration via API
+
+##### POST /config/poolsize
+
+Change pool size without having to restart the service.
+
+All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned
+
+When a pool size is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned.
+
+Pool size configuration changes persist through application restarts, and take precedence over a pool size value configured in the pool configuration provided when the application starts. This persistence is dependent on redis. So, if the redis data is lost then the configuration updates revert to those provided at startup at the next application start.
+
+An authentication token is required in order to change pool configuration when authentication is configured.
+Responses:
+* 200 - No changes required
+* 201 - Changes made on at least one pool with changes requested
+* 404 - An error was encountered while evaluating requested changes
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"2","debian-7-x86_64":"1"}' --url https://vmpooler.company.com/api/v1/config/poolsize
+```
+```json
+{
+  "ok": true
+}
+```
+
+##### POST /config/pooltemplate
+
+Change the template configured for a pool, and replenish the pool with instances built from the new template.
+
+All pool template changes requested must be for pools that exist in the vmpooler configuration running, or a 404 code will be returned
+
+When a pool template is changed due to the configuration posted a 201 status will be returned. When the pool configuration is valid, but will not result in any changes, 200 is returned.
+
+A pool template being updated will cause the following actions, which are logged in vmpooler.log:
+* Destroy all instances for the pool template being updated that are in the ready and pending state
+* Halt repopulating the pool while creating template deltas for the newly configured template
+* Unblock pool population and let the pool replenish with instances based on the newly configured template
+
+Pool template changes persist through application restarts, and take precedence over a pool template configured in the pool configuration provided when the application starts. This persistence is dependent on redis. As a result, if the redis data is lost then the configuration values revert to those provided at startup at the next application start.
+
+An authentication token is required in order to change pool configuration when authentication is configured.
+
+Responses:
+* 200 - No changes required
+* 201 - Changes made on at least one pool with changes requested
+* 404 - An error was encountered while evaluating requested changes
+```
+$ curl -X POST -H "Content-Type: application/json" -d '{"debian-7-i386":"templates/debian-7-i386"}' --url https://vmpooler.company.com/api/v1/config/pooltemplate
+```
+```json
+{
+  "ok": true
+}
+```
diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb
index 0bd6275..20307b5 100644
--- a/lib/vmpooler/api/helpers.rb
+++ b/lib/vmpooler/api/helpers.rb
@@ -378,6 +378,16 @@ module Vmpooler
         result
       end
 
+      def pool_index(pools)
+        pools_hash = {}
+        index = 0
+        for pool in pools
+          pools_hash[pool['name']] = index
+          index += 1
+        end
+        pools_hash
+      end
+
     end
   end
 end
diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb
index 1738558..4b0d25e 100644
--- a/lib/vmpooler/api/v1.rb
+++ b/lib/vmpooler/api/v1.rb
@@ -120,6 +120,44 @@ module Vmpooler
       result
     end
 
+    def update_pool_size(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+
+      payload.each do |poolname, size|
+        unless pools[pool_index[poolname]]['size'] == size.to_i
+          pools[pool_index[poolname]]['size'] = size.to_i
+          backend.hset('vmpooler__config__poolsize', poolname, size)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
+    def update_pool_template(payload)
+      result = { 'ok' => false }
+
+      pool_index = pool_index(pools)
+      pools_updated = 0
+
+      payload.each do |poolname, template|
+        unless pools[pool_index[poolname]]['template'] == template
+          pools[pool_index[poolname]]['template'] = template
+          backend.hset('vmpooler__config__template', poolname, template)
+          pools_updated += 1
+          status 201
+        end
+      end
+      status 200 unless pools_updated > 0
+      result['ok'] = true
+      result
+    end
+
     # Provide run-time statistics
     #
     # Example:
@@ -502,6 +540,24 @@ module Vmpooler
       invalid
     end
 
+    def invalid_template_or_size(payload)
+      invalid = []
+      payload.each do |pool, size|
+        invalid << pool unless pool_exists?(pool)
+        Integer(size) rescue invalid << pool
+      end
+      invalid
+    end
+
+    def invalid_template_or_path(payload)
+      invalid = []
+      payload.each do |pool, template|
+        invalid << pool unless pool_exists?(pool)
+        invalid << pool unless template.include? '/'
+      end
+      invalid
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -747,6 +803,58 @@ module Vmpooler
 
       JSON.pretty_generate(result)
     end
+
+    post "#{api_prefix}/config/poolsize/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = JSON.parse(request.body.read)
+
+      if payload
+        invalid = invalid_template_or_size(payload)
+        if invalid.empty?
+          result = update_pool_size(payload)
+        else
+          invalid.each do |bad_template|
+            metrics.increment("config.invalid.#{bad_template}")
+          end
+          status 404
+        end
+      else
+        metrics.increment('config.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/config/pooltemplate/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = JSON.parse(request.body.read)
+
+      if payload
+        invalid = invalid_template_or_path(payload)
+        if invalid.empty?
+          result = update_pool_template(payload)
+        else
+          invalid.each do |bad_template|
+            metrics.increment("config.invalid.#{bad_template}")
+          end
+          status 404
+        end
+      else
+        metrics.increment('config.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
   end
   end
 end
diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
index d9fe40d..f7872ed 100644
--- a/lib/vmpooler/pool_manager.rb
+++ b/lib/vmpooler/pool_manager.rb
@@ -187,9 +187,9 @@ module Vmpooler
       end
     end
 
-    def move_vm_queue(pool, vm, queue_from, queue_to, msg)
+    def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
       $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
-      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}")
+      $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
     end
 
     # Clone a VM
@@ -555,6 +555,75 @@ module Vmpooler
       end
     end
 
+    def prepare_template(pool, provider)
+      if $config[:config]['create_template_delta_disks']
+        # Ensure templates are evaluated for delta disk creation on startup
+        return if $redis.hget('vmpooler__config__updating', pool['name'])
+        unless $redis.hget('vmpooler__template__prepared', pool['name'])
+          begin
+            $redis.hset('vmpooler__config__updating', pool['name'], 1)
+            provider.create_template_delta_disks(pool)
+            $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
+          ensure
+            $redis.hdel('vmpooler__config__updating', pool['name'])
+          end
+        end
+      end
+    end
+
+    def update_pool_template(pool, provider)
+      $redis.hset('vmpooler__template', pool['name'], pool['template']) unless $redis.hget('vmpooler__template', pool['name'])
+      return unless $redis.hget('vmpooler__config__template', pool['name'])
+      unless $redis.hget('vmpooler__config__template', pool['name']) == $redis.hget('vmpooler__template', pool['name'])
+        # Ensure we are only updating a template once
+        return if $redis.hget('vmpooler__config__updating', pool['name'])
+        begin
+          $redis.hset('vmpooler__config__updating', pool['name'], 1)
+
+          old_template_name = $redis.hget('vmpooler__template', pool['name'])
+          new_template_name = $redis.hget('vmpooler__config__template', pool['name'])
+          pool['template'] = new_template_name
+          $redis.hset('vmpooler__template', pool['name'], new_template_name)
+          $logger.log('s', "[*] [#{pool['name']}] template updated from #{old_template_name} to #{new_template_name}")
+          # Remove all ready and pending VMs so new instances are created from the new template
+          if $redis.scard("vmpooler__ready__#{pool['name']}") > 0
+            $logger.log('s', "[*] [#{pool['name']}] removing ready and pending instances")
+            $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
+              $redis.smove("vmpooler__ready__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+            end
+          end
+          if $redis.scard("vmpooler__pending__#{pool['name']}") > 0
+            $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+              $redis.smove("vmpooler__pending__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+            end
+          end
+          # Prepare template for deployment
+          $logger.log('s', "[*] [#{pool['name']}] creating template deltas")
+          provider.create_template_delta_disks(pool)
+          $logger.log('s', "[*] [#{pool['name']}] template deltas have been created")
+        ensure
+          $redis.hdel('vmpooler__config__updating', pool['name'])
+        end
+      end
+    end
+
+    def remove_excess_vms(pool, provider, ready, total)
+      unless ready.nil?
+        if total > pool['size']
+          difference = ready - pool['size']
+          difference.times do
+            next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
+            move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
+          end
+          if total > ready
+            $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
+              move_vm_queue(pool['name'], vm, 'pending', 'completed')
+            end
+          end
+        end
+      end
+    end
+
     def _check_pool(pool, provider)
       pool_check_response = {
         discovered_vms: 0,
@@ -683,36 +752,59 @@ module Vmpooler
         end
       end
 
+      # Create template delta disks
+      prepare_template(pool, provider)
+
+      # UPDATE TEMPLATE
+      # Check to see if a pool template change has been made via the configuration API
+      # Since check_pool runs in a loop it does not otherwise identify this change when running
+      update_pool_template(pool, provider)
+
       # REPOPULATE
-      ready = $redis.scard("vmpooler__ready__#{pool['name']}")
-      total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
+      # Do not attempt to repopulate a pool while a template is updating
+      unless $redis.hget('vmpooler__config__updating', pool['name'])
+        ready = $redis.scard("vmpooler__ready__#{pool['name']}")
+        total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
 
-      $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
-      $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
+        $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
+        $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
 
-      if $redis.get("vmpooler__empty__#{pool['name']}")
-        $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
-      elsif ready.zero?
-        $redis.set("vmpooler__empty__#{pool['name']}", 'true')
-        $logger.log('s', "[!] [#{pool['name']}] is empty")
-      end
+        if $redis.get("vmpooler__empty__#{pool['name']}")
+          $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
+        elsif ready.zero?
+          $redis.set("vmpooler__empty__#{pool['name']}", 'true')
+          $logger.log('s', "[!] [#{pool['name']}] is empty")
+        end
 
-      if total < pool['size']
-        (1..(pool['size'] - total)).each do |_i|
-          if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
-            begin
-              $redis.incr('vmpooler__tasks__clone')
-              pool_check_response[:cloned_vms] += 1
-              clone_vm(pool, provider)
-            rescue => err
-              $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
-              $redis.decr('vmpooler__tasks__clone')
-              raise
+        # Check to see if a pool size change has been made via the configuration API
+        # Since check_pool runs in a loop it does not
+        # otherwise identify this change when running
+        if $redis.hget('vmpooler__config__poolsize', pool['name'])
+          unless $redis.hget('vmpooler__config__poolsize', pool['name']).to_i == pool['size']
+            pool['size'] = Integer($redis.hget('vmpooler__config__poolsize', pool['name']))
+          end
+        end
+
+        if total < pool['size']
+          (1..(pool['size'] - total)).each do |_i|
+            if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
+              begin
+                $redis.incr('vmpooler__tasks__clone')
+                pool_check_response[:cloned_vms] += 1
+                clone_vm(pool, provider)
+              rescue => err
+                $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
+                $redis.decr('vmpooler__tasks__clone')
+                raise
+              end
             end
           end
         end
       end
 
+      # Remove VMs in excess of the configured pool size
+      remove_excess_vms(pool, provider, ready, total)
+
       pool_check_response
     end
 
@@ -739,6 +831,10 @@ module Vmpooler
       $redis.set('vmpooler__tasks__clone', 0)
       # Clear out vmpooler__migrations since stale entries may be left after a restart
       $redis.del('vmpooler__migration')
+      # Clear out any configuration changes in flight that were interrupted
+      $redis.del('vmpooler__config__updating')
+      # Ensure template deltas are created on each startup
+      $redis.del('vmpooler__template__prepared')
 
       # Copy vSphere settings to correct location.  This happens with older configuration files
       if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb
index a33e07d..71b281d 100644
--- a/lib/vmpooler/providers/base.rb
+++ b/lib/vmpooler/providers/base.rb
@@ -217,6 +217,14 @@ module Vmpooler
         def vm_exists?(pool_name, vm_name)
           !get_vm(pool_name, vm_name).nil?
         end
+
+        # inputs
+        #   [Hash] pool : Configuration for the pool
+        # returns
+        #   nil when successful. Raises error when encountered
+        def create_template_delta_disks(pool)
+          raise("#{self.class.name} does not implement create_template_delta_disks")
+        end
       end
     end
   end
diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb
index cfd087c..5bd364b 100644
--- a/lib/vmpooler/providers/vsphere.rb
+++ b/lib/vmpooler/providers/vsphere.rb
@@ -197,17 +197,10 @@ module Vmpooler
             target_cluster_name = get_target_cluster_from_config(pool_name)
             target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
-            # Extract the template VM name from the full path
+            # Get the template VM object
             raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless template_path =~ /\//
-            templatefolders = template_path.split('/')
-            template_name = templatefolders.pop
 
-            # Get the actual objects from vSphere
-            template_folder_object = find_folder(templatefolders.join('/'), connection, target_datacenter_name)
-            raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil?
-
-            template_vm_object = template_folder_object.find(template_name)
-            raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil?
+            template_vm_object = find_template_vm(pool, connection)
 
             # Annotate with creation time, origin template, etc.
             # Add extraconfig options that can be queried by vmtools
@@ -964,6 +957,30 @@ module Vmpooler
           raise("Cannot create folder #{new_folder}") if folder_object.nil?
           folder_object
         end
+
+        def find_template_vm(pool, connection)
+          datacenter = get_target_datacenter_from_config(pool['name'])
+          raise('cannot find datacenter') if datacenter.nil?
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => "#{datacenter}/vm/#{pool['template']}"
+          }
+
+          template_vm_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+          raise("Pool #{pool['name']} specifies a template VM of #{pool['template']} which does not exist for the provider #{name}") if template_vm_object.nil?
+
+          template_vm_object
+        end
+
+        def create_template_delta_disks(pool)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+            template_vm_object = find_template_vm(pool, connection)
+
+            template_vm_object.add_delta_disk_layer_on_all_disks
+          end
+        end
       end
     end
   end
diff --git a/spec/integration/api/v1/config_spec.rb b/spec/integration/api/v1/config_spec.rb
new file mode 100644
index 0000000..3cab23a
--- /dev/null
+++ b/spec/integration/api/v1/config_spec.rb
@@ -0,0 +1,160 @@
+require 'spec_helper'
+require 'rack/test'
+
+module Vmpooler
+  class API
+    module Helpers
+      def authenticate(auth, username_str, password_str)
+        username_str == 'admin' and password_str == 's3cr3t'
+      end
+    end
+  end
+end
+
+describe Vmpooler::API::V1 do
+  include Rack::Test::Methods
+
+  def app()
+    Vmpooler::API
+  end
+
+  describe '/config/pooltemplate' do
+    let(:prefix) { '/api/v1' }
+    let(:metrics) { Vmpooler::DummyStatsd.new }
+    let(:config) {
+      {
+        config: {
+          'site_name' => 'test pooler',
+          'vm_lifetime_auth' => 2,
+        },
+        pools: [
+          {'name' => 'pool1', 'size' => 5, 'template' => 'templates/pool1'},
+          {'name' => 'pool2', 'size' => 10}
+        ],
+        statsd: { 'prefix' => 'stats_prefix'},
+        alias: { 'poolone' => 'pool1' },
+        pool_names: [ 'pool1', 'pool2', 'poolone' ]
+      }
+    }
+
+    let(:current_time) { Time.now }
+
+    before(:each) do
+      redis.flushdb
+
+      app.settings.set :config, config
+      app.settings.set :redis, redis
+      app.settings.set :metrics, metrics
+      app.settings.set :config, auth: false
+      create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+    end
+
+    describe 'POST /config/pooltemplate' do
+      it 'updates a pool template' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails on nonexistent pools' do
+        post "#{prefix}/config/pooltemplate", '{"poolpoolpool":"templates/newtemplate"}'
+        expect_json(ok = false, http = 404)
+      end
+
+      it 'updates multiple pools' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool2":"templates/new_template2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when not all pools exist' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/new_template","pool3":"templates/new_template2"}'
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'returns no changes when the template does not change' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"templates/pool1"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a invalid template parameter is provided' do
+        post "#{prefix}/config/pooltemplate", '{"pool1":"template1"}'
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+
+    describe 'POST /config/poolsize' do
+      it 'changes a pool size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'changes a pool size for multiple pools' do
+        post "#{prefix}/config/poolsize", '{"pool1":"2","pool2":"2"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a specified pool does not exist' do
+        post "#{prefix}/config/poolsize", '{"pool10":"2"}'
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 200 when no change is required' do
+        post "#{prefix}/config/poolsize", '{"pool1":"5"}'
+        expect_json(ok = true, http = 200)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'succeeds with 201 when at least one pool changes' do
+        post "#{prefix}/config/poolsize", '{"pool1":"5","pool2":"5"}'
+        expect_json(ok = true, http = 201)
+
+        expected = { ok: true }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      it 'fails when a non-integer value is provided for size' do
+        post "#{prefix}/config/poolsize", '{"pool1":"four"}'
+        expect_json(ok = false, http = 404)
+
+        expected = { ok: false }
+
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+  end
+end
diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb
index 1073e0e..4e2ad90 100644
--- a/spec/unit/pool_manager_spec.rb
+++ b/spec/unit/pool_manager_spec.rb
@@ -1503,6 +1503,220 @@ EOT
     end
   end
 
+  describe 'update_pool_template' do
+    let(:template) { 'templates/pool_template' }
+    let(:new_template) { 'templates/new_pool_template' }
+    let(:vsphere_provider) { double('vsphere_provider') }
+    let(:config) {
+      YAML.load(<<-EOT
+---
+:pools:
+  - name: #{pool}
+    template: "#{template}"
+EOT
+      )
+    }
+
+    before(:each) do
+      expect(subject).not_to be_nil
+      redis.del('vmpooler__template')
+      redis.del('vmpooler__config__template')
+      redis.del('vmpooler__config__updating')
+    end
+
+    it 'returns when vmpooler config template is not set' do
+      expect(subject.update_pool_template(config[:pools][0], provider)).to be_nil
+    end
+
+    context 'with template that requires no change' do
+      before(:each) do
+        redis.hset('vmpooler__template', pool, template)
+        redis.hset('vmpooler__config__template', pool, template)
+      end
+
+      it 'should return' do
+        expect(subject.update_pool_template(config[:pools][0], provider)).to be_nil
+      end
+    end
+
+    context 'with a pool that requires an update' do
+      before(:each) do
+        redis.hset('vmpooler__template', pool, template)
+        redis.hset('vmpooler__config__template', pool, new_template)
+        allow(logger).to receive(:log)
+        allow(redis).to receive(:hset)
+        expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])
+      end
+
+      it 'should update the configuration value' do
+        expect(redis).to receive(:hset).with('vmpooler__template', pool, new_template)
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+
+      it 'should log a message for updating the template' do
+        expect(logger).to receive(:log).with('s', "[*] [#{pool}] template updated from #{template} to #{new_template}")
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+
+      it 'should log messages for creating template deltas' do
+        expect(logger).to receive(:log).with('s', "[*] [#{pool}] creating template deltas")
+        expect(logger).to receive(:log).with('s', "[*] [#{pool}] template deltas have been created")
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+    end
+
+    context 'with ready and pending vms' do
+      let(:vmname) { 'vm2' }
+      before(:each) do
+        create_ready_vm(pool,vmname)
+        create_pending_vm(pool,vmname)
+        redis.hset('vmpooler__template', pool, template)
+        redis.hset('vmpooler__config__template', pool, new_template)
+        allow(logger).to receive(:log)
+        allow(redis).to receive(:smove)
+        expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])
+      end
+
+      it 'should log a message for removing ready vms' do
+
+        expect(logger).to receive(:log).with('s', "[*] [#{pool}] removing ready and pending instances")
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+      it 'should remove ready vms' do
+        expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vmname)
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+
+      it 'should remove pending vms' do
+        expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vmname)
+
+        subject.update_pool_template(config[:pools][0], provider)
+      end
+    end
+
+    context 'when already updating' do
+      before(:each) do
+        redis.hset('vmpooler__template', pool, template)
+        redis.hset('vmpooler__config__template', pool, new_template)
+        redis.hset('vmpooler__config__updating', pool, 1)
+      end
+
+      it 'should return' do
+        expect(subject.update_pool_template(config[:pools][0], provider)).to be_nil
+      end
+    end
+  end
+
+  describe 'remove_excess_vms' do
+    let(:config) {
+      YAML.load(<<-EOT
+---
+:pools:
+  - name: #{pool}
+    size: 2
+EOT
+      )
+    }
+
+    before(:each) do
+      expect(subject).not_to be_nil
+    end
+
+    context 'with a nil ready value' do
+      it 'should return nil' do
+        expect(subject.remove_excess_vms(config[:pools][0], provider, nil, nil)).to be_nil
+      end
+    end
+
+    context 'with a total size less than the pool size' do
+      it 'should return nil' do
+        expect(subject.remove_excess_vms(config[:pools][0], provider, 1, 2)).to be_nil
+      end
+    end
+
+    context 'with a total size greater than the pool size' do
+      it 'should remove excess ready vms' do
+        expect(subject).to receive(:move_vm_queue).exactly(2).times
+
+        subject.remove_excess_vms(config[:pools][0], provider, 4, 4)
+      end
+
+      it 'should remove excess pending vms' do
+        create_pending_vm(pool,'vm1')
+        create_pending_vm(pool,'vm2')
+        create_pending_vm(pool,'vm3')
+        expect(subject).to receive(:move_vm_queue).exactly(3).times
+
+        subject.remove_excess_vms(config[:pools][0], provider, 2, 5)
+      end
+    end
+  end
+
+  describe 'prepare_template' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+  create_template_delta_disks: true
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    template: 'templates/pool1'
+EOT
+      )
+    }
+
+    it 'should return if a pool configuration is updating' do
+      redis.hset('vmpooler__config__updating', pool, 1)
+
+      expect(subject.prepare_template(config[:pools][0], provider)).to be_nil
+    end
+
+    it 'should return when a template is prepared' do
+      redis.hset('vmpooler__template__prepared', pool, pool['template'])
+
+      expect(subject.prepare_template(config[:pools][0], provider)).to be_nil
+    end
+
+    context 'when creating the template delta disks' do
+      before(:each) do
+        allow(redis).to receive(:hset)
+        allow(redis).to receive(:hdel)
+        allow(provider).to receive(:create_template_delta_disks)
+      end
+
+      it 'should mark the pool as updating' do
+        expect(redis).to receive(:hset).with('vmpooler__config__updating', pool, 1)
+
+        subject.prepare_template(config[:pools][0], provider)
+      end
+
+      it 'should run create template delta disks' do
+        expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])
+
+        subject.prepare_template(config[:pools][0], provider)
+      end
+
+      it 'should mark the template as prepared' do
+        expect(redis).to receive(:hset).with('vmpooler__template__prepared', pool, config[:pools][0]['template'])
+
+        subject.prepare_template(config[:pools][0], provider)
+      end
+
+      it' should mark the configuration as completed' do
+        expect(redis).to receive(:hdel).with('vmpooler__config__updating', pool)
+
+        subject.prepare_template(config[:pools][0], provider)
+      end
+    end
+  end
+
   describe "#execute!" do
     let(:config) {
       YAML.load(<<-EOT
@@ -1824,6 +2038,8 @@ EOT
       it 'should run startup tasks only once' do
         expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once
         expect(redis).to receive(:del).with('vmpooler__migration').once
+        expect(redis).to receive(:del).with('vmpooler__config__updating').once
+        expect(redis).to receive(:del).with('vmpooler__template__prepared').once
 
         subject.execute!(maxloop,0)
       end
@@ -2785,6 +3001,54 @@ EOT
         end
       end
 
+      context 'when a pool size configuration change is detected' do
+        let(:poolsize) { 2 }
+        let(:newpoolsize) { 3 }
+        before(:each) do
+          config[:pools][0]['size'] = poolsize
+          redis.hset('vmpooler__config__poolsize', pool, newpoolsize)
+          expect(provider).to receive(:vms_in_pool).with(pool).and_return([])
+        end
+
+        it 'should change the pool size configuration' do
+          subject._check_pool(config[:pools][0],provider)
+
+          expect(config[:pools][0]['size']).to be(newpoolsize)
+        end
+      end
+
+      context 'when a pool template is updating' do
+        before(:each) do
+          redis.hset('vmpooler__config__updating', pool, 1)
+          expect(provider).to receive(:vms_in_pool).with(pool).and_return([])
+        end
+
+        it 'should not call clone_vm to populate the pool' do
+          pool_size = 5
+          config[:pools][0]['size'] = pool_size
+
+          expect(subject).to_not receive(:clone_vm)
+
+          subject._check_pool(pool_object,provider)
+        end
+      end
+
+      context 'when an excess number of ready vms exist' do
+
+        before(:each) do
+          allow(redis).to receive(:scard)
+          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
+          expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
+          expect(provider).to receive(:vms_in_pool).with(pool).and_return([])
+        end
+
+        it 'should call remove_excess_vms' do
+          expect(subject).to receive(:remove_excess_vms).with(config[:pools][0], provider, 1, 2)
+
+          subject._check_pool(config[:pools][0],provider)
+        end
+      end
+
       context 'export metrics' do
         it 'increments metrics for ready queue' do
           create_ready_vm(pool,'vm1')
diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb
index 39c891e..4085321 100644
--- a/spec/unit/providers/vsphere_spec.rb
+++ b/spec/unit/providers/vsphere_spec.rb
@@ -283,6 +283,7 @@ EOT
 
     let(:clone_vm_task) { mock_RbVmomi_VIM_Task() }
     let(:new_vm_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
+    let(:new_template_object)  { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
 
     before(:each) do
       allow(subject).to receive(:connect_to_vsphere).and_return(connection)
@@ -305,19 +306,10 @@ EOT
       end
     end
 
-    context 'Given a template path that does not exist' do
-      before(:each) do
-        config[:pools][0]['template'] = 'missing_Templates/pool1'
-      end
-
-      it 'should raise an error' do
-        expect{ subject.create_vm(poolname, vmname) }.to raise_error(/specifies a template folder of .+ which does not exist/)
-      end
-    end
-
     context 'Given a template VM that does not exist' do
       before(:each) do
         config[:pools][0]['template'] = 'Templates/missing_template'
+        expect(subject).to receive(:find_template_vm).and_raise("specifies a template VM of #{vmname} which does not exist")
       end
 
       it 'should raise an error' do
@@ -327,7 +319,8 @@ EOT
 
     context 'Given a successful creation' do
       before(:each) do
-        template_vm = subject.find_folder('Templates',connection,datacenter_name).find('pool1')
+        template_vm = new_template_object
+        allow(subject).to receive(:find_template_vm).and_return(new_template_object)
         allow(template_vm).to receive(:CloneVM_Task).and_return(clone_vm_task)
         allow(clone_vm_task).to receive(:wait_for_completion).and_return(new_vm_object)
       end
@@ -339,7 +332,7 @@ EOT
       end
 
       it 'should use the appropriate Create_VM spec' do
-        template_vm = subject.find_folder('Templates',connection,datacenter_name).find('pool1')
+        template_vm = new_template_object
         expect(template_vm).to receive(:CloneVM_Task)
           .with(create_vm_spec(vmname,'pool1','datastore0'))
           .and_return(clone_vm_task)
@@ -3578,5 +3571,52 @@ EOT
     end
   end
 
+  describe 'find_template_vm' do
+    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine() }
 
+    before(:each) do
+      allow(connection.searchIndex).to receive(:FindByInventoryPath)
+    end
+    it 'should raise an error when the datacenter cannot be found' do
+      config[:providers][:vsphere]['datacenter'] = nil
+
+      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error('cannot find datacenter')
+    end
+
+    it 'should raise an error when the template specified cannot be found' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(nil)
+
+      expect{ subject.find_template_vm(config[:pools][0],connection) }.to raise_error("Pool #{poolname} specifies a template VM of #{config[:pools][0]['template']} which does not exist for the provider vsphere")
+    end
+
+    it 'should return the vm object' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(vm_object)
+
+      subject.find_template_vm(config[:pools][0],connection)
+    end
+  end
+
+  describe 'create_template_delta_disks' do
+    let(:template_object) { mock_RbVmomi_VIM_VirtualMachine({
+        :name => vmname,
+      })
+    }
+
+    before(:each) do
+      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
+    end
+
+    context 'with a template VM found' do
+
+      before(:each) do
+        expect(subject).to receive(:find_template_vm).and_return(template_object)
+      end
+
+      it 'should reconfigure the VM creating delta disks' do
+        expect(template_object).to receive(:add_delta_disk_layer_on_all_disks)
+
+        subject.create_template_delta_disks(config[:pools][0])
+      end
+    end
+  end
 end