Add tests for api additions. Begin adding tests for pool_manager additions

lib/vmpooler/api/v1.rb

@@ -359,8 +359,7 @@ module Vmpooler
       status 201
 
       platforms_with_aliases = []
-      payload.delete('request_id')
+      payload.reject { |k,v| k == 'request_id' }.each do |poolname, count|
-      payload.each do |poolname, count|
         selection = evaluate_template_aliases(poolname, count)
         selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
       end
@@ -800,22 +799,30 @@ module Vmpooler
 
       result = { 'ok' => false }
 
-      payload = JSON.parse(request.body.read)
+      begin
+        payload = JSON.parse(request.body.read)
 
-      if payload
+        if payload
-        invalid = invalid_templates(payload.reject { |k,v| k == 'request_id' })
+          invalid = invalid_templates(payload.reject { |k,v| k == 'request_id' })
-        if invalid.empty?
+          if invalid.empty?
-          result = generate_ondemand_request(payload)
+            result = generate_ondemand_request(payload)
-        else
+          else
-          result[:bad_templates] = invalid
+            result[:bad_templates] = invalid
-          invalid.each do |bad_template|
+            invalid.each do |bad_template|
-            metrics.increment('ondemandrequest.invalid.' + bad_template)
+              metrics.increment('ondemandrequest.invalid.' + bad_template)
+            end
+            status 404
           end
+        else
+          metrics.increment('ondemandrequest.invalid.unknown')
           status 404
         end
-      else
+      rescue JSON::ParserError
-        metrics.increment('ondemandrequest.invalid.unknown')
+        status 400
-        status 404
+        result = {
+          'ok' => false,
+          'message' => 'JSON payload could not be parsed'
+        }
       end
 
       JSON.pretty_generate(result)
@@ -908,16 +915,16 @@ module Vmpooler
 
     def check_ondemand_request(request_id)
       result = { 'ok' => false }
-      result['request_id'] = request_id
-      result['ready'] = false
       request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
       if request_hash.empty?
         result['message'] = "no request found for request_id '#{request_id}'"
         return result
       end
 
-      status 202
+      result['request_id'] = request_id
+      result['ready'] = false
       result['ok'] = true
+      status 202
 
       if request_hash['status'] == 'ready'
         result['ready'] = true

lib/vmpooler/pool_manager.rb

@@ -36,6 +36,7 @@ module Vmpooler
       @name_generator = Spicy::Proton.new
 
       @tasks = Concurrent::Hash.new
+      @tasks['ondemand_clone_count'] = 0
 
       # load specified providers from config file
       load_used_providers
@@ -113,17 +114,13 @@ module Vmpooler
       clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
       return true unless clone_stamp
 
-    # if clone_stamp == 'QUEUED'
-    #   $logger.log('s', "Waiting for clone_stamp. Got 'QUEUED'.")
-    #   return true
-    # end
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
       if time_since_clone > timeout
         if exists
-          pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if $request_id
+          pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
           redis.multi
           redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
-          redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}") if request_id
+          result = redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
           redis.exec
           $metrics.increment("errors.markedasfailed.#{pool}")
           $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
@@ -271,12 +268,7 @@ module Vmpooler
         @redis.with_metrics do |redis|
           # Check that VM is within defined lifetime
           checkouttime = redis.hget('vmpooler__active__' + pool, vm)
-      #   return if checkouttime == 'QUEUED'
           if checkouttime
-      #     if checkouttime == 'QUEUED'
-      #       $logger.log('s', "checkouttime is #{checkouttime}")
-      #       return
-      #     end
             time_since_checkout = Time.now - Time.parse(checkouttime)
             running = time_since_checkout / 60 / 60
 
@@ -376,6 +368,8 @@ module Vmpooler
           redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
           redis.exec
 
+          vm_hash = redis.hgetall("vmpooler__vm__#{new_vmname}")
+
           begin
             $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
             start = Time.now
@@ -425,11 +419,13 @@ module Vmpooler
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
-          redis.hdel('vmpooler__active__' + pool, vm)
+          redis.pipelined do
-          redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
+            redis.hdel('vmpooler__active__' + pool, vm)
+            redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
 
-          # Auto-expire metadata key
+            # Auto-expire metadata key
-          redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+            redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
+          end
 
           start = Time.now
 
@@ -516,7 +512,7 @@ module Vmpooler
         if provider_purge
           Thread.new do
             begin
-              purge_vms_and_folders(provider.to_s)
+              purge_vms_and_folders($providers[provider.to_s])
             rescue StandardError => e
               $logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
             end
@@ -527,13 +523,14 @@ module Vmpooler
     end
 
     # Return a list of pool folders
-    def pool_folders(provider_name)
+    def pool_folders(provider)
+      provider_name = provider.name
       folders = {}
       $config[:pools].each do |pool|
         next unless pool['provider'] == provider_name
 
         folder_parts = pool['folder'].split('/')
-        datacenter = $providers[provider_name].get_target_datacenter_from_config(pool['name'])
+        datacenter = provider.get_target_datacenter_from_config(pool['name'])
         folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
       end
       folders
@@ -550,8 +547,8 @@ module Vmpooler
     def purge_vms_and_folders(provider)
       configured_folders = pool_folders(provider)
       base_folders = get_base_folders(configured_folders)
-      whitelist = $providers[provider].provider_config['folder_whitelist']
+      whitelist = provider.provider_config['folder_whitelist']
-      $providers[provider].purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
     end
 
     def create_vm_disk(pool_name, vm, disk_size, provider)
@@ -855,20 +852,25 @@ module Vmpooler
             end
 
             if options[:pool_reset]
-              break if redis.sismember('vmpooler__poolreset', options[:poolname])
+              pending = redis.sismember('vmpooler__poolreset', options[:poolname])
-            end
-
-            if options[:pending_vm]
-              pending = redis.scard("vmpooler__pending__#{options[:poolname]}")
               break if pending
             end
 
-            if options[:ondemand_request]
+            if options[:pending_vm]
-              break if redis.zcard('vmpooler__provisioning__request')
+              pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
-              break if redis.zcard('vmpooler__provisioning__processing')
+              break unless pending_vm_count == 0
-              break if redis.zcard('vmpooler__odcreate__task')
             end
 
+            if options[:ondemand_request]
+              redis.multi
+              redis.zcard('vmpooler__provisioning__request')
+              redis.zcard('vmpooler__provisioning__processing')
+              redis.zcard('vmpooler__odcreate__task')
+              od_request, od_processing, od_createtask = redis.exec
+              break unless od_request == 0
+              break unless od_processing == 0
+              break unless od_createtask == 0
+            end
           end
 
           break if time_passed?(:exit_by, exit_by)
@@ -1462,6 +1464,8 @@ module Vmpooler
       in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1)
       in_progress_requests&.each do |request_id|
         next unless vms_ready?(request_id, redis)
+        $logger.log('s', 'vms are ready')
+
         redis.multi
         redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'ready')
         redis.zrem('vmpooler__provisioning__processing', request_id)

spec/helpers.rb

@@ -129,3 +129,16 @@ end
 def pool_has_ready_vm?(pool, vm, redis)
   !!redis.sismember('vmpooler__ready__' + pool, vm)
 end
+
+def create_ondemand_request_for_test(request_id, score, platforms_string, redis)
+  redis.zadd('vmpooler__provisioning__request', score, request_id)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+end
+
+def set_ondemand_request_ready(request_id, redis)
+  redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'ready')
+end
+
+def create_ondemand_vm(vmname, request_id, pool, pool_alias, redis)
+  redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vmname)
+end

spec/integration/api/v1/ondemandvm_spec.rb

@@ -5,30 +5,34 @@ describe Vmpooler::API::V1 do
   include Rack::Test::Methods
 
   def app()
-    Vmpooler::API
+    Vmpooler::API end
-  end
 
-  describe '/vm' do
+  describe '/ondemandvm' do
     let(:prefix) { '/api/v1' }
     let(:metrics) { Vmpooler::DummyStatsd.new }
     let(:config) {
       {
         config: {
           'site_name' => 'test pooler',
-          'vm_lifetime_auth' => 2
+          'vm_lifetime_auth' => 2,
+          'backend_weight' => {
+            'compute1' => 5
+          }
         },
         pools: [
           {'name' => 'pool1', 'size' => 0},
-          {'name' => 'pool2', 'size' => 0},
+          {'name' => 'pool2', 'size' => 0, 'clone_target' => 'compute1'},
-          {'name' => 'pool3', 'size' => 0}
+          {'name' => 'pool3', 'size' => 0, 'clone_target' => 'compute1'}
         ],
         alias: { 'poolone' => ['pool1'] },
-        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone', 'genericpool' ]
+        pool_names: [ 'pool1', 'pool2', 'pool3', 'poolone' ]
       }
     }
     let(:current_time) { Time.now }
     let(:vmname) { 'abcdefghijkl' }
     let(:checkoutlock) { Mutex.new }
+    let(:redis) { MockRedis.new }
+    let(:uuid) { SecureRandom.uuid }
 
     before(:each) do
       app.settings.set :config, config
@@ -37,16 +41,18 @@ describe Vmpooler::API::V1 do
       app.settings.set :config, auth: false
       app.settings.set :checkoutlock, checkoutlock
       create_token('abcdefghijklmnopqrstuvwxyz012345', 'jdoe', current_time)
+      config[:pools].each do |pool|
+        redis.sadd('vmpooler__pools', pool['name'])
+      end
     end
 
     describe 'POST /ondemandvm' do
-      let(:uuid) { SecureRandom.uuid }
 
       context 'with a configured pool' do
         it 'generates a request_id when none is provided' do
           expect(SecureRandom).to receive(:uuid).and_return(uuid)
           post "#{prefix}/ondemandvm", '{"pool1":"1"}'
-          expect_json(ok = true, http = 201)
+          expect_json(true, 201)
 
           expected = {
             "ok": true,
@@ -57,7 +63,7 @@ describe Vmpooler::API::V1 do
 
         it 'uses the given request_id when provided' do
           post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
-          expect_json(ok = true, http = 201)
+          expect_json(true, 201)
 
           expected = {
             "ok": true,
@@ -66,10 +72,10 @@ describe Vmpooler::API::V1 do
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
         end
 
-        it 'returns 404 when the request_id has been used' do
+        it 'returns 409 conflict error when the request_id has been used' do
           post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
           post "#{prefix}/ondemandvm", '{"pool1":"1","request_id":"1234"}'
-          expect_json(ok = false, http = 409)
+          expect_json(false, 409)
 
           expected = {
             "ok": false,
@@ -78,13 +84,50 @@ describe Vmpooler::API::V1 do
           }
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
         end
+
+        it 'uses a configured platform to fulfill a ondemand request' do
+          expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+          expect_json(true, 201)
+          expected = {
+            "ok": true,
+            "request_id": uuid
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        it 'creates a provisioning request in redis' do
+          expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          expect(redis).to receive(:zadd).with('vmpooler__provisioning__request', Integer, uuid).and_return(1)
+          post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+        end
+
+        it 'sets a platform string in redis for the request to indicate selected platforms' do
+          expect(SecureRandom).to receive(:uuid).and_return(uuid)
+          expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'requested', 'poolone:pool1:1')
+          post "#{prefix}/ondemandvm", '{"poolone":"1"}'
+        end
+
+        context 'with auth configured' do
+
+          it 'sets the token and user' do
+            app.settings.set :config, auth: true
+            expect(SecureRandom).to receive(:uuid).and_return(uuid)
+            allow(redis).to receive(:hset)
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:token', 'abcdefghijklmnopqrstuvwxyz012345')
+            expect(redis).to receive(:hset).with("vmpooler__odrequest__#{uuid}", 'token:user', 'jdoe')
+            post "#{prefix}/ondemandvm", '{"pool1":"1"}', {
+              'HTTP_X_AUTH_TOKEN' => 'abcdefghijklmnopqrstuvwxyz012345'
+            }
+          end
+        end
       end
 
       context 'with a pool that is not configured' do
         let(:badpool) { 'pool4' }
         it 'returns the bad template' do
           post "#{prefix}/ondemandvm", '{"pool4":"1"}'
-          expect_json(ok = false, http = 404)
+          expect_json(false, 404)
 
           expected = {
             "ok": false,
@@ -93,6 +136,74 @@ describe Vmpooler::API::V1 do
           expect(last_response.body).to eq(JSON.pretty_generate(expected))
         end
       end
+
+      it 'returns 400 and a message when JSON is invalid' do
+        post "#{prefix}/ondemandvm", '{"pool1":"1}'
+        expect_json(false, 400)
+        expected = {
+          "ok": false,
+          "message": "JSON payload could not be parsed"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+    end
+
+    describe 'GET /ondemandvm' do
+      it 'returns 404 with message when request is not found' do
+        get "#{prefix}/ondemandvm/#{uuid}"
+        expect_json(false, 404)
+        expected = {
+          "ok": false,
+          "message": "no request found for request_id '#{uuid}'"
+        }
+        expect(last_response.body).to eq(JSON.pretty_generate(expected))
+      end
+
+      context 'when the request is found' do
+        let(:score) { current_time }
+        let(:platforms_string) { 'pool1:pool1:1' }
+        before(:each) do
+          create_ondemand_request_for_test(uuid, score, platforms_string, redis)
+        end
+
+        it 'returns 202 while the request is waiting' do
+          get "#{prefix}/ondemandvm/#{uuid}"
+          expect_json(true, 202)
+          expected = {
+            "ok": true,
+            "request_id": uuid,
+            "ready": false,
+            "pool1": {
+              "ready": "0",
+              "pending": "1"
+            }
+          }
+          expect(last_response.body).to eq(JSON.pretty_generate(expected))
+        end
+
+        context 'with ready instances' do
+          before(:each) do
+            create_ondemand_vm(vmname, uuid, 'pool1', 'pool1', redis)
+            set_ondemand_request_ready(uuid, redis)
+          end
+
+          it 'returns 200 with hostnames when the request is ready' do
+            get "#{prefix}/ondemandvm/#{uuid}"
+            expect_json(true, 200)
+            expected = {
+              "ok": true,
+              "request_id": uuid,
+              "ready": true,
+              "pool1": {
+                "hostname": [
+                  vmname
+                ]
+              }
+            }
+            expect(last_response.body).to eq(JSON.pretty_generate(expected))
+          end
+        end
+      end
     end
   end
 end

spec/unit/pool_manager_spec.rb

@@ -15,7 +15,9 @@ describe 'Pool Manager' do
   let(:vm) { 'vm1' }
   let(:timeout) { 5 }
   let(:host) { double('host') }
-  let(:token) { 'token1234'}
+  let(:token) { 'token1234' }
+  let(:request_id) { '1234' }
+  let(:current_time) { Time.now }
 
   let(:provider_options) { {} }
   let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
@@ -25,6 +27,7 @@ describe 'Pool Manager' do
     timeout: 5
   ) { MockRedis.new }
   }
+  let(:redis) { MockRedis.new }
 
   let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'mock_provider', provider_options) }
 
@@ -217,6 +220,18 @@ EOT
         subject.fail_pending_vm(vm, pool, timeout, redis, true)
       end
     end
+
+    context 'with request_id' do
+
+      it 'creates a new odcreate task' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'clone',(Time.now - 900).to_s)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          subject.fail_pending_vm(vm, pool, timeout, redis, true, request_id)
+          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to eq(["#{pool}:#{pool}:1:#{request_id}"])
+        end
+      end
+    end
   end
 
   describe '#move_pending_vm_to_ready' do
@@ -294,6 +309,28 @@ EOT
         end
       end
     end
+
+    context 'with request_id' do
+      it 'sets the vm as active' do
+        redis_connection_pool.with do |redis|
+          expect(Time).to receive(:now).and_return(current_time).at_least(:once)
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+          expect(redis.hget("vmpooler__active__#{pool}", vm)).to eq(current_time.to_s)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to eq(current_time.to_s)
+          expect(redis.sismember("vmpooler__#{request_id}__#{pool}__#{pool}", vm)).to be true
+        end
+      end
+
+      it 'logs that the vm is ready for the request' do
+        redis_connection_pool.with do |redis|
+          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
+          expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'")
+
+          subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
+        end
+      end
+    end
   end
 
   describe '#check_ready_vm' do
@@ -711,12 +748,10 @@ EOT
           subject._clone_vm(pool,provider)
 
           expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1)
-          # Get the new VM Name from the pending pool queue as it should be the only entry
+          expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil
-          vm_name = redis.smembers("vmpooler__pending__#{pool}")[0]
+          expect(redis.hget("vmpooler__vm__#{vm}", 'template')).to eq(pool)
-          expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil
+          expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm}")).to_not be_nil
-          expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(pool)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'clone_time')).to_not be_nil
-          expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm_name}")).to_not be_nil
-          expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil
         end
       end
 
@@ -785,6 +820,37 @@ EOT
       it 'should raise the error' do
         expect{subject._clone_vm(pool,provider)}.to raise_error(/MockError/)
       end
+
+    end
+
+    context 'with request_id' do
+      before(:each) do
+        allow(metrics).to receive(:timing)
+        expect(metrics).to receive(:timing).with(/clone\./,/0/)
+        expect(provider).to receive(:create_vm).with(pool, String)
+        allow(logger).to receive(:log)
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:find_unique_hostname).with(pool, redis).and_return(vm)
+        end
+      end
+
+      it 'should set request_id and pool_alias on the vm data' do
+        redis_connection_pool.with do |redis|
+          subject._clone_vm(pool,provider,request_id,pool)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool)
+          expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id)
+        end
+      end
+
+      it 'should reduce the ondemand clone count' do
+        count = { 'ondemand_clone_count' => 1 }
+        subject.instance_variable_set(:@tasks, count)
+        redis_connection_pool.with do |redis|
+          subject._clone_vm(pool,provider,request_id,pool)
+        end
+        count = subject.instance_variable_get(:@tasks)
+        expect(count['ondemand_clone_count']).to eq(0)
+      end
     end
   end
 
@@ -831,9 +897,8 @@ EOT
 
       it 'should call redis expire with 0' do
         redis_connection_pool.with do |redis|
-          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be_nil
+          expect(redis).to receive(:expire).with("vmpooler__vm__#{vm}", 0)
           subject._destroy_vm(vm,pool,provider)
-          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be_nil
         end
       end
     end
@@ -1238,15 +1303,15 @@ EOT
     }
 
     it 'should return a list of pool folders' do
-      expect($providers[provider_name]).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
 
-      expect(subject.pool_folders(provider_name)).to eq(expected_response)
+      expect(subject.pool_folders(provider)).to eq(expected_response)
     end
 
     it 'should raise an error when the provider fails to get the datacenter' do
-      expect($providers[provider_name]).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
 
-      expect{ subject.pool_folders(provider_name) }.to raise_error(RuntimeError, 'mockerror')
+      expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
     end
   end
 
@@ -1277,16 +1342,16 @@ EOT
 
     it 'should run purge_unconfigured_folders' do
       expect(subject).to receive(:pool_folders).and_return(configured_folders)
-      expect($providers[provider_name]).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
-      expect($providers[provider_name]).to receive(:provider_config).and_return({})
+      expect(provider).to receive(:provider_config).and_return({})
 
       subject.purge_vms_and_folders(provider)
     end
 
     it 'should raise any errors' do
       expect(subject).to receive(:pool_folders).and_return(configured_folders)
-      expect($providers[provider_name]).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
-      expect($providers[provider_name]).to receive(:provider_config).and_return({})
+      expect(provider).to receive(:provider_config).and_return({})
 
       expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
     end
@@ -3196,11 +3261,6 @@ EOT
       let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
 
       context 'when a pool reset is requested' do
-        before(:each) do
-          redis_connection_pool.with do |redis|
-            redis.sadd('vmpooler__poolreset', pool)
-          end
-        end
 
         it 'should sleep until the reset request is detected' do
           redis_connection_pool.with do |redis|
@@ -3212,6 +3272,62 @@ EOT
         end
       end
     end
+
+    describe 'with the pending_vm wakeup option' do
+      let(:wakeup_option) {{
+        :pending_vm => true,
+        :poolname => pool
+      }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      context 'when a pending_vm is detected' do
+
+        it 'should sleep until the pending instance' do
+          redis_connection_pool.with do |redis|
+            expect(subject).to receive(:sleep).exactly(3).times
+            expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(0,0,1)
+          end
+
+          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+        end
+      end
+    end
+
+    describe 'with the ondemand_request wakeup option' do
+      let(:wakeup_option) {{ :ondemand_request => true }}
+
+      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur
+
+      it 'should sleep until the provisioning request is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
+          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[1,0,0])
+        end
+
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+
+      it 'should sleep until provisioning processing is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
+          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,1,0])
+        end
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+
+      it 'should sleep until ondemand creation task is detected' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:sleep).exactly(3).times
+          expect(redis).to receive(:multi).and_return('OK').exactly(3).times
+          expect(redis).to receive(:exec).and_return([0,0,0],[0,0,0],[0,0,1])
+        end
+
+        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
+      end
+    end
   end
 
   describe "#check_pool" do
@@ -4478,9 +4594,96 @@ EOT
         subject._check_pool(config[:pools][0],provider)
       end
     end
+  end
 
-    #
+  describe 'process_ondemand_requests' do
+    context 'with no requests' do
+      it 'returns 0' do
+        result = subject.process_ondemand_requests
+        expect(result).to eq(0)
+      end
 
+      it 'runs process_ondemand_vms' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:process_ondemand_vms).with(redis).and_return(0)
+          subject.process_ondemand_requests
+        end
+      end
 
+      it 'checks ready requests' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:check_ondemand_requests_ready).with(redis).and_return(0)
+          subject.process_ondemand_requests
+        end
+      end
+    end
+
+    context 'with provisioning requests' do
+      before(:each) do
+        redis_connection_pool.with do |redis|
+          redis.zadd('vmpooler__provisioning__request', current_time, request_id)
+        end
+      end
+
+      it 'returns the number of requests processed' do
+        result = subject.process_ondemand_requests
+        expect(result).to eq(1)
+      end
+
+      it 'runs create_ondemand_vms for each request' do
+        redis_connection_pool.with do |redis|
+          expect(subject).to receive(:create_ondemand_vms).with(request_id, redis)
+          subject.process_ondemand_requests
+        end
+      end
+    end
+  end
+
+  describe '#create_ondemand_vms' do
+    context 'when requested does not have corresponding data' do
+     #end
+      it 'logs an error' do
+        redis_connection_pool.with do |redis|
+          expect(logger).to receive(:log).with('s', "Failed to find odrequest for request_id '1111'")
+          #expect(redis).to receive(:zrem).with('vmpooler__provisioning__request', '1111')
+          subject.create_ondemand_vms('1111', redis)
+        end
+      end
+    end
+
+    context 'with a request that has data' do
+      let(:request_string) { "#{pool}:#{pool}:1" }
+      before(:each) do
+        expect(Time).to receive(:now).and_return(current_time).at_least(:once)
+        redis_connection_pool.with do |redis|
+          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
+        end
+      end
+
+      it 'creates tasks for instances to be provisioned' do
+        redis_connection_pool.with do |redis|
+          allow(redis).to receive(:zadd)
+          expect(redis).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}")
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+
+      it 'adds a member to provisioning__processing' do
+        redis_connection_pool.with do |redis|
+          allow(redis).to receive(:zadd)
+          expect(redis).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id)
+          subject.create_ondemand_vms(request_id, redis)
+        end
+      end
+    end
+  end
+
+  describe '#process_ondemand_vms' do
+    it 'returns the length of the queue' do
+      redis_connection_pool.with do |redis|
+        result = subject.process_ondemand_vms(redis)
+        expect(result).to eq(0)
+      end
+    end
   end
 end