require 'spec_helper'
require 'time'
require 'mock_redis'

# Custom RSpec :Matchers

# Match a Hashtable['name'] is an expected value
RSpec::Matchers.define :a_pool_with_name_of do |value|
  match { |actual| actual['name'] == value }
end
describe 'Pool Manager' do
  let(:logger) { MockLogger.new }
  let(:metrics) { Vmpooler::Metrics::DummyStatsd.new }
  let(:pool) { 'pool1' }
  let(:vm) { 'vm1' }
  let(:timeout) { 5 }
  let(:timeout_notification) { 2 }
  let(:host) { double('host') }
  let(:token) { 'token1234' }
  let(:request_id) { '1234' }
  let(:current_time) { Time.now }

  let(:provider_options) { {} }
  let(:dns_options) { {} }
  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
    metrics: metrics,
    connpool_type: 'redis_connection_pool',
    connpool_provider: 'testprovider',
    size: 1,
    timeout: 5
  ) { MockRedis.new }
  }

  let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, 'mock_provider', provider_options) }
  let(:dns_plugin) { MockPoolManagerDnsBase.new }
  let(:config) { YAML.load(<<-EOT
---
:config: {}
:dns_configs: {}
:providers:
  :mock:
    dns_class: base
:redis: {}
:pools:
  - name: '#{pool}'
    size: 1
    dns_plugin: 'mock'
EOT
    )
  }

  subject { Vmpooler::PoolManager.new(config, logger, redis_connection_pool, metrics) }

  describe '#config' do
    before do
      expect(subject).not_to be_nil
    end

    it 'should return the current configuration' do
      expect(subject.config).to eq(config)
    end
  end

  describe '#load_used_providers' do
    let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
    )
    }
    it do
      files = ["#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
      expect(subject.load_used_providers).to eq(files)
    end
  end

  describe '#load_used_dns_plugins' do
    let(:config) { YAML.load(<<-EOT
---
:config:
:dns_configs:
  :base:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
    )
    }
    it do
      files = ['vmpooler/dns/base']
      expect(subject.load_used_dns_plugins).to eq(files)
    end


  end

  describe '#used_providers' do
    context 'with no named providers' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
      )
      }
      it do
        result = ["dummy"]
        expect(subject.used_providers).to eq(result)
      end
    end
    context 'with one named provider without a provider_class key' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
      )
      }
      it do
        result = ["mock", "dummy"]
        expect(subject.used_providers).to eq(result)
      end
    end

    context 'with one named provider with a provider_class key' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
    provider_class: 'mock_mock'
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
      )
      }
      it do
        result = ["mock_mock", "dummy"]
        expect(subject.used_providers).to eq(result)
      end
    end

    context 'with one named provider with a provider_class key and one without' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
    provider_class: 'mock_mock'
  :foo:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'spoof'
    EOT
      )
      }
      it do
        result = ["mock_mock", "foo", "dummy"]
        expect(subject.used_providers).to eq(result)
      end
    end
  end

  it '#default_providers' do
    expect(subject.default_providers).to eq(['dummy'])
  end

  describe '#check_pending_vm' do
    before do
      expect(subject).not_to be_nil
    end

    it 'calls _check_pending_vm' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)

      subject.check_pending_vm(vm, pool, timeout, timeout_notification, provider)
    end
  end

  describe '#_check_pending_vm' do
    before do
      expect(subject).not_to be_nil
    end

    context 'host is in pool' do

      it 'calls move_pending_vm_to_ready if host is ready' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(true)
          expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool, redis, nil)
        end

        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
      end

      it 'calls fail_pending_vm if host is not ready' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).with(pool, vm, redis).and_return(false)
          expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, timeout_notification, redis)
        end

        subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)
      end
    end

    context 'with a locked vm mutex' do
      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject).to receive(:vm_mutex).and_return(mutex)

        expect(subject._check_pending_vm(vm, pool, timeout, timeout_notification, provider)).to be_nil
      end
    end
  end

  describe '#remove_nonexistent_vm' do
    before do
      expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      expect(dns_plugin).to receive(:delete_record)
      expect(subject).not_to be_nil
    end

    it 'removes VM from pending in redis' do
      redis_connection_pool.with do |redis|
        create_pending_vm(pool,vm,redis)

        subject.remove_nonexistent_vm(vm, pool, redis)
        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
      end
    end

    it 'logs msg' do
      expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")

      redis_connection_pool.with do |redis|
        subject.remove_nonexistent_vm(vm, pool, redis)
      end
    end
  end

 describe '#fail_pending_vm' do
    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_pending_vm(pool,vm,redis)
        config[:config]['vm_checktime'] = 15
      end
    end

    it 'takes no action if VM is not cloning' do
      redis_connection_pool.with do |redis|
        expect(logger).to_not receive(:log)
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
      end
    end

    it 'takes no action if VM is within timeout' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
        expect(logger).to_not receive(:log)
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis)).to eq(true)
        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
      end
    end

    it 'moves VM to completed queue if VM has exceeded timeout and exists' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
        expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
        expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
      end
    end

    it 'logs message if VM has exceeded timeout and exists' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
        expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: ")
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(true)
      end
    end

    it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s)
        expect(logger).to_not receive(:log)
        expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool, redis)
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: false)).to eq(true)
      end
    end

    it 'swallows error if an error is raised' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
        expect(subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)).to eq(false)
      end
    end

    it 'logs message if an error is raised' do
      redis_connection_pool.with do |redis|
        redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
        expect(logger).to receive(:log).with('d', String)

        subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
      end
    end

    context 'with request_id' do

      it 'creates a new odcreate task' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'clone',(Time.now - 900).to_s)
          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
          subject.fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to eq(["#{pool}:#{pool}:1:#{request_id}"])
        end
      end
    end
  end

  describe '#handle_timed_out_vm' do
    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_pending_vm(pool, vm, redis)
        config[:config]['max_vm_retries'] = 3
      end
    end

    context 'without request_id' do
      it 'moves VM to completed queue and returns error' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'open_socket_error', 'connection failed')
          result = subject.handle_timed_out_vm(vm, pool, redis)
          
          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
          expect(result).to eq('connection failed')
        end
      end
    end

    context 'with request_id and transient error' do
      before(:each) do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'network timeout')
          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'Timeout::Error')
        end
      end

      it 'retries on first failure' do
        redis_connection_pool.with do |redis|
          subject.handle_timed_out_vm(vm, pool, redis)
          
          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'retry_count')).to eq('1')
          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).to include("#{pool}:#{pool}:1:#{request_id}")
        end
      end

      it 'marks as failed after max retries' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__odrequest__#{request_id}", 'retry_count', '3')
          
          subject.handle_timed_out_vm(vm, pool, redis)
          
          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to eq('Max retry attempts exceeded')
          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
        end
      end
    end

    context 'with request_id and permanent error' do
      before(:each) do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'request_id', request_id)
          redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
          redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'pending')
          redis.hset("vmpooler__vm__#{vm}", 'clone_error', 'template not found')
          redis.hset("vmpooler__vm__#{vm}", 'clone_error_class', 'RuntimeError')
        end
      end

      it 'immediately marks as failed without retrying' do
        redis_connection_pool.with do |redis|
          subject.handle_timed_out_vm(vm, pool, redis)
          
          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'status')).to eq('failed')
          expect(redis.hget("vmpooler__odrequest__#{request_id}", 'failure_reason')).to include('Configuration error')
          expect(redis.zrange('vmpooler__odcreate__task', 0, -1)).not_to include("#{pool}:#{pool}:1:#{request_id}")
        end
      end
    end
  end

  describe '#permanent_error?' do
    before do
      expect(subject).not_to be_nil
    end

    it 'identifies template not found errors as permanent' do
      expect(subject.permanent_error?('template not found', 'RuntimeError')).to be(true)
    end

    it 'identifies invalid path errors as permanent' do
      expect(subject.permanent_error?('invalid path specified', 'ArgumentError')).to be(true)
    end

    it 'identifies permission denied errors as permanent' do
      expect(subject.permanent_error?('permission denied', 'SecurityError')).to be(true)
    end

    it 'identifies ArgumentError class as permanent' do
      expect(subject.permanent_error?('some argument error', 'ArgumentError')).to be(true)
    end

    it 'identifies network errors as transient' do
      expect(subject.permanent_error?('connection timeout', 'Timeout::Error')).to be(false)
    end

    it 'identifies socket errors as transient' do
      expect(subject.permanent_error?('connection refused', 'Errno::ECONNREFUSED')).to be(false)
    end

    it 'returns false for nil inputs' do
      expect(subject.permanent_error?(nil, nil)).to be(false)
      expect(subject.permanent_error?('error', nil)).to be(false)
      expect(subject.permanent_error?(nil, 'Error')).to be(false)
    end
  end

  describe '#move_pending_vm_to_ready' do
    let(:host) { { 'hostname' => vm }}

    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_pending_vm(pool,vm,redis)
      end
    end

    context 'when hostname matches VM name' do
      it 'should move the VM from pending to ready pool' do
        redis_connection_pool.with do |redis|
          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true)
          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false)
          subject.move_pending_vm_to_ready(vm, pool, redis)
          expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false)
          expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true)
        end
      end

      it 'should log a message' do
        expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")

        redis_connection_pool.with do |redis|
          subject.move_pending_vm_to_ready(vm, pool, redis)
        end
      end

      it 'should receive time_to_ready_state metric' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
          expect(metrics).to receive(:timing).with(/time_to_ready_state\./,/0/)

          subject.move_pending_vm_to_ready(vm, pool, redis)
        end
      end


      it 'should set the boot time in redis' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s)
          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
          subject.move_pending_vm_to_ready(vm, pool, redis)
          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to_not be_nil
          # TODO Should we inspect the value to see if it's valid?
        end
      end

      it 'should not determine boot timespan if clone start time not set' do
        redis_connection_pool.with do |redis|
          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to be_nil
          subject.move_pending_vm_to_ready(vm, pool, redis)
          expect(redis.hget('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm)).to match(/\d\.\d{2}/)
        end
      end

      it 'should raise error if clone start time is not parsable' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate')
          expect{subject.move_pending_vm_to_ready(vm, pool, redis)}.to raise_error(/iamnotparsable_asdate/)
        end
      end

      it 'should save the last boot time' do
        redis_connection_pool.with do |redis|
          expect(redis.hget('vmpooler__lastboot', pool)).to be(nil)
          subject.move_pending_vm_to_ready(vm, pool, redis)
          expect(redis.hget('vmpooler__lastboot', pool)).to_not be(nil)
        end
      end
    end

    context 'with request_id' do
      context 'with a pending request' do
        before(:each) do
          allow(subject).to receive(:check_ondemand_request_ready)
          config[:config]['ondemand_request_ttl'] = 20
        end

        it 'sets the vm as active' do
          redis_connection_pool.with do |redis|
            expect(Time).to receive(:now).and_return(current_time).at_least(:once)
            redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
            expect(redis.hget("vmpooler__active__#{pool}", vm)).to eq(current_time.to_s)
            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to eq(current_time.to_s)
            expect(redis.sismember("vmpooler__#{request_id}__#{pool}__#{pool}", vm)).to be true
          end
        end

        it 'logs that the vm is ready for the request' do
          redis_connection_pool.with do |redis|
            redis.hset("vmpooler__vm__#{vm}", 'pool_alias', pool)
            expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'")

            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
          end
        end
      end

      context 'when the request has been marked as failed' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
          end
        end

        it 'moves the vm to completed' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
          end
        end

        it 'returns nil' do
          redis_connection_pool.with do |redis|
            result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
            expect(result).to be nil
          end
        end
      end

      context 'when the request has been marked as deleted' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
          end
        end

        it 'moves the vm to completed' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:move_vm_queue).with(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
          end
        end

        it 'returns nil' do
          redis_connection_pool.with do |redis|
            result = subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
            expect(result).to be nil
          end
        end
      end

      context 'with auth on the request' do
        let(:user) { 'vmpuser' }
        let(:platform_alias) { pool }
        let(:platforms_string) { "#{platform_alias}:#{pool}:1" }
        let(:score) { current_time.to_i }
        before(:each) do
          config[:config]['ondemand_request_ttl'] = 20
          allow(subject).to receive(:check_ondemand_request_ready)
          redis_connection_pool.with do |redis|
            create_ondemand_request_for_test(request_id, score, platforms_string, redis, user, token)
          end
        end

        it 'should specify auth data on the vm' do
          redis_connection_pool.with do |redis|
            allow(redis).to receive(:hset)

            subject.move_pending_vm_to_ready(vm, pool, redis, request_id)
            expect(redis.hget("vmpooler__vm__#{vm}", 'token:token')).to eq(token)
            expect(redis.hget("vmpooler__vm__#{vm}", 'token:user')).to eq(user)
          end
        end
      end
    end
  end

  describe '#check_ready_vm' do
    let(:ttl) { 5 }
    let(:poolconfig) { config[:pools][0] }

    before do
      expect(subject).not_to be_nil
    end

    it 'calls _check_ready_vm' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_check_ready_vm).with(vm, poolconfig, ttl, provider)

      subject.check_ready_vm(vm, poolconfig, ttl, provider)
    end
  end

  describe '#_check_ready_vm' do
    let(:ttl) { 5 }
    let(:host) { {} }
    let(:config) { YAML.load(<<-EOT
---
:config: {}
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
:pool_index:
  '#{pool}': 0
EOT
      )
    }

    before(:each) do
      redis_connection_pool.with do |redis|
        create_ready_vm(pool,vm,redis)
      end
      config[:config]['vm_checktime'] = 15

      # Create a VM which is powered on
      host['hostname'] = vm
      host['powerstate'] = 'PoweredOn'
      allow(provider).to receive(:get_vm).with(pool,vm).and_return(host)
    end

    context 'a VM that does not need to be checked' do
      it 'should do nothing' do
        redis_connection_pool.with do |redis|
          check_stamp = (Time.now - 60).to_s
          redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp)
          expect(provider).to receive(:get_vm).exactly(0).times
          subject._check_ready_vm(vm, pool, ttl, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp)
        end
      end
    end

    context 'a VM that has never been checked' do
      it 'should set the current check timestamp' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil
          subject._check_ready_vm(vm, pool, ttl, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil
        end
      end
    end

    context 'a VM that needs to be checked' do
      let(:last_check_date) { Time.now - 901 }
      before(:each) do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'check',last_check_date)
        end
      end

      it 'should set the current check timestamp' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date.to_s)
          subject._check_ready_vm(vm, pool, ttl, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date.to_s)
        end
      end

      context 'and is ready' do
        it 'should only set the next check interval' do
          subject._check_ready_vm(vm, pool, ttl, provider)
        end
      end

      context 'has correct name and is not ready' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now - 200)
            redis.sadd("vmpooler__ready__#{pool}", vm)
          end
        end

        it 'should move the VM to the completed queue' do
          redis_connection_pool.with do |redis|
            expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm)

            subject._check_ready_vm(vm, pool, ttl, provider)
          end
        end

        it 'should move the VM to the completed queue in Redis' do
          redis_connection_pool.with do |redis|
            subject._check_ready_vm(vm, pool, ttl, provider)
            expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
          end
        end

        it 'should log messages about being unreachable' do
          expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' removed from 'ready' queue. vm unreachable with error: ")

          subject._check_ready_vm(vm, pool, ttl, provider)
        end
      end

      context 'with hostname mismatch checking enabled' do

        context 'when less than 60 seconds since a VM moved to ready' do
          before(:each) do
            redis_connection_pool.with do |redis|
              redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
            end
          end

          it 'should return nil' do
            expect(subject._check_ready_vm(vm, pool, ttl, provider)).to be_nil
          end
        end

        context 'with a hostname mismatch' do
          let(:different_hostname) { 'different_name' }
          let(:longer_ttl) { 20 }
          before(:each) do
            host['hostname'] = different_hostname
            redis_connection_pool.with do |redis|
              expect(subject).to receive(:mismatched_hostname?).with(vm, pool, provider, redis).and_return(true)
              redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now - 300)
            end
          end

          it 'should not run vm_still_ready?' do
            expect(subject).to_not receive(:vm_still_ready?)

            subject._check_ready_vm(vm, pool, longer_ttl, provider)
          end
        end
      end

      context 'with hostname mismatch checking disabled on the pool' do
        before(:each) do
          config[:pools][0]['check_hostname_for_mismatch'] = false
        end

        it 'should not run get_vm' do
          expect(provider).to_not receive(:get_vm)

          subject._check_ready_vm(vm, pool, ttl, provider)
        end
      end

      context 'with hostname mismatch checking disabled in config' do
        before(:each) do
          config[:config]['check_ready_vm_hostname_for_mismatch'] = false
        end

        it 'should not run get_vm' do
          expect(provider).to_not receive(:get_vm)

          subject._check_ready_vm(vm, pool, ttl, provider)
        end
      end
    end

    context 'with a locked vm mutex' do
      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject).to receive(:vm_mutex).and_return(mutex)

        expect(subject._check_ready_vm(vm, pool, ttl, provider)).to be_nil
      end
    end
  end

  describe '#check_running_vm' do
    let(:provider) { double('provider') }
    let (:ttl) { 5 }

    before do
      expect(subject).not_to be_nil
    end

    it 'calls _check_running_vm' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_check_running_vm).with(vm, pool, ttl, provider)

      subject.check_running_vm(vm, pool, ttl, provider)
    end
  end

  describe '#_check_running_vm' do
    let(:host) { {} }

    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_running_vm(pool,vm, redis)
      end

      # Create a VM which is powered on
      host['hostname'] = vm
      host['powerstate'] = 'PoweredOn'
      allow(provider).to receive(:get_vm).with(pool,vm).and_return(host)
    end

    it 'moves a missing VM to the completed queue' do
      redis_connection_pool.with do |redis|
        expect(provider).to receive(:vm_ready?).and_return(false)
        expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil)
        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
        subject._check_running_vm(vm, pool, timeout, provider)
        expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
      end
    end

    context 'valid host' do
      it 'should kill a VM if it has no checkout time' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).and_return(true)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
          subject._check_running_vm(vm, pool, 0, provider)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
        end
      end

      it 'should log a message when the machine is removed due to no active data' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).and_return(true)
          expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' is listed as running, but has no checkouttime data. Removing from running")
          subject._check_running_vm(vm, pool, 0, provider)
        end
      end

      it 'should not move VM if TTL is zero' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).and_return(true)
          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
          subject._check_running_vm(vm, pool, 0, provider)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
        end
      end

      it 'should move VM when past TTL' do
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true)
          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false)
          subject._check_running_vm(vm, pool, timeout, provider)
          expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(false)
          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
        end
      end

      it 'should try to tag if it has not been done' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).and_return(true)
          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
          expect(provider).to receive(:tag_vm_user)
          subject._check_running_vm(vm, pool, 0, provider)
        end
      end

      it 'should not try to tag if it has been done' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:vm_ready?).and_return(true)
          redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s)
          redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true')
          expect(provider).not_to receive(:tag_vm_user)
          subject._check_running_vm(vm, pool, 0, provider)
        end
      end
    end

    context 'with a locked vm mutex' do
      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject).to receive(:vm_mutex).and_return(mutex)

        expect(subject._check_running_vm(vm, pool, timeout, provider)).to be_nil
      end
    end
  end

  describe '#move_vm_queue' do
    let(:queue_from) { 'pending' }
    let(:queue_to) { 'completed' }
    let(:message) { 'message' }

    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_pending_vm(pool, vm, redis, token)
      end
    end

    it 'VM should be in the "from queue" before the move' do
      redis_connection_pool.with do |redis|
        expect(redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
      end
    end

    it 'VM should not be in the "from queue" after the move' do
      redis_connection_pool.with do |redis|
        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
        expect(!redis.sismember("vmpooler__#{queue_from}__#{pool}",vm))
      end
    end

    it 'VM should not be in the "to queue" before the move' do
      redis_connection_pool.with do |redis|
        expect(!redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
      end
    end

    it 'VM should be in the "to queue" after the move' do
      redis_connection_pool.with do |redis|
        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
        expect(redis.sismember("vmpooler__#{queue_to}__#{pool}",vm))
      end
    end

    it 'should log a message' do
      allow(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' #{message}")

      redis_connection_pool.with do |redis|
        subject.move_vm_queue(pool, vm, queue_from, queue_to, redis, message)
      end
    end
  end

  describe '#clone_vm' do
    let (:pool_object) { { 'name' => pool } }

    before do
      expect(subject).not_to be_nil
      expect(Thread).to receive(:new).and_yield
    end

    it 'calls _clone_vm' do
      expect(subject).to receive(:_clone_vm).with(pool_object,provider,dns_plugin,nil,nil)

      subject.clone_vm(pool_object,provider,dns_plugin)
    end

    it 'logs a message if an error is raised' do
      allow(logger).to receive(:log)
      expect(logger).to receive(:log).with('s',"[!] [#{pool}] failed while cloning VM with an error: MockError")
      expect(subject).to receive(:_clone_vm).with(pool,provider,dns_plugin,nil,nil).and_raise('MockError')

      expect{subject.clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
    end
  end

  describe '#find_unique_hostname' do
    # it should return the hostname
    before do
      allow(subject).to receive(:generate_and_check_hostname).and_return(["spicy-proton", true])
      allow(subject).to receive(:get_provider_for_pool).and_return(provider)
    end

    context 'with a setting to skip dns check' do
      let(:config) {
        YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:providers:
  :mock_provider:
    skip_dns_check_before_creating_vm: true
:pools:
  - name: '#{pool}'
    size: 1
    dns_config: 'mock'
        EOT
        )
      }
      it 'should skip the dns check' do
        #method is skipped
        expect(subject).not_to receive(:check_dns_available)
        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
      end
    end
    context 'without the setting to skip dns check' do
      let(:config) {
        YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:providers:
  :mock_provider:
:pools:
  - name: '#{pool}'
    size: 1
    dns_config: 'mock'
        EOT
        )
      }
      it 'should run the dns check and pass' do
        #method is skipped
        expect(subject).to receive(:check_dns_available).and_return(["1.1.1.1",true])
        expect(subject.find_unique_hostname(pool)).to eq("spicy-proton")
      end

      it 'should run the dns check and fail' do
        #method is skipped
        allow(subject).to receive(:check_dns_available).and_return(["1.1.1.1",false]).exactly(3).times
        expect{subject.find_unique_hostname(pool)}.to raise_error(RuntimeError)
      end
    end
  end

  describe '#_clone_vm' do
    let (:pool_object) { { 'name' => pool } }
    let (:redis_ttl) { 1 }

    before do
      expect(subject).not_to be_nil
    end

    let(:config) {
      YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:config:
  prefix: "prefix"
:redis:
  ttl: #{redis_ttl}
:pools:
  - name: 'pool1'
    size: 1
    dns_config: 'mock'
EOT
      )
    }

    context 'with no errors during cloning' do
      before(:each) do
        allow(metrics).to receive(:timing)
        allow(metrics).to receive(:gauge)
        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
        expect(provider).to receive(:create_vm).with(pool, String)
        allow(provider).to receive(:get_vm_ip_address).and_return(1)
        allow(subject).to receive(:get_domain_for_pool).and_return('example.com')
        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
        allow(logger).to receive(:log)
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
        end
      end

      it 'should create a cloning VM' do
        redis_connection_pool.with do |redis|
          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)

          subject._clone_vm(pool,provider,dns_plugin)

          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1)
          expect(redis.hget("vmpooler__vm__#{vm}", 'clone')).to_not be_nil
          expect(redis.hget("vmpooler__vm__#{vm}", 'template')).to eq(pool)
          expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm}")).to_not be_nil
          expect(redis.hget("vmpooler__vm__#{vm}", 'clone_time')).to_not be_nil
        end
      end

      it 'should decrement the clone tasks counter' do
        redis_connection_pool.with do |redis|
          redis.incr('vmpooler__tasks__clone')
          redis.incr('vmpooler__tasks__clone')
          expect(redis.get('vmpooler__tasks__clone')).to eq('2')
          subject._clone_vm(pool,provider,dns_plugin)
          expect(redis.get('vmpooler__tasks__clone')).to eq('1')
        end
      end

      it 'should log a message that is being cloned from a template' do
        expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] Starting to clone '(.+)'/)

        subject._clone_vm(pool,provider,dns_plugin)
      end

      it 'should log a message that it completed being cloned' do
        expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned in [0-9.]+ seconds/)

        subject._clone_vm(pool,provider,dns_plugin)
      end
    end

    context 'with a failure to get ip address after cloning' do
      it 'should log a message that it completed being cloned' do
        allow(metrics).to receive(:timing)
        allow(metrics).to receive(:gauge)
        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
        expect(provider).to receive(:create_vm).with(pool, String)
        allow(provider).to receive(:get_vm_ip_address).and_return(nil)

        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(StandardError)
      end
    end

    context 'with an error during cloning' do
      before(:each) do
        expect(provider).to receive(:create_vm).with(pool, String).and_raise('MockError')
        allow(subject).to receive(:get_domain_for_pool)
        allow(logger).to receive(:log)
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
        end
      end

      it 'should not create a cloning VM' do
        redis_connection_pool.with do |redis|
          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)

          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)

          expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0)
          # Get the new VM Name from the pending pool queue as it should be the only entry
          vm_name = redis.smembers("vmpooler__pending__#{pool}")[0]
          expect(vm_name).to be_nil
        end
      end

      it 'should decrement the clone tasks counter' do
        redis_connection_pool.with do |redis|
          redis.incr('vmpooler__tasks__clone')
          redis.incr('vmpooler__tasks__clone')
          expect(redis.get('vmpooler__tasks__clone')).to eq('2')
          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
          expect(redis.get('vmpooler__tasks__clone')).to eq('1')
        end
      end

      it 'should expire the vm metadata' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:expire)
          end
          expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
        end
      end

      it 'should raise the error' do
        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/MockError/)
      end

    end

    context 'with request_id' do
      before(:each) do
        allow(metrics).to receive(:timing)
        allow(metrics).to receive(:gauge)
        expect(metrics).to receive(:gauge).with(/vmpooler_clone\./,/0/)
        expect(provider).to receive(:create_vm).with(pool, String)
        allow(provider).to receive(:get_vm_ip_address).with(vm,pool).and_return(1)
        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
        expect(dns_plugin).to receive(:create_or_replace_record)
        allow(logger).to receive(:log)
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:find_unique_hostname).with(pool).and_return(vm)
        end
      end

      it 'should set request_id and pool_alias on the vm data' do
        redis_connection_pool.with do |redis|
          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
          expect(redis.hget("vmpooler__vm__#{vm}", 'pool_alias')).to eq(pool)
          expect(redis.hget("vmpooler__vm__#{vm}", 'request_id')).to eq(request_id)
        end
      end

      it 'should reduce the clone count' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:decr).with('vmpooler__tasks__ondemandclone')
          subject._clone_vm(pool,provider,dns_plugin,request_id,pool)
        end
      end
    end

    context 'with #check_dns_available' do
      before(:each) do
        allow(logger).to receive(:log)
        allow(provider).to receive(:get_vm_ip_address).and_return(true)
        allow(subject).to receive(:get_dns_plugin_class_name_for_pool).and_return(dns_plugin)
      end
      it 'should error out if DNS already exists' do
        vm_name = "foo"
        resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
        expect(subject).to receive(:generate_and_check_hostname).exactly(3).times.and_return([vm_name, true]) #skip this, make it available all times
        expect(resolv).to receive(:getaddress).exactly(3).times.and_return("1.2.3.4")
        expect(metrics).to receive(:increment).with("vmpooler_errors.staledns.#{pool}").exactly(3).times
        expect{subject._clone_vm(pool,provider,dns_plugin)}.to raise_error(/Unable to generate a unique hostname after/)
      end
      it 'should be successful if DNS does not exist' do
        vm_name = "foo"
        resolv = class_double("Resolv").as_stubbed_const(:transfer_nested_constants => true)
        expect(subject).to receive(:generate_and_check_hostname).and_return([vm_name, true])
        expect(resolv).to receive(:getaddress).exactly(1).times.and_raise(Resolv::ResolvError)
        expect(provider).to receive(:get_vm_ip_address)
        expect(provider).to receive(:create_vm).with(pool, String)
        subject._clone_vm(pool,provider,dns_plugin)
      end
    end
  end

  describe '#destroy_vm' do
    before do
      expect(subject).not_to be_nil
      expect(Thread).to receive(:new).and_yield
    end

    it 'calls _destroy_vm' do
      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin)

      subject.destroy_vm(vm,pool,provider,dns_plugin)
    end

    it 'logs a message if an error is raised' do
      allow(logger).to receive(:log)
      expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: MockError")
      expect(subject).to receive(:_destroy_vm).with(vm,pool,provider,dns_plugin).and_raise('MockError')

      expect{subject.destroy_vm(vm,pool,provider,dns_plugin)}.to raise_error(/MockError/)
    end
  end

  describe "#_destroy_vm" do
    before(:each) do
      expect(subject).not_to be_nil

      redis_connection_pool.with do |redis|
        create_completed_vm(vm,pool,redis,true)
      end

      allow(provider).to receive(:destroy_vm).with(pool,vm).and_return(true)

      # Set redis configuration
      config[:redis] = {}
      config[:redis]['data_ttl'] = 168
    end

    context 'when redis data_ttl is not specified in the configuration' do
      before(:each) do
        config[:redis]['data_ttl'] = nil
      end

      it 'should call redis expire with 0' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:expire).with("vmpooler__vm__#{vm}", 0)
          end
          subject._destroy_vm(vm,pool,provider,dns_plugin)
        end
      end
    end

    context 'when there is no redis section in the configuration' do
      before(:each) do
        config[:redis] = nil
      end

      it 'should raise an error' do
        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(NoMethodError)
      end
    end

    context 'when a VM does not exist' do
      before(:each) do
        # As per base_spec, destroy_vm will return true if the VM does not exist
        expect(provider).to receive(:destroy_vm).with(pool,vm).and_return(true)
      end

      it 'should not raise an error' do
        subject._destroy_vm(vm,pool,provider,dns_plugin)
      end
    end

    context 'when the VM is destroyed without error' do
      it 'should log a message the VM was destroyed' do
        expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/)
        allow(logger).to receive(:log)

        subject._destroy_vm(vm,pool,provider,dns_plugin)
      end

      it 'should emit a timing metric' do
        allow(subject).to receive(:get_vm_usage_labels)
        allow(metrics).to receive(:timing)
        allow(metrics).to receive(:gauge)
        expect(metrics).to receive(:gauge).with("vmpooler_destroy.#{pool}", String)

        subject._destroy_vm(vm,pool,provider,dns_plugin)
      end

      it 'should dereference the mutex' do
        expect(subject).to receive(:dereference_mutex)

        subject._destroy_vm(vm,pool,provider,dns_plugin)
      end
    end

    context 'when the VM destruction raises an eror' do
      before(:each) do
        # As per base_spec, destroy_vm will return true if the VM does not exist
        expect(provider).to receive(:destroy_vm).with(pool,vm).and_raise('MockError')
      end

      it 'should not log a message the VM was destroyed' do
        expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/).exactly(0).times
        allow(logger).to receive(:log)

        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
      end

      it 'should not emit a timing metric' do
        expect(metrics).to receive(:timing).with("destroy.#{pool}", String).exactly(0).times

        expect{ subject._destroy_vm(vm,pool,provider,dns_plugin) }.to raise_error(/MockError/)
      end
    end

    context 'when the VM mutex is locked' do
      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject).to receive(:vm_mutex).with(vm).and_return(mutex)

        expect(subject._destroy_vm(vm,pool,provider,dns_plugin)).to eq(nil)
      end
    end
  end

  describe '#purge_unused_vms_and_resources' do
    let(:config) { YAML.load(<<-EOT
---
:config: {}
:providers:
  :mock: {}
:pools:
  - name: '#{pool}'
    size: 1
EOT
      )
    }

    it 'should return when purging is not enabled' do
      expect(subject.purge_unused_vms_and_resources).to be_nil
    end

    context 'with purging enabled globally' do
      before(:each) do
        config[:config]['purge_unconfigured_resources'] = true
        expect(Thread).to receive(:new).and_yield
      end

      it 'should run a purge for each provider' do
        expect(subject).to receive(:purge_vms_and_resources)

        subject.purge_unused_vms_and_resources
      end

      it 'should log when purging fails' do
        expect(subject).to receive(:purge_vms_and_resources).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')

        subject.purge_unused_vms_and_resources
      end
    end

    context 'with purging enabled on the provider' do
      before(:each) do
        config[:providers][:mock]['purge_unconfigured_resources'] = true
        expect(Thread).to receive(:new).and_yield
      end

      it 'should run a purge for the provider' do
        expect(subject).to receive(:purge_vms_and_resources)

        subject.purge_unused_vms_and_resources
      end
    end
  end

  describe '#purge_vms_and_resources' do
    let(:folder_name) { 'myinstance' }
    let(:folder_base) { 'vmpooler' }
    let(:datacenter) { 'dc1' }
    let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
    let(:folder) { [folder_base,folder_name].join('/') }
    let(:provider_name) { 'mock_provider' }
    let(:allowlist) { nil }
    let(:config) { YAML.load(<<-EOT
---
:config: {}
:providers:
  :mock_provider: {}
:pools:
  - name: '#{pool}'
    folder: '#{folder}'
    size: 1
    datacenter: '#{datacenter}'
    provider: '#{provider_name}'
EOT
      )
    }

    context 'when purging folders' do
      before do
        expect(subject).not_to be_nil
        # Inject mock provider into global variable - Note this is a code smell
        $providers = { provider_name => provider }
      end

      it 'should run purge_unconfigured_resources' do
        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist)
        expect(provider).to receive(:provider_config).and_return({}).twice

        subject.purge_vms_and_resources(provider_name)
      end

      it 'should raise any errors' do
        expect(provider).to receive(:purge_unconfigured_resources).with(allowlist).and_raise('mockerror')
        expect(provider).to receive(:provider_config).and_return({}).twice

        expect{ subject.purge_vms_and_resources(provider_name) }.to raise_error(RuntimeError, 'mockerror')
      end
    end
  end

  describe '#create_vm_disk' do
    let(:provider) { double('provider') }
    let(:disk_size) { 15 }

    before do
      expect(subject).not_to be_nil
    end

    it 'calls _create_vm_disk' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_create_vm_disk).with(pool, vm, disk_size, provider)

      subject.create_vm_disk(pool, vm, disk_size, provider)
    end
  end

  describe "#_create_vm_disk" do
    let(:disk_size) { '15' }

    before(:each) do
      expect(subject).not_to be_nil
      allow(logger).to receive(:log)

      redis_connection_pool.with do |redis|
        create_running_vm(pool,vm,redis,token)
      end
    end

    context 'Given a VM that does not exist' do
      before(:each) do
        # As per base_spec, create_disk will raise if the VM does not exist
        expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_raise("VM #{vm} does not exist")
      end

      it 'should not update redis if the VM does not exist' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hset).exactly(0).times
          expect{ subject._create_vm_disk(pool, vm, disk_size, provider) }.to raise_error(RuntimeError)
        end
      end
    end

    context 'Given an invalid disk size' do
      [{ :description => 'is nil',                            :value => nil },
       { :description => 'is an empty string',                :value => '' },
       { :description => 'is less than 1',                    :value => '0' },
       { :description => 'cannot be converted to an integer', :value => 'abc123' },
      ].each do |testcase|
        it "should not attempt the create the disk if the disk size #{testcase[:description]}" do
          expect(provider).to receive(:create_disk).exactly(0).times
          expect{ subject._create_vm_disk(pool, vm, testcase[:value], provider) }.to raise_error(/Invalid disk size/)
        end
      end

      it 'should raise an error if the disk size is a Fixnum' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hset).exactly(0).times
          expect{ subject._create_vm_disk(pool, vm, 10, provider) }.to raise_error(NoMethodError,/empty?/)
        end
      end
    end

    context 'Given a successful disk creation' do
      before(:each) do
        expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_return(true)
      end

      it 'should log a message' do
        expect(logger).to receive(:log).with('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk")
        expect(logger).to receive(:log).with('s', /\[\+\] \[disk_manager\] '#{vm}' attached #{disk_size}gb disk in 0.[\d]+ seconds/)

        subject._create_vm_disk(pool, vm, disk_size, provider)
      end

      it 'should update redis information when attaching the first disk' do
        redis_connection_pool.with do |redis|
          subject._create_vm_disk(pool, vm, disk_size, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("+#{disk_size}gb")
        end
      end

      it 'should update redis information when attaching the additional disks' do
        redis_connection_pool.with do |redis|
          initial_disks = '+10gb:+20gb'
          redis.hset("vmpooler__vm__#{vm}", 'disk', initial_disks)

          subject._create_vm_disk(pool, vm, disk_size, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("#{initial_disks}:+#{disk_size}gb")
        end
      end
    end

    context 'Given a failed disk creation' do
      before(:each) do
        expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_return(false)
      end

      it 'should not update redis information' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hset).exactly(0).times

          subject._create_vm_disk(pool, vm, disk_size, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to be_nil
        end
      end

      it 'should log a message' do
        expect(logger).to receive(:log).with('s', "[+] [disk_manager] '#{vm}' failed to attach disk")

        subject._create_vm_disk(pool, vm, disk_size, provider)
      end
    end
  end

  describe '#create_vm_snapshot' do
    let(:snapshot_name) { 'snapshot' }

    before do
      expect(subject).not_to be_nil
    end

    it 'calls _create_vm_snapshot' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_create_vm_snapshot).with(pool, vm, snapshot_name, provider)

      subject.create_vm_snapshot(pool, vm, snapshot_name, provider)
    end
  end

  describe '#_create_vm_snapshot' do
    let(:snapshot_name) { 'snapshot1' }

    before do
      expect(subject).not_to be_nil
    end

    before(:each) do
      redis_connection_pool.with do |redis|
        create_running_vm(pool,vm,redis,token)
      end
    end

    context 'Given a Pool that does not exist' do
      let(:missing_pool) { 'missing_pool' }

      before(:each) do
        expect(provider).to receive(:create_snapshot).with(missing_pool, vm, snapshot_name).and_raise("Pool #{missing_pool} not found")
      end

      it 'should not update redis' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
          expect{ subject._create_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found")
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
        end
      end
    end

    context 'Given a VM that does not exist' do
      let(:missing_vm) { 'missing_vm' }
      before(:each) do
        expect(provider).to receive(:create_snapshot).with(pool, missing_vm, snapshot_name).and_raise("VM #{missing_vm} not found")
      end

      it 'should not update redis' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
          expect{ subject._create_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found")
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
        end
      end
    end

    context 'Given a snapshot creation that succeeds' do
      before(:each) do
        expect(provider).to receive(:create_snapshot).with(pool, vm, snapshot_name).and_return(true)
      end

      it 'should log messages' do
        expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to snapshot #{vm} in pool #{pool}")
        expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' snapshot created in 0.[\d]+ seconds/)

        subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
      end

      it 'should add snapshot redis information' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
          subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to_not be_nil
        end
      end
    end

    context 'Given a snapshot creation that fails' do
      before(:each) do
        expect(provider).to receive(:create_snapshot).with(pool, vm, snapshot_name).and_return(false)
      end

      it 'should log messages' do
        expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to snapshot #{vm} in pool #{pool}")
        expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to snapshot '#{vm}'")

        subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
      end

      it 'should not update redis' do
        redis_connection_pool.with do |redis|
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
          subject._create_vm_snapshot(pool, vm, snapshot_name, provider)
          expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil
        end
      end
    end
  end

  describe '#revert_vm_snapshot' do
    let(:snapshot_name) { 'snapshot' }

    before do
      expect(subject).not_to be_nil
    end

    it 'calls _revert_vm_snapshot' do
      expect(Thread).to receive(:new).and_yield
      expect(subject).to receive(:_revert_vm_snapshot).with(pool, vm, snapshot_name, provider)

      subject.revert_vm_snapshot(pool, vm, snapshot_name, provider)
    end
  end

  describe '#_revert_vm_snapshot' do
    let(:snapshot_name) { 'snapshot1' }

    before do
      expect(subject).not_to be_nil
    end

    context 'Given a Pool that does not exist' do
      let(:missing_pool) { 'missing_pool' }

      before(:each) do
        expect(provider).to receive(:revert_snapshot).with(missing_pool, vm, snapshot_name).and_raise("Pool #{missing_pool} not found")
      end

      it 'should not log a result message' do
        expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/).exactly(0).times
        expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{vm}' in pool #{missing_pool} to snapshot '#{snapshot_name}'").exactly(0).times

        expect{ subject._revert_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found")
      end
    end

    context 'Given a VM that does not exist' do
      let(:missing_vm) { 'missing_vm' }
      before(:each) do
        expect(provider).to receive(:revert_snapshot).with(pool, missing_vm, snapshot_name).and_raise("VM #{missing_vm} not found")
      end

      it 'should not log a result message' do
        expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{missing_vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/).exactly(0).times
        expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{missing_vm}' in pool #{pool} to snapshot '#{snapshot_name}'").exactly(0).times

        expect{ subject._revert_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found")
      end
    end

    context 'Given a snapshot revert that succeeds' do
      before(:each) do
        expect(provider).to receive(:revert_snapshot).with(pool, vm, snapshot_name).and_return(true)
      end

      it 'should log success messages' do
        expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'")
        expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/)

        subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)
      end

      it 'should return true' do
        expect(subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)).to be true
      end
    end

    context 'Given a snapshot creation that fails' do
      before(:each) do
        expect(provider).to receive(:revert_snapshot).with(pool, vm, snapshot_name).and_return(false)
      end

      it 'should log failure messages' do
        expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'")
        expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'")

        subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)
      end

      it 'should return false' do
        expect(subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)).to be false
      end
    end
  end

  describe '#get_pool_name_for_vm' do
    context 'Given a valid VM' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_running_vm(pool, vm, redis, token)
        end
      end

      it 'should return the pool name' do
        redis_connection_pool.with do |redis|
          expect(subject.get_pool_name_for_vm(vm,redis)).to eq(pool)
        end
      end
    end

    context 'Given an invalid VM' do
      it 'should return nil' do
        redis_connection_pool.with do |redis|
          expect(subject.get_pool_name_for_vm('does_not_exist',redis)).to be_nil
        end
      end
    end
  end

  describe '#get_provider_for_pool' do
    let(:provider_name) { 'mock_provider' }

    before do
      expect(subject).not_to be_nil
      # Inject mock provider into global variable - Note this is a code smell
      $providers = { provider_name => provider }
    end

    after(:each) do
      # Reset the global variable - Note this is a code smell
      $providers = nil
    end

    context 'Given a pool name which does not exist' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
EOT
      )}

      it 'should return nil' do
        expect(subject.get_provider_for_pool('pool_does_not_exist')).to be_nil
      end
    end

    context 'Given a pool which does not have a provider' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
EOT
      )}

      it 'should return nil' do
        expect(subject.get_provider_for_pool(pool)).to be_nil
      end
    end

    context 'Given a pool which uses an invalid provider' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    provider: 'does_not_exist'
EOT
      )}

      it 'should return nil' do
        expect(subject.get_provider_for_pool(pool)).to be_nil
      end
    end

    context 'Given a pool which uses a valid provider' do
      let(:config) { YAML.load(<<-EOT
---
:config:
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    provider: #{provider_name}
EOT
      )}

      it 'should return a provider object' do
        result = subject.get_provider_for_pool(pool)
        expect(result).to_not be_nil
        expect(result.name).to eq(provider_name)
      end
    end
  end

  describe '#get_dns_plugin_class_name_for_pool' do
    let(:config) { YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT

    )}
    before(:each) do
      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
    end

    it 'calls Vmpooler::Dns.get_dns_plugin_class_by_name' do
      expect(Vmpooler::Dns).to receive(:get_dns_plugin_class_by_name).with(config, 'mock')

      subject.get_dns_plugin_class_name_for_pool(pool)
    end
  end

  describe '#get_dns_plugin_domain_for_pool' do
    let(:config) { YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
  )}
    before(:each) do
      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
    end

    it 'calls Vmpooler::Dns.get_dns_plugin_domain_for_pool' do
      expect(Vmpooler::Dns).to receive(:get_dns_plugin_domain_by_name).with(config, 'mock')

      subject.get_dns_plugin_domain_for_pool(pool)
    end
  end

  describe '#check_disk_queue' do
    let(:threads) {[]}

    before(:each) do
      expect(Thread).to receive(:new).and_yield
      allow(subject).to receive(:_check_disk_queue)
    end

    it 'should log the disk manager is starting' do
      expect(logger).to receive(:log).with('d', "[*] [disk_manager] starting worker thread")

      expect($threads.count).to be(0)
      subject.check_disk_queue(1,0)
      expect($threads.count).to be(1)
    end

    it 'should add the manager to the global thread list' do
      # Note - Ruby core types are not necessarily thread safe
      expect($threads.count).to be(0)
      subject.check_disk_queue(1,0)
      expect($threads.count).to be(1)
    end

    it 'should call _check_disk_queue' do
      expect(subject).to receive(:_check_disk_queue).with(no_args)

      subject.check_disk_queue(1,0)
    end

    context 'delays between loops' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }
      # Note a maxloop of zero can not be tested as it never terminates

      it 'defaults to 5 second loop delay' do
        expect(subject).to receive(:sleep).with(5).exactly(maxloop).times
        subject.check_disk_queue(maxloop)
      end

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times

        subject.check_disk_queue(maxloop,loop_delay)
      end
    end

    context 'loops specified number of times (5)' do
      let(:maxloop) { 5 }
      # Note a maxloop of zero can not be tested as it never terminates

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      it 'should call _check_disk_queue 5 times' do
        expect(subject).to receive(:_check_disk_queue).with(no_args).exactly(maxloop).times

        subject.check_disk_queue(maxloop,0)
      end
    end
  end

  describe '#_check_disk_queue' do
    before do
      expect(subject).not_to be_nil
    end

    context 'when no VMs in the queue' do
      it 'should not call create_vm_disk' do
        expect(subject).to receive(:create_vm_disk).exactly(0).times
        subject._check_disk_queue
      end
    end

    context 'when VM in the queue does not exist' do
      before(:each) do
        redis_connection_pool.with do |redis|
          disk_task_vm(vm,"snapshot_#{vm}",redis)
        end
      end

      it 'should log an error' do
        expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/)

        subject._check_disk_queue
      end

      it 'should not call create_vm_disk' do
        expect(subject).to receive(:create_vm_disk).exactly(0).times

        subject._check_disk_queue
      end
    end

    context 'when specified provider does not exist' do
      before(:each) do
        redis_connection_pool.with do |redis|
          disk_task_vm(vm,"snapshot_#{vm}",redis)
          create_running_vm(pool, vm, redis, token)
          expect(subject).to receive(:get_provider_for_pool).and_return(nil)
        end
      end

      it 'should log an error' do
        expect(logger).to receive(:log).with('s', /Missing Provider for/)

        subject._check_disk_queue
      end

      it 'should not call create_vm_disk' do
        expect(subject).to receive(:create_vm_disk).exactly(0).times

        subject._check_disk_queue
      end
    end

    context 'when multiple VMs in the queue' do
      before(:each) do
        ['vm1', 'vm2', 'vm3'].each do |vm_name|
          redis_connection_pool.with do |redis|
            disk_task_vm(vm_name,"snapshot_#{vm_name}",redis)
            create_running_vm(pool, vm_name, redis, token)
          end
        end

        allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
      end

      it 'should call create_vm_disk once' do
        expect(subject).to receive(:create_vm_disk).exactly(1).times
        subject._check_disk_queue
      end

      it 'should create the disk for the first VM in the queue' do
        expect(subject).to receive(:create_vm_disk).with(pool,'vm1','snapshot_vm1',provider)
        subject._check_disk_queue
      end

      it 'should log an error if one occurs' do
        expect(subject).to receive(:create_vm_disk).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('s', "[!] [disk_manager] disk creation appears to have failed: MockError")
        subject._check_disk_queue
      end
    end
  end

  describe '#check_snapshot_queue' do
    let(:threads) {[]}

    before(:each) do
      expect(Thread).to receive(:new).and_yield
      allow(Vmpooler::Dns).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
      allow(subject).to receive(:_check_snapshot_queue).with(no_args)
    end

    it 'should log the snapshot manager is starting' do
      expect(logger).to receive(:log).with('d', "[*] [snapshot_manager] starting worker thread")

      expect($threads.count).to be(0)
      subject.check_snapshot_queue(1,0)
      expect($threads.count).to be(1)
    end

    it 'should add the manager to the global thread list' do
      # Note - Ruby core types are not necessarily thread safe
      expect($threads.count).to be(0)
      subject.check_snapshot_queue(1,0)
      expect($threads.count).to be(1)
    end

    it 'should call _check_snapshot_queue' do
      expect(subject).to receive(:_check_snapshot_queue).with(no_args)

      subject.check_snapshot_queue(1,0)
    end

    context 'delays between loops' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }
      # Note a maxloop of zero can not be tested as it never terminates

      it 'defaults to 5 second loop delay' do
        expect(subject).to receive(:sleep).with(5).exactly(maxloop).times
        subject.check_snapshot_queue(maxloop)
      end

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times

        subject.check_snapshot_queue(maxloop,loop_delay)
      end
    end

    context 'loops specified number of times (5)' do
      let(:maxloop) { 5 }
      # Note a maxloop of zero can not be tested as it never terminates

      before(:each) do
        allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      it 'should call _check_snapshot_queue 5 times' do
        expect(subject).to receive(:_check_snapshot_queue).with(no_args).exactly(maxloop).times

        subject.check_snapshot_queue(maxloop,0)
      end
    end
  end

  describe '#_check_snapshot_queue' do
    before do
      expect(subject).not_to be_nil
    end

    context 'vmpooler__tasks__snapshot queue' do
      context 'when no VMs in the queue' do
        it 'should not call create_vm_snapshot' do
          expect(subject).to receive(:create_vm_snapshot).exactly(0).times
          subject._check_snapshot_queue
        end
      end

      context 'when VM in the queue does not exist' do
        before(:each) do
          redis_connection_pool.with do |redis|
            snapshot_vm(vm,"snapshot_#{vm}",redis)
          end
        end

        it 'should log an error' do
          expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/)

          subject._check_snapshot_queue
        end

        it 'should not call create_vm_snapshot' do
          expect(subject).to receive(:create_vm_snapshot).exactly(0).times

          subject._check_snapshot_queue
        end
      end

      context 'when specified provider does not exist' do
        before(:each) do
          redis_connection_pool.with do |redis|
            snapshot_vm(vm,"snapshot_#{vm}",redis)
            create_running_vm(pool, vm, redis, token)
            expect(subject).to receive(:get_provider_for_pool).and_return(nil)
          end
        end

        it 'should log an error' do
          expect(logger).to receive(:log).with('s', /Missing Provider for/)

          subject._check_snapshot_queue
        end

        it 'should not call create_vm_snapshot' do
          expect(subject).to receive(:create_vm_snapshot).exactly(0).times

          subject._check_snapshot_queue
        end
      end

      context 'when multiple VMs in the queue' do
        before(:each) do
          ['vm1', 'vm2', 'vm3'].each do |vm_name|
            redis_connection_pool.with do |redis|
              snapshot_vm(vm_name,"snapshot_#{vm_name}",redis)
              create_running_vm(pool, vm_name, redis, token)
            end
          end

          allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
        end

        it 'should call create_vm_snapshot once' do
          expect(subject).to receive(:create_vm_snapshot).exactly(1).times
          subject._check_snapshot_queue
        end

        it 'should snapshot the first VM in the queue' do
          expect(subject).to receive(:create_vm_snapshot).with(pool,'vm1','snapshot_vm1',provider)
          subject._check_snapshot_queue
        end

        it 'should log an error if one occurs' do
          expect(subject).to receive(:create_vm_snapshot).and_raise(RuntimeError,'MockError')
          expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot create appears to have failed: MockError")
          subject._check_snapshot_queue
        end
      end
    end

    context 'vmpooler__tasks__snapshot-revert queue' do
      context 'when no VMs in the queue' do
        it 'should not call revert_vm_snapshot' do
          expect(subject).to receive(:revert_vm_snapshot).exactly(0).times
          subject._check_snapshot_queue
        end
      end

      context 'when VM in the queue does not exist' do
        before(:each) do
          redis_connection_pool.with do |redis|
            snapshot_revert_vm(vm,"snapshot_#{vm}",redis)
          end
        end

        it 'should log an error' do
          expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/)

          subject._check_snapshot_queue
        end

        it 'should not call revert_vm_snapshot' do
          expect(subject).to receive(:revert_vm_snapshot).exactly(0).times

          subject._check_snapshot_queue
        end
      end

      context 'when specified provider does not exist' do
        before(:each) do
          redis_connection_pool.with do |redis|
            snapshot_revert_vm(vm,"snapshot_#{vm}",redis)
            create_running_vm(pool, vm, redis, token)
            expect(subject).to receive(:get_provider_for_pool).and_return(nil)
          end
        end

        it 'should log an error' do
          expect(logger).to receive(:log).with('s', /Missing Provider for/)

          subject._check_snapshot_queue
        end

        it 'should not call revert_vm_snapshot' do
          expect(subject).to receive(:revert_vm_snapshot).exactly(0).times

          subject._check_snapshot_queue
        end
      end

      context 'when multiple VMs in the queue' do
        before(:each) do
          ['vm1', 'vm2', 'vm3'].each do |vm_name|
            redis_connection_pool.with do |redis|
              snapshot_revert_vm(vm_name,"snapshot_#{vm_name}",redis)
              create_running_vm(pool, vm_name, redis, token)
            end
          end

          allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
        end

        it 'should call revert_vm_snapshot once' do
          expect(subject).to receive(:revert_vm_snapshot).exactly(1).times
          subject._check_snapshot_queue
        end

        it 'should snapshot the first VM in the queue' do
          expect(subject).to receive(:revert_vm_snapshot).with(pool,'vm1','snapshot_vm1',provider)
          subject._check_snapshot_queue
        end

        it 'should log an error if one occurs' do
          expect(subject).to receive(:revert_vm_snapshot).and_raise(RuntimeError,'MockError')
          expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot revert appears to have failed: MockError")
          subject._check_snapshot_queue
        end
      end
    end
  end

  describe '#migrate_vm' do
    before(:each) do
      expect(subject).not_to be_nil
      expect(Thread).to receive(:new).and_yield
    end

    it 'calls migrate_vm' do
      expect(provider).to receive(:migrate_vm).with(pool, vm)

      subject.migrate_vm(vm, pool, provider)
    end

    context 'When an error is raised' do
      before(:each) do
        expect(provider).to receive(:migrate_vm).with(pool, vm).and_raise('MockError')
      end

      it 'logs a message' do
        allow(logger).to receive(:log)
        expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm}' migration failed with an error: MockError")

        subject.migrate_vm(vm, pool, provider)
      end
    end

    context 'with a locked vm mutex' do
      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject).to receive(:vm_mutex).and_return(mutex)

        expect(subject.migrate_vm(vm, pool, provider)).to be_nil
      end
    end
  end

  describe '#vm_mutex' do
    it 'should return a mutex' do
      expect(subject.vm_mutex(vm)).to be_a(Mutex)
    end

    it 'should return the same mutex when called twice' do
      first = subject.vm_mutex(vm)
      second = subject.vm_mutex(vm)
      expect(first).to be(second)
    end
  end

  describe '#dereference_mutex' do
    it 'should return nil when no mutex is dereferenced' do
      expect(subject.dereference_mutex(vm)).to be_nil
    end

    it 'should return true when a mutex is dereferenced' do
      subject.vm_mutex(vm)
      expect(subject.dereference_mutex(vm)).to be true
    end

    it 'should dereference the mutex' do
      mutex = subject.vm_mutex(vm)

      subject.dereference_mutex(vm)

      result = subject.vm_mutex(vm)
      expect(result).to_not eq(mutex)
    end
  end

  describe 'sync_pool_template' do
    let(:old_template) { 'templates/old-template' }
    let(:new_template) { 'templates/new-template' }
    let(:config) { YAML.load(<<-EOT
---
:pools:
  - name: '#{pool}'
    size: 1
    template: old_template
EOT
      )
    }

    it 'returns when a template is not set in redis' do
      expect(subject.sync_pool_template(config[:pools][0])).to be_nil
    end

    it 'returns when a template is set and matches the configured template' do
      redis_connection_pool.with do |redis|
        redis.hset('vmpooler__config__template', pool, old_template)
      end
      subject.sync_pool_template(config[:pools][0])

      expect(config[:pools][0]['template']).to eq(old_template)
    end

    it 'updates a pool template when the redis provided value is different' do
      redis_connection_pool.with do |redis|
        redis.hset('vmpooler__config__template', pool, new_template)
      end

      subject.sync_pool_template(config[:pools][0])

      expect(config[:pools][0]['template']).to eq(new_template)
    end
  end

  describe 'pool_mutex' do
    it 'should return a mutex' do
      expect(subject.pool_mutex(pool)).to be_a(Mutex)
    end

    it 'should return the same mutex when called twice' do
      first = subject.pool_mutex(pool)
      second = subject.pool_mutex(pool)
      expect(first).to be(second)
    end
  end

  describe 'update_pool_template' do
    let(:current_template) { 'templates/pool_template' }
    let(:new_template) { 'templates/new_pool_template' }
    let(:config) {
      YAML.load(<<-EOT
---
:config: {}
:pools:
  - name: #{pool}
    template: "#{current_template}"
EOT
      )
    }
    let(:poolconfig) { config[:pools][0] }

    before(:each) do
      allow(logger).to receive(:log)
    end

    it 'should set the pool template to match the configured template' do
      redis_connection_pool.with do |redis|
        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end

      expect(poolconfig['template']).to eq(new_template)
    end

    it 'should log that the template is updated' do
      expect(logger).to receive(:log).with('s', "[*] [#{pool}] template updated from #{current_template} to #{new_template}")

      redis_connection_pool.with do |redis|
        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end
    end

    it 'should run drain_pool' do
      redis_connection_pool.with do |redis|
        expect(subject).to receive(:drain_pool).with(pool, redis)

        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end
    end

    it 'should log that a template is being prepared' do
      expect(logger).to receive(:log).with('s', "[*] [#{pool}] preparing pool template for deployment")

      redis_connection_pool.with do |redis|
        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end
    end

    it 'should run prepare_template' do
      redis_connection_pool.with do |redis|
        expect(subject).to receive(:prepare_template).with(poolconfig, provider, redis)

        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end
    end

    it 'should log that the pool is ready for use' do
      expect(logger).to receive(:log).with('s', "[*] [#{pool}] is ready for use")

      redis_connection_pool.with do |redis|
        subject.update_pool_template(poolconfig, provider, new_template, current_template, redis)
      end
    end
  end

  describe '#remove_excess_vms' do
    let(:config) {
      YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    size: 2
EOT
      )
    }

    before(:each) do
      expect(subject).not_to be_nil
    end

    context 'with a 0 total value' do
      let(:ready) { 0 }
      let(:total) { 0 }
      it 'should return nil' do
        expect(subject.remove_excess_vms(config[:pools][0])).to be_nil
      end
    end

    context 'when the mutex is locked' do
      let(:mutex) { Mutex.new }
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool,'vm1',redis)
          create_ready_vm(pool,'vm2',redis)
          create_pending_vm(pool,'vm3',redis)
        end
        mutex.lock
        expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
      end

      it 'should return nil' do
        expect(subject.remove_excess_vms(config[:pools][0])).to be_nil
      end
    end

    context 'with a total size less than the pool size' do
      it 'should return nil' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
        end
        expect(subject.remove_excess_vms(config[:pools][0])).to be_nil
      end
    end

    context 'with a total size greater than the pool size' do
      it 'should remove excess ready vms' do
        redis_connection_pool.with do |redis|
          ['vm1','vm2','vm3','vm4'].each do |v|
            create_ready_vm(pool,v,redis)
          end
        end
        expect(subject).to receive(:move_vm_queue).exactly(2).times

        subject.remove_excess_vms(config[:pools][0])
      end

      it 'should remove excess pending vms' do
        redis_connection_pool.with do |redis|
          create_pending_vm(pool,'vm1',redis)
          create_pending_vm(pool,'vm2',redis)
          create_ready_vm(pool,'vm3',redis)
          create_ready_vm(pool,'vm4',redis)
          create_ready_vm(pool,'vm5',redis)
          expect(subject).to receive(:move_vm_queue).exactly(3).times
        end

        subject.remove_excess_vms(config[:pools][0])
      end
    end
  end

  describe 'prepare_template' do
    let(:config) { YAML.load(<<-EOT
---
:config:
  create_template_delta_disks: true
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    template: 'templates/pool1'
EOT
      )
    }

    context 'when creating the template delta disks' do
      before(:each) do
        redis_connection_pool.with do |redis|
          allow(redis).to receive(:hset)
        end
        allow(provider).to receive(:create_template_delta_disks)
      end

      it 'should run create template delta disks' do
        redis_connection_pool.with do |redis|
          expect(provider).to receive(:create_template_delta_disks).with(config[:pools][0])

          subject.prepare_template(config[:pools][0], provider, redis)
        end
      end

      it 'should mark the template as prepared' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hset).with('vmpooler__template__prepared', pool, config[:pools][0]['template'])

          subject.prepare_template(config[:pools][0], provider, redis)
        end
      end
    end

    context 'when template delta disk creation fails' do
      before(:each) do
        redis_connection_pool.with do |redis|
          allow(redis).to receive(:hset)
        end
        expect(provider).to receive(:create_template_delta_disks).and_raise("MockError")
      end

      it 'should log a message when delta disk creation returns an error' do
        redis_connection_pool.with do |redis|
          expect(logger).to receive(:log).with('s', "[!] [#{pool}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: MockError")

          subject.prepare_template(config[:pools][0], provider, redis)
        end
      end
    end
  end

  describe 'evaluate_template' do
    let(:mutex) { Mutex.new }
    let(:current_template) { 'templates/template1' }
    let(:new_template) { 'templates/template2' }
    let(:config) { YAML.load(<<-EOT
---
:config:
  task_limit: 5
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: 1
    template: '#{current_template}'
EOT
      )
    }

    before(:each) do
      redis_connection_pool.with do |redis|
        allow(redis).to receive(:hget)
      end
      expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
    end

    it 'should retreive the prepared template' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
      end

      subject.evaluate_template(config[:pools][0], provider)
    end

    it 'should retrieve the redis configured template' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
      end

      subject.evaluate_template(config[:pools][0], provider)
    end

    context 'when the mutex is locked' do
      before(:each) do
        mutex.lock
      end

      it 'should return' do
        expect(subject.evaluate_template(config[:pools][0], provider)).to be_nil
      end
    end

    context 'when prepared template is nil' do

      it 'should prepare the template' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(nil)
          expect(subject).to receive(:prepare_template).with(config[:pools][0], provider, redis)
        end

        subject.evaluate_template(config[:pools][0], provider)
      end

      it 'should not prepare the template again' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
        end
        expect(subject).to_not receive(:prepare_template).with(config[:pools][0], provider)

        subject.evaluate_template(config[:pools][0], provider)
      end
    end

    context 'when the configured pool template does not match the prepared template' do
      before(:each) do
        redis_connection_pool.with do |redis|
          config[:pools][0]['template'] = new_template
          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
        end
      end

      it 'should prepare the template' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:prepare_template).with(config[:pools][0], provider, redis)
        end

        subject.evaluate_template(config[:pools][0], provider)
      end

      context 'if configured_template is provided' do
        it 'should not run prepare_template' do
          redis_connection_pool.with do |redis|
            expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(current_template)
          end
          expect(subject).to_not receive(:prepare_template)

          subject.evaluate_template(config[:pools][0], provider)
        end
      end
    end

    context 'when a new template is requested' do
      before(:each) do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).and_return(current_template)
          expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(new_template)
        end
      end

      it 'should update the template' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, new_template, current_template, redis)
        end

        subject.evaluate_template(config[:pools][0], provider)
      end
    end
  end

  describe 'drain_pool' do
    before(:each) do
      allow(logger).to receive(:log)
    end

    context 'with no vms' do
      it 'should return nil' do
        redis_connection_pool.with do |redis|
          expect(subject.drain_pool(pool, redis)).to be_nil
        end
      end

      it 'should not log any messages' do
        expect(logger).to_not receive(:log)

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end

      it 'should not try to move any vms' do
        expect(subject).to_not receive(:move_vm_queue)

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end
    end

    context 'with ready vms' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool, 'vm1', redis)
          create_ready_vm(pool, 'vm2', redis)
        end
      end

      it 'removes the ready instances' do
        expect(subject).to receive(:move_vm_queue).twice

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end

      it 'logs that ready instances are being removed' do
        expect(logger).to receive(:log).with('s', "[*] [#{pool}] removing ready instances")

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end
    end

    context 'with pending instances' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_pending_vm(pool, 'vm1', redis)
          create_pending_vm(pool, 'vm2', redis)
        end
      end

      it 'removes the pending instances' do
        expect(subject).to receive(:move_vm_queue).twice

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end

      it 'logs that pending instances are being removed' do
        expect(logger).to receive(:log).with('s', "[*] [#{pool}] removing pending instances")

        redis_connection_pool.with do |redis|
          subject.drain_pool(pool, redis)
        end
      end
    end
  end

  describe 'update_pool_size' do
    let(:newsize) { '3' }
    let(:config) {
      YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    size: 2
EOT
      )
    }
    let(:poolconfig) { config[:pools][0] }

    context 'with a locked mutex' do

      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
        expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
      end

      it 'should return nil' do
        expect(subject.update_pool_size(poolconfig)).to be_nil
      end
    end

    it 'should get the pool size configuration from redis' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool)
      end

      subject.update_pool_size(poolconfig)
    end

    it 'should return when poolsize is not set in redis' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(nil)
      end

      expect(subject.update_pool_size(poolconfig)).to be_nil
    end

    it 'should return when no change in configuration is required' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return('2')
      end

      expect(subject.update_pool_size(poolconfig)).to be_nil
    end

    it 'should update the pool size' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__poolsize', pool).and_return(newsize)
      end

      subject.update_pool_size(poolconfig)

      expect(poolconfig['size']).to eq(Integer(newsize))
    end
  end

  describe 'update_clone_target' do
    let(:newtarget) { 'cluster2' }
    let(:config) {
      YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    clone_target: 'cluster1'
EOT
      )
    }
    let(:poolconfig) { config[:pools][0] }

    context 'with a locked mutex' do

      let(:mutex) { Mutex.new }
      before(:each) do
        mutex.lock
        expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
      end

      it 'should return nil' do
        expect(subject.update_clone_target(poolconfig)).to be_nil
      end
    end

    it 'should get the pool clone target configuration from redis' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool)
      end

      subject.update_clone_target(poolconfig)
    end

    it 'should return when clone_target is not set in redis' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(nil)
      end

      expect(subject.update_clone_target(poolconfig)).to be_nil
    end

    it 'should return when no change in configuration is required' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return('cluster1')
      end

      expect(subject.update_clone_target(poolconfig)).to be_nil
    end

    it 'should update the clone target' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:hget).with('vmpooler__config__clone_target', pool).and_return(newtarget)
      end

      subject.update_clone_target(poolconfig)

      expect(poolconfig['clone_target']).to eq(newtarget)
    end
  end

  describe "#execute!" do
    let(:config) {
      YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
EOT
      )
    }

    before(:each) do
      expect(subject).not_to be_nil

      allow(subject).to receive(:check_disk_queue)
      allow(subject).to receive(:check_snapshot_queue)
      allow(subject).to receive(:check_pool)
      allow(subject).to receive(:check_ondemand_requests)
      allow(subject).to receive(:get_domain_for_pool).and_return('example.com')

      allow(logger).to receive(:log)
    end

    after(:each) do
      # Reset the global variable - Note this is a code smell
      $threads = nil
    end

    context 'on startup' do
      let(:config) {
        YAML.load(<<-EOT
---
:config:
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
        )}

      before(:each) do
        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
      end

      it 'should log a message that VMPooler has started' do
        expect(logger).to receive(:log).with('d', 'starting vmpooler')

        subject.execute!(1,0)
      end

      it 'should set clone tasks to zero' do
        redis_connection_pool.with do |redis|
          redis.set('vmpooler__tasks__clone', 1)
          subject.execute!(1,0)
          expect(redis.get('vmpooler__tasks__clone')).to eq('0')
        end
      end

      it 'should clear migration tasks' do
        redis_connection_pool.with do |redis|
          redis.set('vmpooler__migration', 1)
          subject.execute!(1,0)
          expect(redis.get('vmpooler__migration')).to be_nil
        end
      end

      it 'should run the check_disk_queue method' do
        expect(subject).to receive(:check_disk_queue)

        subject.execute!(1,0)
      end

      it 'should run the check_snapshot_queue method' do
        expect(subject).to receive(:check_snapshot_queue)

        subject.execute!(1,0)
      end

      it 'should check the pools in the config' do
        expect(subject).to receive(:check_pool).with(a_pool_with_name_of(pool))

        subject.execute!(1,0)
      end

      it 'should run the check_ondemand_requests method' do
        expect(subject).to receive(:check_ondemand_requests)

        subject.execute!(1,0)
      end

      context 'creating Dns plugins' do
        let(:mock_dns_plugin) { double('mock_dns_plugin') }
        let(:config) {
        YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
  - name: 'dummy'
    dns_plugin: 'mock'
  - name: 'dummy2'
    dns_plugin: 'mock'
EOT
        )}

        it 'should call create_dns_object idempotently' do
          # Even though there are two pools using the mock dns plugin, it should only
          # create the dns object once.
          expect(subject).to receive(:create_dns_object).and_return(mock_dns_plugin)

          subject.execute!(1,0)
        end

        it 'should raise an error if the dns plugin cannot be created' do
          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")

          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
        end

        it 'should log a message if the dns plugin cannot be created' do
          expect(subject).to receive(:create_dns_object).and_raise(RuntimeError, "MockError")
          expect(logger).to receive(:log).with('s',"Error while creating dns plugin for pool #{pool}: MockError")

          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
        end
      end

      context 'creating Providers' do
        let(:vsphere_provider) { double('vsphere_provider') }
        let(:config) {
        YAML.load(<<-EOT
---
:providers:
  :vsphere: {}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    provider: 'vsphere'
    dns_plugin: 'mock'
  - name: 'dummy'
    provider: 'vsphere'
    dns_plugin: 'mock'
EOT
        )}

        before(:each) do
          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
        end

        it 'should call create_provider_object idempotently' do
          # Even though there are two pools using the vsphere provider, it should only
          # create the provider object once.
          expect(subject).to receive(:create_provider_object).with(Object, Object, Object, redis_connection_pool, 'vsphere', 'vsphere', Object).and_return(vsphere_provider)

          subject.execute!(1,0)
        end

        it 'should raise an error if the provider can not be created' do
          expect(subject).to receive(:create_provider_object).and_raise(RuntimeError, "MockError")

          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
        end

        it 'should log a message if the provider can not be created' do
          expect(subject).to receive(:create_provider_object).and_raise(RuntimeError, "MockError")
          expect(logger).to receive(:log).with('s',"Error while creating provider for pool #{pool}: MockError")

          expect{ subject.execute!(1,0) }.to raise_error(/MockError/)
        end
      end
    end

    context 'modify configuration on startup' do
      context 'move vSphere configuration to providers location' do
        let(:config) {
        YAML.load(<<-EOT
:dns_configs:
  :mock:
    dns_class: base
:vsphere:
  server: 'vsphere.example.com'
  username: 'vmpooler'
  password: 'password'
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
        )}
        before(:each) do
          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
        end

        it 'should set providers with no provider to vsphere' do
          expect(subject.config[:providers]).to be nil

          subject.execute!(1,0)
          expect(subject.config[:providers][:vsphere]['server']).to eq('vsphere.example.com')
          expect(subject.config[:providers][:vsphere]['username']).to eq('vmpooler')
          expect(subject.config[:providers][:vsphere]['password']).to eq('password')
        end

        it 'should log a message' do
          expect(logger).to receive(:log).with('d', "[!] Detected an older configuration file. Copying the settings from ':vsphere:' to ':providers:/:vsphere:'")

          subject.execute!(1,0)
        end
      end


      context 'default to the dummy provider' do
        let(:config) {
        YAML.load(<<-EOT
---
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
  - name: 'dummy'
    provider: 'dummy'
    dns_plugin: 'mock'
EOT
        )}

        before(:each) do
          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
        end

        it 'should set providers with no provider to dummy' do
          expect(subject.config[:pools][0]['provider']).to be_nil
          expect(subject.config[:pools][1]['provider']).to eq('dummy')

          subject.execute!(1,0)

          expect(subject.config[:pools][0]['provider']).to eq('dummy')
          expect(subject.config[:pools][1]['provider']).to eq('dummy')
        end

        it 'should log a message' do
          expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'dummy' as default")

          subject.execute!(1,0)
        end
      end
    end

    context 'with dead disk_manager thread' do
      let(:disk_manager_thread) { double('thread', :alive? => false) }
      let(:check_loop_delay_min) { 7 }
      let(:check_loop_delay_max) { 20 }
      let(:check_loop_delay_decay) { 2.1 }
      let(:config) {
      YAML.load(<<-EOT
---
:config:
  check_loop_delay_min: #{check_loop_delay_min}
  check_loop_delay_max: #{check_loop_delay_max}
  check_loop_delay_decay: #{check_loop_delay_decay}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }

      before(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = {}
        $threads['disk_manager'] = disk_manager_thread
        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
      end

      it 'should run the check_disk_queue method and log a message' do
        expect(subject).to receive(:check_disk_queue)
        expect(logger).to receive(:log).with('d', "[!] [disk_manager] worker thread died, restarting")

        subject.execute!(1,0)
      end
    end

    context 'with dead snapshot_manager thread' do
      let(:snapshot_manager_thread) { double('thread', :alive? => false) }
      let(:check_loop_delay_min) { 7 }
      let(:check_loop_delay_max) { 20 }
      let(:check_loop_delay_decay) { 2.1 }
      let(:config) {
      YAML.load(<<-EOT
---
:config:
  check_loop_delay_min: #{check_loop_delay_min}
  check_loop_delay_max: #{check_loop_delay_max}
  check_loop_delay_decay: #{check_loop_delay_decay}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
      before(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = {}
        $threads['snapshot_manager'] = snapshot_manager_thread
        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
      end

      it 'should run the check_snapshot_queue method and log a message' do
        expect(subject).to receive(:check_snapshot_queue)
        expect(logger).to receive(:log).with('d', "[!] [snapshot_manager] worker thread died, restarting")
        $threads['snapshot_manager'] = snapshot_manager_thread

        subject.execute!(1,0)
      end
    end

    context 'with dead pool thread' do
      context 'without check_loop_delay_xxx settings' do
        let(:pool_thread) { double('thread', :alive? => false) }
        let(:default_check_loop_delay_min) { 5 }
        let(:default_check_loop_delay_max) { 60 }
        let(:default_check_loop_delay_decay) { 2.0 }
        let(:check_loop_delay_min) { 7 }
        let(:check_loop_delay_max) { 20 }
        let(:check_loop_delay_decay) { 2.1 }
        let(:config) {
      YAML.load(<<-EOT
---
:config:
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
        before(:each) do
          # Reset the global variable - Note this is a code smell
          $threads = {}
          $threads[pool] = pool_thread
          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
        end

        it 'should run the check_pool method and log a message' do
          expect(subject).to receive(:check_pool).with(a_pool_with_name_of(pool),
                                                       default_check_loop_delay_min,
                                                       default_check_loop_delay_max,
                                                       default_check_loop_delay_decay)
          expect(logger).to receive(:log).with('d', "[!] [#{pool}] worker thread died, restarting")

          subject.execute!(1,0)
        end
      end

      context 'with dead ondemand provisioner thread' do
        let(:ondemand_provisioner_thread) { double('thread', :alive? => false) }
        let(:default_check_loop_delay_min) { 5 }
        let(:default_check_loop_delay_max) { 60 }
        let(:default_check_loop_delay_decay) { 2.0 }
        let(:config) {
      YAML.load(<<-EOT
---
:config:
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
        before(:each) do
          # Reset the global variable - Note this is a code smell
          $threads = {}
          $threads['ondemand_provisioner'] = ondemand_provisioner_thread
        end

        it 'should run the process_ondemand_requests method' do
          expect(subject).to receive(:check_ondemand_requests).with(default_check_loop_delay_min, default_check_loop_delay_max, default_check_loop_delay_decay)
          subject.execute!(1,0)
        end

      end

      context 'with check_loop_delay_xxx settings' do
        let(:pool_thread) { double('thread', :alive? => false) }
        let(:check_loop_delay_min) { 7 }
        let(:check_loop_delay_max) { 20 }
        let(:check_loop_delay_decay) { 2.1 }

        let(:config) {
      YAML.load(<<-EOT
---
:config:
  check_loop_delay_min: #{check_loop_delay_min}
  check_loop_delay_max: #{check_loop_delay_max}
  check_loop_delay_decay: #{check_loop_delay_decay}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
        before(:each) do
          # Reset the global variable - Note this is a code smell
          $threads = {}
          $threads[pool] = pool_thread
          allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
          # allow(subject).to receive(:load_used_dns_plugins).and_return('vmpooler/dns/mock')
        end

        it 'should run the check_pool method and log a message' do
          expect(subject).to receive(:check_pool).with(a_pool_with_name_of(pool),
                                                       check_loop_delay_min,
                                                       check_loop_delay_max,
                                                       check_loop_delay_decay)
          expect(logger).to receive(:log).with('d', "[!] [#{pool}] worker thread died, restarting")

          subject.execute!(1,0)
        end
      end
    end

    context 'delays between loops' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }
      let(:check_loop_delay_min) { 7 }
      let(:check_loop_delay_max) { 20 }
      let(:check_loop_delay_decay) { 2.1 }
      let(:maxloop) { 5 }
      let(:config) {
      YAML.load(<<-EOT
---
:config:
  check_loop_delay_min: #{check_loop_delay_min}
  check_loop_delay_max: #{check_loop_delay_max}
  check_loop_delay_decay: #{check_loop_delay_decay}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
      # Note a maxloop of zero can not be tested as it never terminates
      before(:each) do

        allow(subject).to receive(:check_disk_queue)
        allow(subject).to receive(:check_snapshot_queue)
        allow(subject).to receive(:check_pool)
        allow(subject).to receive(:check_ondemand_requests)
        allow(Vmpooler::Dns).to receive(:get_domain_for_pool)
      end

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times

        subject.execute!(maxloop,loop_delay)
      end
    end

    context 'loops specified number of times (5)' do
      let(:alive_thread) { double('thread', :alive? => true) }
      let(:check_loop_delay_min) { 7 }
      let(:check_loop_delay_max) { 20 }
      let(:check_loop_delay_decay) { 2.1 }
      let(:maxloop) { 5 }
      let(:config) {
      YAML.load(<<-EOT
---
:config:
  check_loop_delay_min: #{check_loop_delay_min}
  check_loop_delay_max: #{check_loop_delay_max}
  check_loop_delay_decay: #{check_loop_delay_decay}
:dns_configs:
  :mock:
    dns_class: base
:pools:
  - name: #{pool}
    dns_plugin: 'mock'
EOT
          )
        }
      # Note a maxloop of zero can not be tested as it never terminates
      before(:each) do
        allow(subject).to receive(:get_domain_for_pool)
        allow(subject).to receive(:get_dns_plugin_config_classes)
      end

      it 'should run startup tasks only once' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once
          expect(redis).to receive(:set).with('vmpooler__tasks__ondemandclone', 0).once
          expect(redis).to receive(:del).with('vmpooler__migration').once
        end

        subject.execute!(maxloop,0)
      end

      it 'should run per thread tasks 5 times when threads are not remembered' do
        expect(subject).to receive(:check_disk_queue).exactly(maxloop).times
        expect(subject).to receive(:check_snapshot_queue).exactly(maxloop).times
        expect(subject).to receive(:check_pool).exactly(maxloop).times
        expect(subject).to receive(:check_ondemand_requests).exactly(maxloop).times

        subject.execute!(maxloop,0)
      end

      it 'should not run per thread tasks when threads are alive' do
        expect(subject).to receive(:check_disk_queue).exactly(0).times
        expect(subject).to receive(:check_snapshot_queue).exactly(0).times
        expect(subject).to receive(:check_pool).exactly(0).times
        expect(subject).to receive(:check_ondemand_requests).exactly(0).times

        $threads[pool] = alive_thread
        $threads['disk_manager'] = alive_thread
        $threads['snapshot_manager'] = alive_thread
        $threads['ondemand_provisioner'] = alive_thread

        subject.execute!(maxloop,0)
      end
    end

    context 'when redis server connection is not available' do
      let(:maxloop) { 2 }
      it 'should log a failure and raise the error' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).and_raise(Redis::CannotConnectError)
        end
        expect(logger).to receive(:log).with('s', 'Cannot connect to the redis server: Redis::CannotConnectError')

        expect{subject.execute!(maxloop,0)}.to raise_error Redis::CannotConnectError
      end
    end
  end

  describe "#sleep_with_wakeup_events" do
    let(:loop_delay) { 10 }
    before(:each) do
      allow(Kernel).to receive(:sleep).and_raise("sleep should not be called")
      allow(subject).to receive(:time_passed?).with(:wakeup_by, Time).and_call_original
      allow(subject).to receive(:time_passed?).with(:exit_by, Time).and_call_original
    end

    it 'should not sleep if the loop delay is negative' do
      expect(subject).to receive(:sleep).exactly(0).times

      subject.sleep_with_wakeup_events(-1)
    end

    it 'should sleep until the loop delay is exceeded' do
      # This test is a little brittle as it requires knowledge of the implementation
      # Basically the number of sleep delays will dictate how often the time_passed? method is called

      expect(subject).to receive(:sleep).exactly(2).times
      expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, false, false, true)

      subject.sleep_with_wakeup_events(loop_delay)
    end

    describe 'with the pool_size_change wakeup option' do
      let(:wakeup_option) {{
        :pool_size_change => true,
        :poolname => pool,
      }}
      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      it 'should check the number of VMs ready in Redis' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").once
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end

      it 'should sleep until the number of VMs ready in Redis increases' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:sleep).exactly(3).times
          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1,1,1,2)
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end

      it 'should sleep until the number of VMs ready in Redis decreases' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:sleep).exactly(3).times
          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(2,2,2,1)
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end
    end

    describe 'with the pool_template_change wakeup option' do
      let(:wakeup_option) {{
        :pool_template_change => true,
        :poolname => pool
      }}
      let(:new_template) { 'templates/newtemplate' }
      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      context 'with a template configured' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.hset('vmpooler__config__template', pool, new_template)
            allow(redis).to receive(:hget)
          end
        end

        it 'should check if a template is configured in redis' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:time_passed?).with(:exit_by, Time).and_return(false, true)
            expect(redis).to receive(:hget).with('vmpooler__template__prepared', pool).once
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end

        it 'should sleep until a template change is detected' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:sleep).exactly(3).times
            expect(redis).to receive(:hget).with('vmpooler__config__template', pool).and_return(nil,nil,new_template)
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end
      end
    end

    describe 'with the pool_reset wakeup option' do
      let(:wakeup_option) {{
        :pool_reset => true,
        :poolname => pool
      }}

      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      context 'when a pool reset is requested' do

        it 'should sleep until the reset request is detected' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:sleep).exactly(3).times
            expect(redis).to receive(:sismember).with('vmpooler__poolreset', pool).and_return(false,false,true)
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end
      end
    end

    describe 'with the undo_override wakeup option' do
      let(:wakeup_option) {{
        :undo_override => true,
        :poolname => pool
      }}

      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      context 'when a undoing a template override is requested' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.sadd('vmpooler__pool__undo_template_override', pool)
            allow(redis).to receive(:hget)
          end
        end

        it 'should sleep until the undo override request is detected' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:sleep).at_least(2).times
            expect(subject).to receive(:sleep).at_most(3).times
            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,true)
            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false)
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end
      end

      context 'when a undoing a size override is requested' do
        before(:each) do
          redis_connection_pool.with do |redis|
            redis.sadd('vmpooler__pool__undo_size_override', pool)
            allow(redis).to receive(:hget)
          end
        end

        it 'should sleep until the undo override request is detected' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:sleep).exactly(3).times
            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(false,false,false)
            expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(false,false,true)
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end
      end
    end

    describe 'with the pending_vm wakeup option' do
      let(:wakeup_option) {{
        :pending_vm => true,
        :poolname => pool
      }}

      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      context 'when a pending_vm is detected' do

        it 'should sleep until the pending instance' do
          redis_connection_pool.with do |redis|
            expect(subject).to receive(:sleep).exactly(3).times
            expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(0,0,1)
          end

          subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
        end
      end
    end

    describe 'with the ondemand_request wakeup option' do
      let(:wakeup_option) {{ :ondemand_request => true }}

      let(:wakeup_period) { -1 } # A negative number forces the wakeup evaluation to always occur

      it 'should sleep until the provisioning request is detected' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:sleep).exactly(3).times
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,1)
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end

      it 'should sleep until provisioning processing is detected' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:sleep).exactly(3).times
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,1)
          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,0)
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end

      it 'should sleep until ondemand creation task is detected' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:sleep).exactly(3).times
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__request').and_return(0,0,0)
          expect(redis).to receive(:zcard).with('vmpooler__provisioning__processing').and_return(0,0,0)
          expect(redis).to receive(:zcard).with('vmpooler__odcreate__task').and_return(0,0,1)
        end

        subject.sleep_with_wakeup_events(loop_delay, wakeup_period, wakeup_option)
      end
    end
  end

  describe "#check_pool" do
    let(:threads) {{}}
    let(:provider_name) { 'mock_provider' }
    let(:config) {
      YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    provider: #{provider_name}
EOT
      )
    }

    let(:pool_object) { config[:pools][0] }
    let(:check_pool_response) {{
        :discovered_vms      => 0,
        :checked_running_vms => 0,
        :checked_ready_vms   => 0,
        :checked_pending_vms => 0,
        :destroyed_vms       => 0,
        :migrated_vms        => 0,
        :cloned_vms          => 0,
    }}

    before do
      expect(subject).not_to be_nil
      expect(Thread).to receive(:new).and_yield
      allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider)
    end

    context 'on startup' do
      before(:each) do
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      it 'should log a message the worker thread is starting' do
        subject.check_pool(pool_object,1,0)
      end

      it 'should populate the threads global variable' do
        subject.check_pool(pool_object,1,0)

        # Unable to test for nil as the Thread is mocked
        expect($threads.keys.include?(pool))
      end
    end

    context 'delays between loops' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }
      # Note a maxloop of zero can not be tested as it never terminates

      before(:each) do
        allow(logger).to receive(:log)
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay, Numeric, Hash).exactly(maxloop).times

        subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)
      end

      it 'specifies a wakeup option for pool size change' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay, Numeric, hash_including(:pool_size_change => true)).exactly(maxloop).times

        subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)
      end

      it 'specifies a wakeup option for poolname' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay, Numeric, hash_including(:poolname => pool)).exactly(maxloop).times

        subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)
      end
    end

    context 'when redis connection fails' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }

      it 'should raise the error' do
        allow(subject).to receive(:_check_pool).and_raise(Redis::CannotConnectError)
        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")

        expect{subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)}.to raise_error Redis::CannotConnectError
      end
    end

    context 'delays between loops configured in the pool configuration' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }
      let(:pool_loop_delay) { 2 }
      let(:config) {
        YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    provider: #{provider_name}
    check_loop_delay_min: #{pool_loop_delay}
    check_loop_delay_max: #{pool_loop_delay}
EOT
        )
      }

      before(:each) do
        allow(logger).to receive(:log)
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(pool_loop_delay, pool_loop_delay, Hash).exactly(maxloop).times

        subject.check_pool(pool_object,maxloop,loop_delay)
      end
    end

    context 'delays between loops with a specified min and max value' do
      let(:maxloop) { 5 }
      let(:loop_delay_min) { 1 }
      let(:loop_delay_max) { 60 }
      # Note a maxloop of zero can not be tested as it never terminates

      before(:each) do
        allow(logger).to receive(:log)
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      [:checked_pending_vms, :discovered_vms, :cloned_vms].each do |testcase|
        describe "when #{testcase} is greater than zero" do
          it "delays the minimum delay time" do
            expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay_min, loop_delay_min, Hash).exactly(maxloop).times
            check_pool_response[testcase] = 1

            subject.check_pool(pool_object,maxloop,loop_delay_min,loop_delay_max)
          end
        end
      end

      [:checked_running_vms, :checked_ready_vms, :destroyed_vms, :migrated_vms].each do |testcase|
        describe "when #{testcase} is greater than zero" do
          let(:loop_decay) { 3.0 }
          it "delays increases with a decay" do
            expect(subject).to receive(:sleep_with_wakeup_events).with(3, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(9, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(27, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(60, Numeric, Hash).twice
            check_pool_response[testcase] = 1

            subject.check_pool(pool_object,maxloop,loop_delay_min,loop_delay_max,loop_decay)
          end
        end
      end
    end

    context 'delays between loops with a specified min and max value configured in the pool configuration' do
      let(:maxloop) { 5 }
      let(:loop_delay_min) { 1 }
      let(:loop_delay_max) { 60 }
      let(:loop_decay) { 3.0 }
      let(:pool_loop_delay_min) { 3 }
      let(:pool_loop_delay_max) { 70 }
      let(:pool_loop_delay_decay) { 2.5 }
      let(:config) {
        YAML.load(<<-EOT
---
:pools:
  - name: #{pool}
    provider: #{provider_name}
    check_loop_delay_min: #{pool_loop_delay_min}
    check_loop_delay_max: #{pool_loop_delay_max}
    check_loop_delay_decay: #{pool_loop_delay_decay}
EOT
        )
      }

      before(:each) do
        allow(logger).to receive(:log)
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
      end

      [:checked_pending_vms, :discovered_vms, :cloned_vms].each do |testcase|
        describe "when #{testcase} is greater than zero" do
          it "delays the minimum delay time" do
            expect(subject).to receive(:sleep_with_wakeup_events).with(pool_loop_delay_min, Numeric, Hash).exactly(maxloop).times
            check_pool_response[testcase] = 1

            subject.check_pool(pool_object,maxloop,loop_delay_min,loop_delay_max,loop_decay)
          end
        end
      end

      [:checked_running_vms, :checked_ready_vms, :destroyed_vms, :migrated_vms].each do |testcase|
        describe "when #{testcase} is greater than zero" do
          it "delays increases with a decay" do
            expect(subject).to receive(:sleep_with_wakeup_events).with(7, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(17, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(42, Numeric, Hash).once
            expect(subject).to receive(:sleep_with_wakeup_events).with(70, Numeric, Hash).twice
            check_pool_response[testcase] = 1

            subject.check_pool(pool_object,maxloop,loop_delay_min,loop_delay_max,loop_decay)
          end
        end
      end
    end


    context 'loops specified number of times (5)' do
      let(:maxloop) { 5 }
      # Note a maxloop of zero can not be tested as it never terminates
      before(:each) do
        allow(logger).to receive(:log)
        allow(subject).to receive(:_check_pool).and_return(check_pool_response)
      end

      after(:each) do
        # Reset the global variable - Note this is a code smell
        $threads = nil
        $provider = nil
      end

      it 'should run startup tasks only once' do
        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")

        subject.check_pool(pool_object,maxloop,0)
      end

      it 'should run per thread tasks 5 times' do
        expect(subject).to receive(:_check_pool).exactly(maxloop).times

        subject.check_pool(pool_object,maxloop,0)
      end
    end
  end

  describe 'undo_override' do
    let(:mutex) { Mutex.new }
    let(:original_template) { 'templates/template1' }
    let(:override_template) { 'templates/template2' }
    let(:original_size) { 2 }
    let(:override_size) { 10 }
    let(:config) { YAML.load(<<-EOT
---
:config:
  task_limit: 5
:providers:
  :mock:
:pools:
  - name: '#{pool}'
    size: #{override_size}
    template: '#{override_template}'
:pool_index:
  '#{pool}': 0
:pools_at_startup:
  - name: '#{pool}'
    size: #{original_size}
    template: '#{original_template}'
EOT
      )
    }

    before(:each) do
      redis_connection_pool.with do |redis|
        redis.sadd('vmpooler__pool__undo_template_override', pool)
        redis.sadd('vmpooler__pool__undo_size_override', pool)
        # allow(redis).to receive(:hget)
      end
    end

    it 'should revert to the original template and pool size' do
      redis_connection_pool.with do |redis|
        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_template_override', pool).and_return(true)
        expect(redis).to receive(:srem).with('vmpooler__pool__undo_template_override', pool).and_return(true)
        expect(subject).to receive(:update_pool_template).with(config[:pools][0], provider, original_template, override_template, redis)

        expect(redis).to receive(:sismember).with('vmpooler__pool__undo_size_override', pool).and_return(true)
        expect(redis).to receive(:srem).with('vmpooler__pool__undo_size_override', pool).and_return(true)
      end

      subject.undo_override(config[:pools][0], provider)
    end
  end

  describe '#create_inventory' do

    it 'should log an error if one occurs' do
      allow(provider).to receive(:vms_in_pool).and_raise(
        RuntimeError,'Mock Error'
      )

      expect {
        subject.create_inventory(config[:pools].first, provider, {})
      }.to raise_error(RuntimeError, 'Mock Error')
    end
  end

  describe '#check_running_pool_vms' do
    let(:pool_object) { config[:pools][0] }
    let(:pool_check_response) {
      {:checked_running_vms => 0}
    }
    context 'Running VM not in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {}
      }
      before(:each) do
        redis_connection_pool.with do |redis|
          create_running_vm(pool,vm,redis,token)
        end
      end

      it 'should not call check_running_vm' do
        expect(subject).to receive(:check_running_vm).exactly(0).times

        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should move the VM to completed queue' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:move_vm_queue).with(pool,vm,'running','completed',redis,String).and_call_original
        end

        subject.check_running_pool_vms(pool,provider, pool_check_response, inventory)
      end
    end

    context 'Running VM in the inventory' do
      let(:provider) { double('provider') }
      let(:inventory) {
        # mock response from create_inventory
        { vm => 1 }
      }
      before(:each) do
        redis_connection_pool.with do |redis|
          allow(subject).to receive(:check_running_vm)
          create_running_vm(pool,vm,redis,token)
        end
      end

      it 'should log an error if one occurs' do
        expect(subject).to receive(:check_running_vm).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool with an error while evaluating running VMs: MockError")

        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should return the number of checked running VMs' do
        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)

        expect(pool_check_response[:checked_running_vms]).to be(1)
      end

      it 'should use the VM lifetime in preference to defaults' do
        big_lifetime = 2000
        redis_connection_pool.with do |redis|
          redis.hset("vmpooler__vm__#{vm}", 'lifetime',big_lifetime)
        end
        # The lifetime comes in as string
        expect(subject).to receive(:check_running_vm).with(vm,pool,"#{big_lifetime}",provider)

        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should use the configuration default if the VM lifetime is not set' do
        config[:config]['vm_lifetime'] = 50
        expect(subject).to receive(:check_running_vm).with(vm,pool,50,provider)

        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should use a lifetime of 12 if nothing is set' do
        expect(subject).to receive(:check_running_vm).with(vm,pool,12,provider)

        subject.check_running_pool_vms(pool, provider, pool_check_response, inventory)
      end
    end
  end

  describe '#check_ready_pool_vms' do
    let(:provider) { double('provider') }
    let(:pool_ttl) { 5 }
    let(:pool_check_response) {
      {:checked_ready_vms => 0}
    }

    context 'Ready VM not in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {}
      }
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool,vm,redis,token)
        end
      end

      it 'should not call check_ready_vm' do
        expect(subject).to receive(:check_ready_vm).exactly(0).times

        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
      end

      it 'should move the VM to completed queue' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:move_vm_queue).with(pool,vm,'ready','completed',redis,String).and_call_original
        end

        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
      end
    end

    context 'Ready VM in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {vm => 1}
      }
      let(:big_lifetime) { 2000 }
      before(:each) do
        redis_connection_pool.with do |redis|
          allow(subject).to receive(:check_ready_vm)
          create_ready_vm(pool,vm,redis,token)
        end
      end

      it 'should return the number of checked ready VMs' do
        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)

        expect(pool_check_response[:checked_ready_vms]).to be(1)
      end

      it 'should log an error if one occurs' do
        expect(subject).to receive(:check_ready_vm).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating ready VMs: MockError")

        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
      end

      it 'should use the pool TTL if set' do
        expect(subject).to receive(:check_ready_vm).with(vm,pool,big_lifetime,provider)

        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
      end

      it 'should use a pool TTL of zero if none set' do
        expect(subject).to receive(:check_ready_vm).with(vm,pool,pool_ttl,provider)

        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, pool_ttl)
      end
    end
  end

  describe '#check_pending_pool_vms' do
    let(:provider) { double('provider') }
    let(:timeout) { 10 }
    let(:pool_check_response) {
      {:checked_pending_vms => 0}
    }

    context 'Pending VM not in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_pending_vm(pool,vm,redis,token)
        end
      end

      it 'should call fail_pending_vm' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:fail_pending_vm).with(vm, pool, Integer, Integer, redis, exists: false)
        end

        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
      end
    end

    context 'Pending VM in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {vm => 1}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_pending_vm(pool,vm,redis,token)
        end
      end

      it 'should return the number of checked pending VMs' do
        allow(subject).to receive(:check_pending_vm)
        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)

        expect(pool_check_response[:checked_pending_vms]).to be(1)
      end

      it 'should log an error if one occurs' do
        expect(subject).to receive(:check_pending_vm).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating pending VMs: MockError")

        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
      end

      it 'should use the pool timeout if set' do
        big_lifetime = 2000

        config[:pools][0]['timeout'] = big_lifetime
        expect(subject).to receive(:check_pending_vm).with(vm,pool, big_lifetime, timeout_notification, provider)

        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
      end

      it 'should use the configuration setting if the pool timeout is not set' do
        big_lifetime = 2000

        config[:config]['timeout'] = big_lifetime
        expect(subject).to receive(:check_pending_vm).with(vm, pool, big_lifetime, timeout_notification, provider)

        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime, timeout_notification)
      end

      it 'should use a pool timeout of 15 if nothing is set' do
        expect(subject).to receive(:check_pending_vm).with(vm, pool, timeout, timeout_notification, provider)

        subject.check_pending_pool_vms(pool, provider, pool_check_response, inventory, timeout, timeout_notification)
      end
    end
  end

  describe '#check_completed_pool_vms' do
    let(:provider) { double('provider') }
    let(:pool_check_response) { {
        :checked_completed_vms => 0,
        :destroyed_vms => 0
      }
    }
    context 'Completed VM not in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_completed_vm(vm,pool,redis,true)
        end
      end

      it 'should log a message' do
        subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should not call destroy_vm' do
        expect(subject).to receive(:destroy_vm).exactly(0).times

        subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should remove redis information' do
        redis_connection_pool.with do |redis|
          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
          expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)

          subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)

          expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
          expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
          expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
        end
      end
    end

    context 'Completed VM in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {vm => 1}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_completed_vm(vm,pool,redis,true)
        end
      end

      it 'should call destroy_vm' do
        expect(subject).to receive(:destroy_vm)

        subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should return the number of destroyed VMs' do
        subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)

        expect(pool_check_response[:destroyed_vms]).to be(1)
      end

      context 'with an error during destroy_vm' do
        before(:each) do
          expect(subject).to receive(:destroy_vm).and_raise(RuntimeError,"MockError")
          expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating completed VMs: MockError")
        end

        it 'should log a message' do
          subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)
        end

        it 'should remove redis information' do
          redis_connection_pool.with do |redis|
            expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(true)
            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be(nil)
            expect(redis.hget("vmpooler__active__#{pool}",vm)).to_not be(nil)

            subject.check_completed_pool_vms(pool, provider, pool_check_response, inventory)

            expect(redis.sismember("vmpooler__completed__#{pool}",vm)).to be(false)
            expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be(nil)
            expect(redis.hget("vmpooler__active__#{pool}",vm)).to be(nil)
          end
        end
      end
    end
  end

  describe "#check_discovered_pool_vms" do
    context 'Discovered VM' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_discovered_vm(vm,pool,redis)
        end
      end

      it 'should be moved to the completed queue' do
        subject.check_discovered_pool_vms(pool)

        redis_connection_pool.with do |redis|
          expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
        end
      end

      it 'should log a message if an error occurs' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:smove).with("vmpooler__discovered__#{pool}", "vmpooler__completed__#{pool}", vm).and_raise(RuntimeError,'MockError')
        end
        expect(logger).to receive(:log).with("d", "[!] [#{pool}] _check_pool failed with an error while evaluating discovered VMs: MockError")

        subject.check_discovered_pool_vms(pool)
      end

      ['pending','ready','running','completed'].each do |queue_name|
        context "exists in the #{queue_name} queue" do
          before(:each) do
            allow(subject).to receive(:migrate_vm)
            allow(subject).to receive(:check_running_vm)
            allow(subject).to receive(:check_ready_vm)
            allow(subject).to receive(:check_pending_vm)
            allow(subject).to receive(:destroy_vm)
            allow(subject).to receive(:clone_vm)
          end

          it "should remain in the #{queue_name} queue" do
            redis_connection_pool.with do |redis|
              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
              allow(logger).to receive(:log)

              subject.check_discovered_pool_vms(pool)

              expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", vm)).to be(true)
            end
          end

          it "should be removed from the discovered queue" do
            redis_connection_pool.with do |redis|
              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
              allow(logger).to receive(:log)

              expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(true)
              subject.check_discovered_pool_vms(pool)
              expect(redis.sismember("vmpooler__discovered__#{pool}", vm)).to be(false)
            end
          end

          it "should log a message" do
            redis_connection_pool.with do |redis|
              redis.sadd("vmpooler__#{queue_name}__#{pool}", vm)
            end
            expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' found in '#{queue_name}', removed from 'discovered' queue")

            subject.check_discovered_pool_vms(pool)
          end
        end
      end
    end
  end

  describe "#check_migrating_pool_vms" do
    let(:provider) { double('provider') }
    let(:pool_check_response) { {
        :migrated_vms => 0
      }
    }

    context 'Migrating VM not in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_migrating_vm(vm,pool,redis)
        end
      end

      it 'should not do anything' do
        expect(subject).to receive(:migrate_vm).exactly(0).times

        subject.check_migrating_pool_vms(pool, provider, pool_check_response, inventory)
      end
    end

    context 'Migrating VM in the inventory' do
      let(:inventory) {
        # mock response from create_inventory
        {vm => 1}
      }

      before(:each) do
        redis_connection_pool.with do |redis|
          create_migrating_vm(vm,pool,redis)
        end
      end

      it 'should return the number of migrated VMs' do
        allow(subject).to receive(:migrate_vm).with(vm,pool,provider)
        subject.check_migrating_pool_vms(pool, provider, pool_check_response, inventory)

        expect(pool_check_response[:migrated_vms]).to be(1)
      end

      it 'should log an error if one occurs' do
        expect(subject).to receive(:migrate_vm).and_raise(RuntimeError,'MockError')
        expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm}' failed to migrate: MockError")

        subject.check_migrating_pool_vms(pool, provider, pool_check_response, inventory)
      end

      it 'should call migrate_vm' do
        expect(subject).to receive(:migrate_vm).with(vm,pool,provider)

        subject.check_migrating_pool_vms(pool, provider, pool_check_response, inventory)
      end
    end
  end

  describe '#repopulate_pool_vms' do
    let(:pool_size) { 1 }
    let(:config) {
      YAML.load(<<-EOT
---
:config:
  task_limit: 10
EOT
      )
    }
    let(:provider) { double('provider') }
    let(:pool_check_response) {
      {
        :cloned_vms => 0
      }
    }

    before(:each) do
      allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
    end

    it 'should not call clone_vm when number of VMs is equal to the pool size' do
      expect(subject).to receive(:clone_vm).exactly(0).times

      subject.repopulate_pool_vms(pool, provider, pool_check_response, 0)
    end

    it 'should not call clone_vm when number of VMs is greater than the pool size' do
      redis_connection_pool.with do |redis|
        create_ready_vm(pool,vm,redis,token)
      end
      expect(subject).to receive(:clone_vm).exactly(0).times

      subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
    end

    ['ready','pending'].each do |queue_name|
      it "should use VMs in #{queue_name} queue to calculate pool size" do
        expect(subject).to receive(:clone_vm).exactly(0).times
        # Modify the pool size to 1 and add a VM in the queue
        redis_connection_pool.with do |redis|
          redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
        end
        pool_size = 1

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end
    end

    ['running','completed','discovered','migrating'].each do |queue_name|
      it "should not use VMs in #{queue_name} queue to calculate pool size" do
        expect(subject).to receive(:clone_vm)
        # Modify the pool size to 1 and add a VM in the queue
        redis_connection_pool.with do |redis|
          redis.sadd("vmpooler__#{queue_name}__#{pool}",vm)
        end
        pool_size = 1

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end
    end

    it 'should log a message the first time a pool is empty' do
      expect(logger).to receive(:log).with('s', "[!] [#{pool}] is empty")

      subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
    end

    it 'should not log a message when the pool size is 0' do
      expect(logger).to_not receive(:log).with('s', "[!] [#{pool}] is empty")

      subject.repopulate_pool_vms(pool, provider, pool_check_response, 0)
    end

    context 'when pool is marked as empty' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      before(:each) do
        redis_connection_pool.with do |redis|
          redis.set("vmpooler__empty__#{pool}", 'true')
        end
      end

      it 'should not log a message when the pool remains empty' do
        expect(logger).to receive(:log).with('s', "[!] [#{pool}] is empty").exactly(0).times

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end

      it 'should remove the empty pool mark if it is no longer empty' do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool,vm,redis,token)

          expect(redis.get("vmpooler__empty__#{pool}")).to be_truthy
          subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
          expect(redis.get("vmpooler__empty__#{pool}")).to be_falsey
        end
      end
    end

    context 'when number of VMs is less than the pool size' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      it 'should return the number of cloned VMs' do
        pool_size = 5

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)

        expect(pool_check_response[:cloned_vms]).to be(pool_size)
      end

      it 'should call clone_vm to populate the pool' do
        pool_size = 5

        expect(subject).to receive(:clone_vm).exactly(pool_size).times

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end

      it 'should call clone_vm until task_limit is hit' do
        task_limit = 2
        pool_size = 5
        config[:config]['task_limit'] = task_limit

        expect(subject).to receive(:clone_vm).exactly(task_limit).times

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end

      it 'log a message if a cloning error occurs' do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool,'vm',redis)
        end
        pool_size = 2

        expect(subject).to receive(:clone_vm).and_raise(RuntimeError,"MockError")
        expect(logger).to receive(:log).with("s", "[!] [#{pool}] clone failed during check_pool with an error: MockError")
        expect{ subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size) }.to raise_error(RuntimeError,'MockError')

      end
    end

    context 'when a pool template is updating' do
      let(:poolsize) { 2 }
      let(:mutex) { Mutex.new }
      before(:each) do
        expect(subject).to receive(:pool_mutex).with(pool).and_return(mutex)
        mutex.lock
      end

      it 'should not call clone_vm to populate the pool' do
        expect(subject).to_not receive(:clone_vm)

        subject.repopulate_pool_vms(pool, provider, pool_check_response, poolsize)
      end
    end

    context 'export metrics' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      it 'increments metrics for ready queue' do
        redis_connection_pool.with do |redis|
          create_ready_vm(pool,'vm1',redis)
          create_ready_vm(pool,'vm2',redis)
          create_ready_vm(pool,'vm3',redis)
        end

        expect(metrics).to receive(:gauge).with("ready.#{pool}", 3)
        allow(metrics).to receive(:gauge)

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end

      it 'increments metrics for running queue' do
        redis_connection_pool.with do |redis|
          create_running_vm(pool,'vm1',redis,token)
          create_running_vm(pool,'vm2',redis,token)
          create_running_vm(pool,'vm3',redis,token)
        end

        expect(metrics).to receive(:gauge).with("running.#{pool}", 3)
        allow(metrics).to receive(:gauge)

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end

      it 'increments metrics with 0 when pool empty' do

        allow(metrics).to receive(:gauge)
        expect(metrics).to receive(:gauge).with("ready.#{pool}", 0)
        expect(metrics).to receive(:gauge).with("running.#{pool}", 0)

        subject.repopulate_pool_vms(pool, provider, pool_check_response, pool_size)
      end
    end
  end

  describe '#_check_pool' do
    let(:new_vm_response) {
      # Mock response from Base Provider for vms_in_pool
      [{ 'name' => new_vm}]
    }
    let(:vm_response) {
      # Mock response from Base Provider for vms_in_pool
      [{ 'name' => vm}]
    }
    let(:multi_vm_response) {
      # Mock response from Base Provider for vms_in_pool
      [{ 'name' => 'vm1'},
       { 'name' => 'vm2'},
       { 'name' => 'vm3'}]
    }

    # Default test fixtures will consist of;
    #   - Empty Redis dataset
    #   - A single pool with a pool size of zero i.e. no new VMs should be created
    #   - Task limit of 10
    let(:config) {
      YAML.load(<<-EOT
---
:config:
  task_limit: 10
:pools:
  - name: #{pool}
    size: 10
    ready_ttl: 1234
    timeout: 5678
EOT
      )
    }
    let(:pool_object) { config[:pools][0] }
    let(:new_vm) { 'newvm'}
    let(:pool_name) { pool_object['name'] }
    let(:mutex) { Mutex.new }

    before do
      expect(subject).not_to be_nil
      allow(logger).to receive(:log)
    end

    # INVENTORY
    context 'Conducting inventory' do
      before(:each) do
        allow(subject).to receive(:migrate_vm)
        allow(subject).to receive(:check_running_vm)
        allow(subject).to receive(:check_ready_vm)
        allow(subject).to receive(:check_pending_vm)
        allow(subject).to receive(:destroy_vm)
        allow(subject).to receive(:clone_vm)
        allow(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response)
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      it 'calls inventory correctly' do
        expect(subject).to receive(:create_inventory)
        subject._check_pool(pool_object, provider)
      end

      it 'captures #create_inventory errors correctly' do
        allow(subject).to receive(:create_inventory).and_raise(RuntimeError,'Mock Error')
        subject._check_pool(pool_object, provider)
      end

      it 'should return early if an error occurs' do
        allow(subject).to receive(:create_inventory).and_raise(
          RuntimeError,'Mock Error'
        )

        expect(subject).to_not receive(:check_running_pool_vms)
        subject._check_pool(pool_object, provider)
      end

      it 'should return that no actions were taken' do
        expect(provider).to receive(:vms_in_pool).and_raise(RuntimeError,'Mock Error')

        result = subject._check_pool(pool_object,provider)

        expect(result[:discovered_vms]).to be(0)
        expect(result[:checked_running_vms]).to be(0)
        expect(result[:checked_ready_vms]).to be(0)
        expect(result[:checked_pending_vms]).to be(0)
        expect(result[:destroyed_vms]).to be(0)
        expect(result[:migrated_vms]).to be(0)
        expect(result[:cloned_vms]).to be(0)
      end

      it 'should log the discovery of VMs' do
        expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue")

        subject._check_pool(pool_object,provider)
      end

      it 'should return the number of discovered VMs' do
        result = subject._check_pool(pool_object,provider)

        expect(result[:discovered_vms]).to be(1)
      end

      it 'should add undiscovered VMs to the completed queue' do
        redis_connection_pool.with do |redis|
          allow(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue")

          expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
          expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(false)

          subject._check_pool(pool_object,provider)

          expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
          expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
        end
      end

      ['running','ready','pending','completed','discovered','migrating'].each do |queue_name|
        it "should not discover VMs in the #{queue_name} queue" do
          redis_connection_pool.with do |redis|
            expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue").exactly(0).times
            expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false)
            redis.sadd("vmpooler__#{queue_name}__#{pool}", new_vm)

            subject._check_pool(pool_object,provider)

            if queue_name == 'discovered'
              # Discovered VMs end up in the completed queue
              expect(redis.sismember("vmpooler__completed__#{pool}", new_vm)).to be(true)
            else
              expect(redis.sismember("vmpooler__#{queue_name}__#{pool}", new_vm)).to be(true)
            end
          end
        end
      end

      it 'should get the pool mutex' do
        expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)

        subject._check_pool(pool_object,provider)
      end

      it 'should run synchronize' do
        expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)
        expect(mutex).to receive(:synchronize).at_least(:once)

        subject._check_pool(pool_object,provider)
      end

      it 'should yield when locked' do
        expect(subject).to receive(:pool_mutex).and_return(mutex).at_least(:once)
        mutex.lock
        expect(mutex).to receive(:synchronize).and_yield

        subject._check_pool(pool_object,provider)
      end
    end

    # RUNNING
    context 'when checking running VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
          discovered_vms: 0,
          checked_running_vms: 0,
          checked_ready_vms: 0,
          checked_pending_vms: 0,
          destroyed_vms: 0,
          migrated_vms: 0,
          cloned_vms: 0
        }
      }

      it 'should call #check_running_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_running_pool_vms).with(pool, provider, pool_check_response, {})

        subject._check_pool(pool_object,provider)
      end
    end

    # READY
    context 'when checking ready VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
          discovered_vms: 0,
          checked_running_vms: 0,
          checked_ready_vms: 0,
          checked_pending_vms: 0,
          destroyed_vms: 0,
          migrated_vms: 0,
          cloned_vms: 0
        }
      }

      it 'should call #check_ready_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_ready_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['ready_ttl'])

        subject._check_pool(pool_object,provider)
      end
    end

    # PENDING
    context 'when checking ready VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
          discovered_vms: 0,
          checked_running_vms: 0,
          checked_ready_vms: 0,
          checked_pending_vms: 0,
          destroyed_vms: 0,
          migrated_vms: 0,
          cloned_vms: 0
        }
      }

      it 'should call #check_ready_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_pending_pool_vms).with(pool, provider, pool_check_response, {}, pool_object['timeout'], pool_object['timeout_notification'])

        subject._check_pool(pool_object,provider)
      end
    end


    # COMPLETED
    context 'when checking completed VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
            discovered_vms: 0,
            checked_running_vms: 0,
            checked_ready_vms: 0,
            checked_pending_vms: 0,
            destroyed_vms: 0,
            migrated_vms: 0,
            cloned_vms: 0
        }
      }

      it 'should call #check_completed_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_completed_pool_vms).with(pool, provider, pool_check_response, {})

        subject._check_pool(pool_object,provider)
      end
    end

    # DISCOVERED
    context 'when checking discovered VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
            discovered_vms: 0,
            checked_running_vms: 0,
            checked_ready_vms: 0,
            checked_pending_vms: 0,
            destroyed_vms: 0,
            migrated_vms: 0,
            cloned_vms: 0
        }
      }

      it 'should call #check_discovered_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_discovered_pool_vms).with(pool)

        subject._check_pool(pool_object,provider)
      end
    end

    # MIGRATIONS
    context 'when checking migrating VMs' do
      before(:each) do
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      let(:pool_check_response) {
        {
          discovered_vms: 0,
          checked_running_vms: 0,
          checked_ready_vms: 0,
          checked_pending_vms: 0,
          destroyed_vms: 0,
          migrated_vms: 0,
          cloned_vms: 0
        }
      }

      it 'should call #check_migrating_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:check_migrating_pool_vms).with(pool, provider, pool_check_response, {})

        subject._check_pool(pool_object,provider)
      end
    end

    # update_pool_size
    context 'when a pool size configuration change is detected' do
      let(:poolsize) { 2 }
      let(:newpoolsize) { 3 }
      before(:each) do
        config[:pools][0]['size'] = poolsize
        redis_connection_pool.with do |redis|
          redis.hset('vmpooler__config__poolsize', pool, newpoolsize)
        end
        allow(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      it 'should change the pool size configuration' do
        allow(subject).to receive(:create_inventory).and_return({})

        expect(subject).to receive(:update_pool_size).and_call_original

        subject._check_pool(config[:pools][0],provider)

        expect(config[:pools][0]['size']).to be(newpoolsize)
      end
    end

    #REPOPULATE
    context 'when checking if pools need to be repopulated' do
      let(:pool_check_response) { {
        discovered_vms: 0,
        checked_running_vms: 0,
        checked_ready_vms: 0,
        checked_pending_vms: 0,
        destroyed_vms: 0,
        migrated_vms: 0,
        cloned_vms: 0
      } }
      it 'should call #repopulate_pool_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:repopulate_pool_vms).with(pool, provider, pool_check_response, config[:pools][0]['size'])

        subject._check_pool(pool_object, provider)
      end
    end

    #remove_excess_vms
    context 'when an excess number of ready vms exist' do

      before(:each) do
        redis_connection_pool.with do |redis|
          allow(redis).to receive(:scard)
          expect(redis).to receive(:scard).with("vmpooler__ready__#{pool}").and_return(1)
          expect(redis).to receive(:scard).with("vmpooler__pending__#{pool}").and_return(1)
        end
        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
      end

      it 'should call remove_excess_vms' do
        allow(subject).to receive(:create_inventory).and_return({})
        expect(subject).to receive(:remove_excess_vms).with(config[:pools][0])

        subject._check_pool(config[:pools][0],provider)
      end
    end
  end

  describe 'process_ondemand_requests' do
    context 'with no requests' do
      it 'returns 0' do
        result = subject.process_ondemand_requests
        expect(result).to eq(0)
      end

      it 'runs process_ondemand_vms' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:process_ondemand_vms).with(redis).and_return(0)
          subject.process_ondemand_requests
        end
      end

      it 'checks ready requests' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:check_ondemand_requests_ready).with(redis).and_return(0)
          subject.process_ondemand_requests
        end
      end
    end

    context 'with provisioning requests' do
      before(:each) do
        redis_connection_pool.with do |redis|
          redis.zadd('vmpooler__provisioning__request', current_time, request_id)
        end
      end

      it 'returns the number of requests processed' do
        result = subject.process_ondemand_requests
        expect(result).to eq(1)
      end

      it 'runs create_ondemand_vms for each request' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:create_ondemand_vms).with(request_id, redis)
          subject.process_ondemand_requests
        end
      end
    end
  end

  describe '#create_ondemand_vms' do
    context 'when requested does not have corresponding data' do
      it 'logs an error' do
        redis_connection_pool.with do |redis|
          expect(logger).to receive(:log).with('s', "Failed to find odrequest for request_id '1111'")
          subject.create_ondemand_vms('1111', redis)
        end
      end
    end

    context 'when request is already marked as failed' do
      let(:request_string) { "#{pool}:#{pool}:1" }
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
          set_ondemand_request_status(request_id, 'failed', redis)
        end
      end

      it 'logs that the request is already failed' do
        redis_connection_pool.with do |redis|
          expect(logger).to receive(:log).with('s', "Request '#{request_id}' already marked as failed, skipping VM creation")
          subject.create_ondemand_vms(request_id, redis)
        end
      end

      it 'removes the request from provisioning__request queue' do
        redis_connection_pool.with do |redis|
          subject.create_ondemand_vms(request_id, redis)
          expect(redis.zscore('vmpooler__provisioning__request', request_id)).to be_nil
        end
      end

      it 'does not create VM tasks' do
        redis_connection_pool.with do |redis|
          subject.create_ondemand_vms(request_id, redis)
          expect(redis.zcard('vmpooler__odcreate__task')).to eq(0)
        end
      end

      it 'does not add to provisioning__processing queue' do
        redis_connection_pool.with do |redis|
          subject.create_ondemand_vms(request_id, redis)
          expect(redis.zscore('vmpooler__provisioning__processing', request_id)).to be_nil
        end
      end
    end

    context 'with a request that has data' do
      let(:request_string) { "#{pool}:#{pool}:1" }
      before(:each) do
        expect(Time).to receive(:now).and_return(current_time).at_least(:once)
        redis_connection_pool.with do |redis|
          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
        end
      end

      it 'creates tasks for instances to be provisioned' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            allow(pipe).to receive(:zadd)
            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string}:#{request_id}")
          end
          subject.create_ondemand_vms(request_id, redis)
        end
      end

      it 'adds a member to provisioning__processing' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            allow(pipe).to receive(:zadd)
            expect(pipe).to receive(:zadd).with('vmpooler__provisioning__processing', current_time.to_i, request_id)
          end
          subject.create_ondemand_vms(request_id, redis)
        end
      end
    end
  end

  describe '#process_ondemand_vms' do
    it 'returns the length of the queue' do
      redis_connection_pool.with do |redis|
        result = subject.process_ondemand_vms(redis)
        expect(result).to eq(0)
      end
    end

    context 'with a request to create a single vm' do
      let(:request_string) { "#{pool}:#{pool}:1" }
      let(:pool_alias) { pool }
      before(:each) do
        config[:config]['ondemand_clone_limit'] = 10
        expect(subject).to receive(:get_provider_for_pool).and_return(provider)
        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
        redis_connection_pool.with do |redis|
          create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
        end
      end

      it 'creates the vm' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias)
          subject.process_ondemand_vms(redis)
        end
      end
    end

    context 'with a request to create more instances than the limit' do
      let(:request_string) { "#{pool}:#{pool}:5" }
      let(:request_string_remaining) { "#{pool}:#{pool}:2" }
      let(:pool_alias) { pool }
      before(:each) do
        config[:config]['ondemand_clone_limit'] = 3
        expect(subject).to receive(:get_provider_for_pool).and_return(provider)
        expect(subject).to receive(:get_dns_plugin_class_for_pool).and_return(dns_plugin)
        redis_connection_pool.with do |redis|
          create_ondemand_creationtask("#{request_string}:#{request_id}", current_time.to_i, redis)
        end
      end

      it 'should create the maximum number of vms' do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:clone_vm).with(pool, provider, dns_plugin, request_id, pool_alias).exactly(3).times
          subject.process_ondemand_vms(redis)
        end
      end

      it 'should add the remaining number back as a new create task with the same score' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:zadd).with('vmpooler__odcreate__task', current_time.to_i, "#{request_string_remaining}:#{request_id}")
          end
          subject.process_ondemand_vms(redis)
        end
      end

      it 'should return the number of requests processed' do
        redis_connection_pool.with do |redis|
          result = subject.process_ondemand_vms(redis)
          expect(result).to eq(1)
        end
      end
    end

    context 'when the limit has been reached' do
      let(:clone_count) { { 'ondemand_clone_count' => 3 } }
      before(:each) do
        config[:config]['ondemand_clone_limit'] = 3
        subject.instance_variable_set(:@tasks, clone_count)
      end

      it 'does not create any instances' do
        redis_connection_pool.with do |redis|
          expect(subject).to_not receive(:clone_vm)
          subject.process_ondemand_vms(redis)
        end
      end
    end
  end

  describe '#vms_ready?' do
    let(:request_string) { "#{pool}:#{pool}:5" }
    let(:platform_alias) { pool }
    before(:each) do
      redis_connection_pool.with do |redis|
        create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
      end
    end

    it 'fails if the request is not valid' do
      redis_connection_pool.with do |redis|
        request_id = "#{request_id}-wrong"
        create_ondemand_request_for_test(request_id, current_time.to_i, "#{pool}:5", redis)
        expect{subject.vms_ready?(request_id, redis)}.to raise_error(ArgumentError)
      end
    end

    it 'returns false when vms for request_id are not ready' do
      redis_connection_pool.with do |redis|
        result = subject.vms_ready?(request_id, redis)
        expect(result).to be false
      end
    end

    context 'with a request that has all instances ready' do
      before(:each) do
        redis_connection_pool.with do |redis|
          ['vm1','vm2','vm3','vm4','vm5'].each do |v|
            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
          end
        end
      end

      it 'returns true' do
        redis_connection_pool.with do |redis|
          result = subject.vms_ready?(request_id, redis)
          expect(result).to be true
        end
      end
    end

    context 'with a request that has some instances ready' do
      let(:request_string) { "#{pool}:#{pool}:3,#{pool}2:#{pool}2:3" }
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ondemand_request_for_test(request_id, current_time.to_i, request_string, redis)
          ['vm1','vm2','vm3'].each do |v|
            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
          end
        end
      end

      it 'returns false' do
        redis_connection_pool.with do |redis|
          result = subject.vms_ready?(request_id, redis)
          expect(result).to be false
        end
      end
    end
  end

  describe '#check_ondemand_requests_ready' do
    before(:each) do
      config[:config]['ondemand_request_ttl'] = 5
    end

    it 'returns 0 when no provisoning requests are in progress' do
      redis_connection_pool.with do |redis|
        result = subject.check_ondemand_requests_ready(redis)
        expect(result).to eq(0)
      end
    end

    context 'with requests in progress' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ondemand_processing(request_id, current_time, redis)
        end
      end

      it 'returns the number of requests processed' do
        expect(subject).to receive(:vms_ready?).and_return(false)
        redis_connection_pool.with do |redis|
          result = subject.check_ondemand_requests_ready(redis)
          expect(result).to eq(1)
        end
      end
    end
  end

  describe '#check_ondemand_request_ready' do
    let(:score) { current_time.to_f }
    before(:each) do
      config[:config]['ondemand_request_ttl'] = 5
    end

    context 'when the request is ready' do
      before(:each) do
        expect(subject).to receive(:vms_ready?).and_return(true)
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:zscore).and_return(score)
        end
      end

      it 'sets the request as ready' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'ready')
          subject.check_ondemand_request_ready(request_id, redis)
        end
      end

      it 'marks the ondemand request hash key for expiration in one month' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:expire).with("vmpooler__odrequest__#{request_id}", 2592000)
          subject.check_ondemand_request_ready(request_id, redis)
        end
      end

      it 'removes the request from processing' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
          subject.check_ondemand_request_ready(request_id, redis)
        end
      end
    end

    context 'with the score provided' do
      it 'should not request the score' do
        redis_connection_pool.with do |redis|
          expect(redis).to_not receive(:zscore)
          expect(subject).to receive(:vms_ready?).and_return(true)
          expect(redis).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
          subject.check_ondemand_request_ready(request_id, redis, score)
        end
      end
    end

    context 'when a request has taken too long to be filled' do
      it 'should return true for request_expired?' do
        redis_connection_pool.with do |redis|
          expect(redis).to receive(:zscore).and_return(score)
          expect(subject).to receive(:request_expired?).with(request_id, Float, redis).and_return(true)
          subject.check_ondemand_request_ready(request_id, redis)
        end
      end
    end
  end

  describe '#request_expired?' do
    let(:ondemand_request_ttl) { 5 }
    let(:expiration_ttl) { 10 }
    before(:each) do
      config[:config]['ondemand_request_ttl'] = ondemand_request_ttl
      config[:redis]['data_ttl'] = expiration_ttl
    end

    context 'with a request that has taken too long to be filled' do
      let(:expired_time) { (Time.now - 960).to_i }
      before(:each) do
        redis_connection_pool.with do |redis|
          expect(subject).to receive(:remove_vms_for_failed_request)
          create_ondemand_processing(request_id, expired_time, redis)
        end
      end

      it 'returns true when the request is expired' do
        redis_connection_pool.with do |redis|
          result = subject.request_expired?(request_id, expired_time, redis)
          expect(result).to be true
        end
      end

      it 'logs a message that the request has expired' do
        redis_connection_pool.with do |redis|
          expect(logger).to receive(:log).with('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
          subject.request_expired?(request_id, expired_time, redis)
        end
      end

      it 'removes the request from processing requests' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:zrem).with('vmpooler__provisioning__processing', request_id)
          end
          subject.request_expired?(request_id, expired_time, redis)
        end
      end

      it 'sets the status as failed on the request hash' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:hset).with("vmpooler__odrequest__#{request_id}", 'status', 'failed')
          end
          subject.request_expired?(request_id, expired_time, redis)
        end
      end

      it 'marks the request hash for expiration' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:expire).with("vmpooler__odrequest__#{request_id}", expiration_ttl * 60 * 60)
          end
          subject.request_expired?(request_id, expired_time, redis)
        end
      end
    end

    context 'with a request that has been made within the ttl' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ondemand_processing(request_id, current_time, redis)
        end
      end

      it 'should return false' do
        redis_connection_pool.with do |redis|
          result = subject.request_expired?(request_id, current_time, redis)
          expect(result).to be false
        end
      end
    end
  end

  describe '#remove_vms_for_failed_request)' do
    let(:expiration_ttl) { 100 * 60 * 60 }
    let(:platform_alias) { pool }
    let(:platforms_string) { "#{platform_alias}:#{pool}:3" }
    context 'with two vms marked as ready for the request' do
      before(:each) do
        redis_connection_pool.with do |redis|
          create_ondemand_request_for_test(request_id, current_time, platforms_string, redis)
          [vm,"#{vm}2"].each do |v|
            create_running_vm(pool, v, redis)
            redis.sadd("vmpooler__#{request_id}__#{platform_alias}__#{pool}", v)
          end
        end
      end

      it 'should remove the associated vms' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(subject).to receive(:move_vm_queue).with(pool, String, 'running', 'completed', pipe, "moved to completed queue. '#{request_id}' could not be filled in time").twice
          end
          subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
        end
      end

      it 'should mark the ready set for expiration' do
        redis_connection_pool.with do |redis|
          redis.pipelined do |pipe|
            expect(pipe).to receive(:expire).with("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
          end
          subject.remove_vms_for_failed_request(request_id, expiration_ttl, redis)
        end
      end
    end
  end

  describe 'check_ondemand_requests' do
    let(:threads) {[]}
    let(:maxloop) { 0 }
    let(:loop_delay_min) { 5 }
    let(:loop_delay_max) { 60 }
    let(:loop_delay_decay) { 2.0 }

    before(:each) do
      expect(Thread).to receive(:new).and_yield
    end

    it 'should log the ondemand provisioner is starting' do
      expect(subject).to receive(:_check_ondemand_requests).with(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
      expect(logger).to receive(:log).with('d', "[*] [ondemand_provisioner] starting worker thread")

      expect($threads.count).to be(0)
      subject.check_ondemand_requests
      expect($threads.count).to be(1)
    end

    context' delays between loops' do
      let(:maxloop) { 2 }
      let(:loop_delay) { 1 }

      it 'when a non-default loop delay is specified' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(loop_delay, Numeric, Hash).exactly(maxloop).times

        subject.check_ondemand_requests(maxloop,loop_delay,loop_delay)
      end
    end

    context 'delays between loops with a specified min and max value' do
      let(:maxloop) { 5 }
      let(:loop_delay_min) { 1 }
      let(:loop_delay_max) { 60 }
      let(:loop_decay) { 3.0 }

      it 'delay values increase with a decay' do
        expect(subject).to receive(:sleep_with_wakeup_events).with(3, Numeric, Hash).once
        expect(subject).to receive(:sleep_with_wakeup_events).with(9, Numeric, Hash).once
        expect(subject).to receive(:sleep_with_wakeup_events).with(27, Numeric, Hash).once
        expect(subject).to receive(:sleep_with_wakeup_events).with(60, Numeric, Hash).twice

        subject.check_ondemand_requests(maxloop,loop_delay_min,loop_delay_max,loop_decay)
      end
    end

    context 'loops specified number of times (5)' do
      let(:maxloop) { 5 }

      it 'should run startup tasks only once' do
        expect(logger).to receive(:log).with('d', "[*] [ondemand_provisioner] starting worker thread")

        subject.check_ondemand_requests(maxloop,0)
      end

      it 'should run per thread tasks 5 times' do
        expect(subject).to receive(:process_ondemand_requests).and_return(0).exactly(maxloop).times

        subject.check_ondemand_requests(maxloop,0)
      end
    end
  end
end