Replace connection_pool with vmpooler generic_connection_pool. Fix tests for changes.

bin/vmpooler

@@ -38,7 +38,7 @@ if torun.include? 'manager'
     Vmpooler::PoolManager.new(
       config,
       Vmpooler.new_logger(logger_file),
-      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout),
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
       metrics
     ).execute!
   end

lib/vmpooler.rb

@@ -160,9 +160,16 @@ module Vmpooler
     pools
   end
 
-  def self.redis_connection_pool(host, port, password, size, timeout)
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
-    ConnectionPool.new(size: size, timeout: timeout) {
+    Vmpooler::PoolManager::GenericConnectionPool.new(
-      redis_connection(host, port, password)
+      metrics: metrics,
+      metric_prefix: 'redis_connection_pool',
+      size: size,
+      timeout: timeout
+    ) {
+      connection = Concurrent::Hash.new
+      redis = redis_connection(host, port, password)
+      connection['connection'] = redis
     }
   end
 

lib/vmpooler/generic_connection_pool.rb

@@ -15,35 +15,17 @@ module Vmpooler
         @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
       end
 
-      if Thread.respond_to?(:handle_interrupt)
+      def with_metrics(options = {})
-        # MRI
+        Thread.handle_interrupt(Exception => :never) do
-        def with_metrics(options = {})
-          Thread.handle_interrupt(Exception => :never) do
-            start = Time.now
-            conn = checkout(options)
-            timespan_ms = ((Time.now - start) * 1000).to_i
-            @metrics&.gauge(@metric_prefix + '.available', @available.length)
-            @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
-            begin
-              Thread.handle_interrupt(Exception => :immediate) do
-                yield conn
-              end
-            ensure
-              checkin
-              @metrics&.gauge(@metric_prefix + '.available', @available.length)
-            end
-          end
-        end
-      else
-        # jruby 1.7.x
-        def with_metrics(options = {})
           start = Time.now
           conn = checkout(options)
           timespan_ms = ((Time.now - start) * 1000).to_i
           @metrics&.gauge(@metric_prefix + '.available', @available.length)
           @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
           begin
-            yield conn
+            Thread.handle_interrupt(Exception => :immediate) do
+              yield conn
+            end
           ensure
             checkin
             @metrics&.gauge(@metric_prefix + '.available', @available.length)

lib/vmpooler/pool_manager.rb

@@ -48,7 +48,7 @@ module Vmpooler
 
     # Place pool configuration in redis so an API instance can discover running pool configuration
     def load_pools_to_redis
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         previously_configured_pools = redis.smembers('vmpooler__pools')
         currently_configured_pools = []
         config[:pools].each do |pool|
@@ -80,7 +80,7 @@ module Vmpooler
           _check_pending_vm(vm, pool, timeout, provider)
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
             fail_pending_vm(vm, pool, timeout, redis, request_id=request_id)
           end
@@ -94,10 +94,10 @@ module Vmpooler
       return if mutex.locked?
 
       mutex.synchronize do
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
           if provider.vm_ready?(pool, vm)
-            move_pending_vm_to_ready(vm, pool, request_id, redis)
+            move_pending_vm_to_ready(vm, pool, redis, request_id)
           else
             fail_pending_vm(vm, pool, timeout, redis, request_id = request_id)
           end
@@ -138,13 +138,8 @@ module Vmpooler
       false
     end
 
-    def move_pending_vm_to_ready(vm, pool, request_id, redis)
+    def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
       clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
-    # return false unless clone_time
-    # if clone_time == 'QUEUED'
-    #   $logger.log('s', "Waiting for clone_time. Got 'QUEUED'.")
-    #   return false
-    # end
       finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
 
       if request_id
@@ -198,7 +193,7 @@ module Vmpooler
       return if mutex.locked?
 
       mutex.synchronize do
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
           return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
 
@@ -220,7 +215,7 @@ module Vmpooler
             end
           end
 
-          return if mismatched_hostname?(vm, pool_name, provider)
+          return if mismatched_hostname?(vm, pool_name, provider, redis)
 
           vm_still_ready?(pool_name, vm, provider, redis)
         end
@@ -274,7 +269,7 @@ module Vmpooler
       return if mutex.locked?
 
       mutex.synchronize do
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           # Check that VM is within defined lifetime
           checkouttime = redis.hget('vmpooler__active__' + pool, vm)
       #   return if checkouttime == 'QUEUED'
@@ -320,7 +315,7 @@ module Vmpooler
         rescue StandardError => e
           if request_id
             $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
-            @redis.with do |redis|
+            @redis.with_metrics do |redis|
               redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
             end
           else
@@ -367,7 +362,7 @@ module Vmpooler
     end
 
     def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         new_vmname = find_unique_hostname(pool_name, redis)
         mutex = vm_mutex(new_vmname)
         mutex.synchronize do
@@ -430,7 +425,7 @@ module Vmpooler
       return if mutex.locked?
 
       mutex.synchronize do
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           redis.hdel('vmpooler__active__' + pool, vm)
           redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
 
@@ -581,7 +576,7 @@ module Vmpooler
       finish = format('%<time>.2f', time: Time.now - start)
 
       if result
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
           disks = rdisks ? rdisks.split(':') : []
           disks.push("+#{disk_size}gb")
@@ -616,7 +611,7 @@ module Vmpooler
       finish = format('%<time>.2f', time: Time.now - start)
 
       if result
-        @redis.with do |redis|
+        @redis.with_metrics do |redis|
           redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
         end
         $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
@@ -708,7 +703,7 @@ module Vmpooler
     end
 
     def _check_disk_queue
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         task_detail = redis.spop('vmpooler__tasks__disk')
         unless task_detail.nil?
           begin
@@ -746,7 +741,7 @@ module Vmpooler
     end
 
     def _check_snapshot_queue
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         task_detail = redis.spop('vmpooler__tasks__snapshot')
 
         unless task_detail.nil?
@@ -788,7 +783,7 @@ module Vmpooler
         begin
           mutex = vm_mutex(vm_name)
           mutex.synchronize do
-            @redis.with do |redis|
+            @redis.with_metrics do |redis|
               redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
             end
             provider.migrate_vm(pool_name, vm_name)
@@ -823,7 +818,7 @@ module Vmpooler
       wakeup_by = Time.now + wakeup_period
       return if time_passed?(:exit_by, exit_by)
 
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
 
         initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
@@ -942,7 +937,7 @@ module Vmpooler
     end
 
     def sync_pool_template(pool)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         pool_template = redis.hget('vmpooler__config__template', pool['name'])
         if pool_template
           pool['template'] = pool_template unless pool['template'] == pool_template
@@ -966,7 +961,7 @@ module Vmpooler
 
     def evaluate_template(pool, provider)
       mutex = pool_mutex(pool['name'])
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
         configured_template = redis.hget('vmpooler__config__template', pool['name'])
         return if mutex.locked?
@@ -1013,10 +1008,10 @@ module Vmpooler
       pool['template'] = configured_template
       $logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
       # Remove all ready and pending VMs so new instances are created from the new template
-      drain_pool(pool['name'])
+      drain_pool(pool['name'], redis)
       # Prepare template for deployment
       $logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
-      prepare_template(pool, provider)
+      prepare_template(pool, provider, redis)
       $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
@@ -1024,7 +1019,7 @@ module Vmpooler
       mutex = pool_mutex(pool['name'])
       return if mutex.locked?
 
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
         return if clone_target.nil?
         return if clone_target == pool['clone_target']
@@ -1040,7 +1035,7 @@ module Vmpooler
     end
 
     def remove_excess_vms(pool)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.multi
         redis.scard("vmpooler__ready__#{pool['name']}")
         redis.scard("vmpooler__pending__#{pool['name']}")
@@ -1072,7 +1067,7 @@ module Vmpooler
       mutex = pool_mutex(pool['name'])
       return if mutex.locked?
 
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
         return if poolsize.nil?
 
@@ -1087,7 +1082,7 @@ module Vmpooler
 
     def reset_pool(pool)
       poolname = pool['name']
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         return unless redis.sismember('vmpooler__poolreset', poolname)
 
         redis.srem('vmpooler__poolreset', poolname)
@@ -1104,7 +1099,7 @@ module Vmpooler
       begin
         mutex = pool_mutex(pool['name'])
         mutex.synchronize do
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             provider.vms_in_pool(pool['name']).each do |vm|
               if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
                  !redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
@@ -1131,7 +1126,7 @@ module Vmpooler
     end
 
     def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
           if inventory[vm]
             begin
@@ -1149,12 +1144,12 @@ module Vmpooler
     end
 
     def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
           if inventory[vm]
             begin
               pool_check_response[:checked_ready_vms] += 1
-              check_ready_vm(vm, pool_name, pool_ttl || 0, provider, redis)
+              check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
             rescue StandardError => e
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
             end
@@ -1167,7 +1162,7 @@ module Vmpooler
 
     def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout = nil)
       pool_timeout ||= $config[:config]['timeout'] || 15
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
           if inventory[vm]
             begin
@@ -1184,7 +1179,7 @@ module Vmpooler
     end
 
     def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
           if inventory[vm]
             begin
@@ -1211,7 +1206,7 @@ module Vmpooler
     end
 
     def check_discovered_pool_vms(pool_name)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
           %w[pending ready running completed].each do |queue|
             if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
@@ -1228,7 +1223,7 @@ module Vmpooler
     end
 
     def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
           if inventory[vm]
             begin
@@ -1245,7 +1240,7 @@ module Vmpooler
     def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
       return if pool_mutex(pool_name).locked?
 
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         redis.multi
         redis.scard("vmpooler__ready__#{pool_name}")
         redis.scard("vmpooler__pending__#{pool_name}")
@@ -1384,7 +1379,7 @@ module Vmpooler
     end
 
     def process_ondemand_requests
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
         requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
         provisioning_tasks = process_ondemand_vms(redis)
@@ -1479,7 +1474,7 @@ module Vmpooler
     def execute!(maxloop = 0, loop_delay = 1)
       $logger.log('d', 'starting vmpooler')
 
-      @redis.with do |redis|
+      @redis.with_metrics do |redis|
         # Clear out the tasks manager, as we don't know about any tasks at this point
         redis.set('vmpooler__tasks__clone', 0)
         # Clear out vmpooler__migrations since stale entries may be left after a restart

lib/vmpooler/providers/dummy.rb

@@ -9,8 +9,8 @@ module Vmpooler
       class Dummy < Vmpooler::PoolManager::Provider::Base
         # Fake VM Provider for testing
 
-        def initialize(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
-          super(config, logger, metrics, name, options)
+          super(config, logger, metrics, redis_connection_pool, name, options)
           dummyfilename = provider_config['filename']
 
           # This initial_state option is only intended to be used by spec tests

lib/vmpooler/providers/vsphere.rb

@@ -60,7 +60,7 @@ module Vmpooler
         def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
           try = 0 if try.nil?
           max_tries = 3
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             redis.multi
             redis.srem("vmpooler__completed__#{pool}", vm_name)
             redis.hdel("vmpooler__active__#{pool}", vm_name)
@@ -973,26 +973,24 @@ module Vmpooler
             begin
               connection = ensured_vsphere_connection(pool_object)
               vm_hash = get_vm_details(pool_name, vm_name, connection)
-              @redis.with do |redis|
+              @redis.with_metrics do |redis|
-                redis.multi
                 redis.hset("vmpooler__vm__#{vm_name}", 'host', vm_hash['host_name'])
                 migration_count = redis.scard('vmpooler__migration')
-                redis.exec
+                migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
-              end
+                if migration_enabled? @config
-              migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
+                  if migration_count >= migration_limit
-              if migration_enabled? @config
+                    logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
-                if migration_count >= migration_limit
+                    break
-                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
+                  end
-                  break
+                  run_select_hosts(pool_name, @provider_hosts)
-                end
+                  if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
-                run_select_hosts(pool_name, @provider_hosts)
+                    logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
-                if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
+                  else
-                  logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
+                    migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
+                  end
                 else
-                  migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
+                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
                 end
-              else
-                logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
               end
             rescue StandardError
               logger.log('s', "[!] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
@@ -1002,13 +1000,13 @@ module Vmpooler
         end
 
         def migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             redis.sadd('vmpooler__migration', vm_name)
           end
           target_host_name = select_next_host(pool_name, @provider_hosts, vm_hash['architecture'])
           target_host_object = find_host_by_dnsname(connection, target_host_name)
           finish = migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, target_host_name)
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             redis.multi
             redis.hset("vmpooler__vm__#{vm_name}", 'host', target_host_name)
             redis.hset("vmpooler__vm__#{vm_name}", 'migrated', true)
@@ -1016,7 +1014,7 @@ module Vmpooler
           end
           logger.log('s', "[>] [#{pool_name}] '#{vm_name}' migrated from #{vm_hash['host_name']} to #{target_host_name} in #{finish} seconds")
         ensure
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             redis.srem('vmpooler__migration', vm_name)
           end
         end
@@ -1028,7 +1026,7 @@ module Vmpooler
           metrics.timing("migrate.#{pool_name}", finish)
           metrics.increment("migrate_from.#{vm_hash['host_name']}")
           metrics.increment("migrate_to.#{dest_host_name}")
-          @redis.with do |redis|
+          @redis.with_metrics do |redis|
             checkout_to_migration = format('%<time>.2f', time: Time.now - Time.parse(redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
             redis.multi
             redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)

spec/fixtures/vmpooler.yaml

@@ -36,6 +36,8 @@
   - name: 'pool01'
     size: 5
     provider: dummy
+    ready_ttl: 5
   - name: 'pool02'
     size: 5
     provider: dummy
+    ready_ttl: 5

spec/fixtures/vmpooler2.yaml

@@ -36,6 +36,8 @@
   - name: 'pool03'
     size: 5
     provider: dummy
+    ready_ttl: 5
   - name: 'pool04'
     size: 5
     provider: dummy
+    ready_ttl: 5

spec/helpers.rb

@@ -1,12 +1,5 @@
 require 'mock_redis'
 
-def redis
-  unless @redis
-    @redis = MockRedis.new
-  end
-  @redis
-end
-
 # Mock an object which represents a Logger.  This stops the proliferation
 # of allow(logger).to .... expectations in tests.
 class MockLogger
@@ -40,59 +33,54 @@ def token_exists?(token)
   result && !result.empty?
 end
 
-def create_ready_vm(template, name, token = nil)
+def create_ready_vm(template, name, redis, token = nil)
-  create_vm(name, token)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__ready__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_running_vm(template, name, token = nil, user = nil)
+def create_running_vm(template, name, redis, token = nil, user = nil)
-  create_vm(name, token, nil, user)
+  create_vm(name, redis, token, user)
   redis.sadd("vmpooler__running__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", 'template', template)
   redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
   redis.hset("vmpooler__vm__#{name}", 'host', 'host1')
 end
 
-def create_pending_vm(template, name, token = nil)
+def create_pending_vm(template, name, redis, token = nil)
-  create_vm(name, token)
+  create_vm(name, redis, token)
   redis.sadd("vmpooler__pending__#{template}", name)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_vm(name, token = nil, redis_handle = nil, user = nil)
+def create_vm(name, redis, token = nil, user = nil)
-  redis_db = redis_handle ? redis_handle : redis
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__vm__#{name}", 'clone', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'token:token', token) if token
+  redis.hset("vmpooler__vm__#{name}", 'token:token', token) if token
-  redis_db.hset("vmpooler__vm__#{name}", 'token:user', user) if user
+  redis.hset("vmpooler__vm__#{name}", 'token:user', user) if user
 end
 
-def create_completed_vm(name, pool, active = false, redis_handle = nil)
+def create_completed_vm(name, pool, redis, active = false)
-  redis_db = redis_handle ? redis_handle : redis
+  redis.sadd("vmpooler__completed__#{pool}", name)
-  redis_db.sadd("vmpooler__completed__#{pool}", name)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.hset("vmpooler__active__#{pool}", name, Time.now) if active
-  redis_db.hset("vmpooler__active__#{pool}", name, Time.now) if active
 end
 
-def create_discovered_vm(name, pool, redis_handle = nil)
+def create_discovered_vm(name, pool, redis)
-  redis_db = redis_handle ? redis_handle : redis
+  redis.sadd("vmpooler__discovered__#{pool}", name)
-  redis_db.sadd("vmpooler__discovered__#{pool}", name)
 end
 
-def create_migrating_vm(name, pool, redis_handle = nil)
+def create_migrating_vm(name, pool, redis)
-  redis_db = redis_handle ? redis_handle : redis
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
-  redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
+  redis.sadd("vmpooler__migrating__#{pool}", name)
-  redis_db.sadd("vmpooler__migrating__#{pool}", name)
 end
 
-def create_tag(vm, tag_name, tag_value, redis_handle = nil)
+def create_tag(vm, tag_name, tag_value, redis)
-  redis_db = redis_handle ? redis-handle : redis
+  redis.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
-  redis_db.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
 end
 
-def add_vm_to_migration_set(name, redis_handle = nil)
+def add_vm_to_migration_set(name, redis)
-  redis_db = redis_handle ? redis_handle : redis
+  redis.sadd('vmpooler__migration', name)
-  redis_db.sadd('vmpooler__migration', name)
 end
 
 def fetch_vm(vm)

spec/unit/providers/base_spec.rb

@@ -22,7 +22,9 @@ describe 'Vmpooler::PoolManager::Provider::Base' do
     fake_vm
   }
 
-  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, provider_name, provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, redis_connection_pool, provider_name, provider_options) }
 
   # Helper attr_reader methods
   describe '#logger' do

spec/unit/providers/dummy_spec.rb

@@ -91,7 +91,9 @@ EOT
     )
   }
 
-  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, 'dummy', provider_options) }
+  let(:redis_connection_pool) { ConnectionPool.new(size: 1) { MockRedis.new } }
+
+  subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, redis_connection_pool, 'dummy', provider_options) }
 
   describe '#name' do
     it 'should be dummy' do

spec/unit/providers/vsphere_spec.rb

@@ -77,9 +77,15 @@ EOT
   let(:connection_options) {{}}
   let(:connection) { mock_RbVmomi_VIM_Connection(connection_options) }
   let(:vmname) { 'vm1' }
-  let(:redis) { MockRedis.new }
+  let(:redis_connection_pool) { Vmpooler::PoolManager::GenericConnectionPool.new(
+    metrics: metrics,
+    metric_prefix: 'redis_connection_pool',
+    size: 1,
+    timeout: 5
+  ) { MockRedis.new }
+  }
 
-  subject { Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, 'vsphere', provider_options) }
+  subject { Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, redis_connection_pool, 'vsphere', provider_options) }
 
   before(:each) do
     allow(subject).to receive(:vsphere_connection_ok?).and_return(true)
@@ -149,17 +155,6 @@ EOT
         allow(destroy_task).to receive(:wait_for_completion)
         allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
         allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
-        $redis = redis
-      end
-
-      it 'should remove redis data and expire the vm key' do
-        allow(Time).to receive(:now).and_return(now)
-        expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname)
-        expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname)
-        expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now)
-        expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60)
-
-        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
       end
 
       it 'should log a message that the vm is destroyed' do
@@ -169,6 +164,7 @@ EOT
       end
 
       it 'should record metrics' do
+        expect(metrics).to receive(:timing).with('redis_connection_pool.waited', 0)
         expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
 
         subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
@@ -2573,10 +2569,12 @@ EOT
         subject.connection_pool.with_metrics do |pool_object|
           expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
           expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          allow($redis).to receive(:hset)
           expect(subject).to receive(:run_select_hosts).with(poolname, {})
           expect(subject).to receive(:vm_in_target?).and_return false
           expect(subject).to receive(:migration_enabled?).and_return true
+          redis_connection_pool.with do |redis|
+            redis.hset("vmpooler__vm__#{vmname}", 'checkout', Time.now)
+          end
         end
         vm_object.summary.runtime.host = host_object
       end
@@ -2585,7 +2583,6 @@ EOT
         expect(subject).to receive(:select_next_host).and_return(new_host)
         expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
         expect(subject).to receive(:migrate_vm_host).with(vm_object, new_host_object)
-        expect($redis).to receive(:hget).with("vmpooler__vm__#{vmname}", 'checkout').and_return((Time.now - 1).to_s)
         expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from #{parent_host} to #{new_host} in 0.00 seconds")
 
         subject.migrate_vm(poolname, vmname)
@@ -2603,7 +2600,6 @@ EOT
         subject.connection_pool.with_metrics do |pool_object|
           expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
           expect(subject).to receive(:get_vm_details).and_return(vm_details)
-          expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'host', parent_host)
           expect(subject).to receive(:run_select_hosts).with(poolname, {})
           expect(subject).to receive(:vm_in_target?).and_return true
           expect(subject).to receive(:migration_enabled?).and_return true
@@ -2622,11 +2618,13 @@ EOT
       before(:each) do
         subject.connection_pool.with_metrics do |pool_object|
           expect(subject).to receive(:ensured_vsphere_connection).with(pool_object).and_return(connection)
-          expect($redis).to receive(:scard).with('vmpooler__migration').and_return(5)
           expect(subject).to receive(:get_vm_details).and_return(vm_details)
           expect(subject).to_not receive(:run_select_hosts)
           expect(subject).to_not receive(:vm_in_target?)
           expect(subject).to receive(:migration_enabled?).and_return true
+          redis_connection_pool.with do |redis|
+            expect(redis).to receive(:scard).with('vmpooler__migration').and_return(5)
+          end
         end
         vm_object.summary.runtime.host = host_object
       end
@@ -2691,14 +2689,10 @@ EOT
     end
 
     it' migrates a vm' do
-      expect($redis).to receive(:sadd).with('vmpooler__migration', vmname)
       expect(subject).to receive(:select_next_host).and_return(new_host)
-      expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'host', new_host)
-      expect($redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'migrated', true)
       expect(subject).to receive(:find_host_by_dnsname).and_return(new_host_object)
       expect(subject).to receive(:migrate_vm_and_record_timing).and_return(format('%.2f', (Time.now - (Time.now - 15))))
       expect(logger).to receive(:log).with('s', "[>] [#{poolname}] '#{vmname}' migrated from host1 to host2 in 15.00 seconds")
-      expect($redis).to receive(:srem).with('vmpooler__migration', vmname)
       subject.migrate_vm_to_new_host(poolname, vmname, vm_details, connection)
     end
   end