diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 2c3daab..bed2289 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -413,7 +413,7 @@ module Vmpooler def check_disk_queue $logger.log('d', "[*] [disk_manager] starting worker thread") - $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics + $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics $threads['disk_manager'] = Thread.new do loop do @@ -439,7 +439,7 @@ module Vmpooler def check_snapshot_queue $logger.log('d', "[*] [snapshot_manager] starting worker thread") - $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics + $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics $threads['snapshot_manager'] = Thread.new do loop do @@ -547,7 +547,7 @@ module Vmpooler def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") - $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics + $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics $threads[pool['name']] = Thread.new do loop do diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index 946f9a8..cc93250 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -6,8 +6,9 @@ module Vmpooler DISK_TYPE = 'thin' DISK_MODE = 'persistent' - def initialize(credentials, metrics) - $credentials = credentials + def initialize(config, metrics) + $credentials = config[:vsphere] + $conf = config[:config] $metrics = metrics end @@ -19,10 +20,22 @@ module Vmpooler end def connect_to_vsphere(credentials) - @connection = RbVmomi::VIM.connect host: credentials['server'], - user: credentials['username'], - password: credentials['password'], - insecure: credentials['insecure'] || true + max_tries = $conf['max_tries'] || 3 + retry_factor = $conf['retry_factor'] || 10 + try = 1 + begin + @connection = RbVmomi::VIM.connect host: credentials['server'], + user: credentials['username'], + password: credentials['password'], + insecure: credentials['insecure'] || true + $metrics.increment("connect.open") + rescue => err + try += 1 + $metrics.increment("connect.fail") + raise err if try == max_tries + sleep(try * retry_factor) + retry + end end def add_disk(vm, size, datastore) diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index c83808c..76dcb25 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -232,6 +232,17 @@ # in an effort to maintain a more even distribution of load across compute resources. # The migration_limit ensures that no more than n migrations will be evaluated at any one time # and greatly reduces the possibilty of VMs ending up bunched together on a particular host. +# +# - max_tries +# Set the max number of times a connection should retry in vsphere helper. +# This optional setting allows a user to dial in retry limits to +# suit your environment. +# +# - retry_factor +# When retrying, each attempt sleeps for the try count * retry_factor. +# Increase this number to lengthen the delay between retry attempts. +# This is particularly useful for instances with a large number of pools +# to prevent a thundering herd when retrying connections. # Example: