Add retry logic with a delay for vsphere connections

This commit adds retry logic and configurable delays to vsphere helper.
Without this change vmpooler instances that have large numbers of pools
can create enough connections in a short period of time to cause vcenter
issues.

lib/vmpooler/pool_manager.rb

@@ -413,7 +413,7 @@ module Vmpooler
     def check_disk_queue
       $logger.log('d', "[*] [disk_manager] starting worker thread")
 
-      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['disk_manager'] = Thread.new do
         loop do
@@ -439,7 +439,7 @@ module Vmpooler
     def check_snapshot_queue
       $logger.log('d', "[*] [snapshot_manager] starting worker thread")
 
-      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['snapshot_manager'] = Thread.new do
         loop do
@@ -547,7 +547,7 @@ module Vmpooler
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads[pool['name']] = Thread.new do
         loop do

lib/vmpooler/vsphere_helper.rb

@@ -6,8 +6,9 @@ module Vmpooler
     DISK_TYPE = 'thin'
     DISK_MODE = 'persistent'
 
-    def initialize(credentials, metrics)
-      $credentials = credentials
+    def initialize(config, metrics)
+      $credentials = config[:vsphere]
+      $conf = config[:config]
       $metrics = metrics
     end
 
@@ -19,10 +20,22 @@ module Vmpooler
     end
 
     def connect_to_vsphere(credentials)
-      @connection = RbVmomi::VIM.connect host: credentials['server'],
-                                         user: credentials['username'],
-                                         password: credentials['password'],
-                                         insecure: credentials['insecure'] || true
+      max_tries = $conf['max_tries'] || 3
+      retry_factor = $conf['retry_factor'] || 10
+      try = 1
+      begin
+        @connection = RbVmomi::VIM.connect host: credentials['server'],
+                                           user: credentials['username'],
+                                           password: credentials['password'],
+                                           insecure: credentials['insecure'] || true
+        $metrics.increment("connect.open")
+      rescue => err
+        try += 1
+        $metrics.increment("connect.fail")
+        raise err if try == max_tries
+        sleep(try * retry_factor)
+        retry
+      end
     end
 
     def add_disk(vm, size, datastore)

vmpooler.yaml.example

@@ -232,6 +232,17 @@
 #     in an effort to maintain a more even distribution of load across compute resources.
 #     The migration_limit ensures that no more than n migrations will be evaluated at any one time
 #     and greatly reduces the possibilty of VMs ending up bunched together on a particular host.
+#
+#  - max_tries
+#    Set the max number of times a connection should retry in vsphere helper.
+#    This optional setting allows a user to dial in retry limits to
+#    suit your environment.
+#
+#  - retry_factor
+#    When retrying, each attempt sleeps for the try count * retry_factor.
+#    Increase this number to lengthen the delay between retry attempts.
+#    This is particularly useful for instances with a large number of pools
+#    to prevent a thundering herd when retrying connections.
 
 # Example: