Add retry logic with a delay for vsphere connections

This commit adds retry logic and configurable delays to vsphere helper.
Without this change vmpooler instances that have large numbers of pools
can create enough connections in a short period of time to cause vcenter
issues.

lib/vmpooler/pool_manager.rb

@@ -413,7 +413,7 @@ module Vmpooler
     def check_disk_queue
       $logger.log('d', "[*] [disk_manager] starting worker thread")
 
-      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['disk_manager'] = Thread.new do
         loop do
@@ -439,7 +439,7 @@ module Vmpooler
     def check_snapshot_queue
       $logger.log('d', "[*] [snapshot_manager] starting worker thread")
 
-      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['snapshot_manager'] = Thread.new do
         loop do
@@ -547,7 +547,7 @@ module Vmpooler
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads[pool['name']] = Thread.new do
         loop do

lib/vmpooler/vsphere_helper.rb

@@ -6,8 +6,9 @@ module Vmpooler
     DISK_TYPE = 'thin'
     DISK_MODE = 'persistent'
 
-    def initialize(credentials, metrics)
-      $credentials = credentials
+    def initialize(config, metrics)
+      $credentials = config[:vsphere]
+      $conf = config[:config]
       $metrics = metrics
     end
 
@@ -19,10 +20,22 @@ module Vmpooler
     end
 
     def connect_to_vsphere(credentials)
-      @connection = RbVmomi::VIM.connect host: credentials['server'],
-                                         user: credentials['username'],
-                                         password: credentials['password'],
-                                         insecure: credentials['insecure'] || true
+      max_tries = $conf['max_tries'] || 3
+      retry_factor = $conf['retry_factor'] || 10
+      try = 1
+      begin
+        @connection = RbVmomi::VIM.connect host: credentials['server'],
+                                           user: credentials['username'],
+                                           password: credentials['password'],
+                                           insecure: credentials['insecure'] || true
+        $metrics.increment("connect.open")
+      rescue => err
+        try += 1
+        $metrics.increment("connect.fail")
+        raise err if try == max_tries
+        sleep(try * retry_factor)
+        retry
+      end
     end
 
     def add_disk(vm, size, datastore)