Merge pull request #173 from mattkirby/retry_logic

Add retry logic with a delay for vsphere connections

lib/vmpooler/pool_manager.rb

@@ -413,7 +413,7 @@ module Vmpooler
     def check_disk_queue
       $logger.log('d', "[*] [disk_manager] starting worker thread")
 
-      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['disk_manager'] = Thread.new do
         loop do
@@ -439,7 +439,7 @@ module Vmpooler
     def check_snapshot_queue
       $logger.log('d', "[*] [snapshot_manager] starting worker thread")
 
-      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['snapshot_manager'] = Thread.new do
         loop do
@@ -547,7 +547,7 @@ module Vmpooler
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads[pool['name']] = Thread.new do
         loop do

lib/vmpooler/vsphere_helper.rb

@@ -6,8 +6,9 @@ module Vmpooler
     DISK_TYPE = 'thin'
     DISK_MODE = 'persistent'
 
-    def initialize(credentials, metrics)
+    def initialize(config, metrics)
-      $credentials = credentials
+      $credentials = config[:vsphere]
+      $conf = config[:config]
       $metrics = metrics
     end
 
@@ -19,10 +20,22 @@ module Vmpooler
     end
 
     def connect_to_vsphere(credentials)
-      @connection = RbVmomi::VIM.connect host: credentials['server'],
+      max_tries = $conf['max_tries'] || 3
-                                         user: credentials['username'],
+      retry_factor = $conf['retry_factor'] || 10
-                                         password: credentials['password'],
+      try = 1
-                                         insecure: credentials['insecure'] || true
+      begin
+        @connection = RbVmomi::VIM.connect host: credentials['server'],
+                                           user: credentials['username'],
+                                           password: credentials['password'],
+                                           insecure: credentials['insecure'] || true
+        $metrics.increment("connect.open")
+      rescue => err
+        try += 1
+        $metrics.increment("connect.fail")
+        raise err if try == max_tries
+        sleep(try * retry_factor)
+        retry
+      end
     end
 
     def add_disk(vm, size, datastore)

vmpooler.yaml.example

@@ -232,6 +232,17 @@
 #     in an effort to maintain a more even distribution of load across compute resources.
 #     The migration_limit ensures that no more than n migrations will be evaluated at any one time
 #     and greatly reduces the possibilty of VMs ending up bunched together on a particular host.
+#
+#  - max_tries
+#    Set the max number of times a connection should retry in vsphere helper.
+#    This optional setting allows a user to dial in retry limits to
+#    suit your environment.
+#
+#  - retry_factor
+#    When retrying, each attempt sleeps for the try count * retry_factor.
+#    Increase this number to lengthen the delay between retry attempts.
+#    This is particularly useful for instances with a large number of pools
+#    to prevent a thundering herd when retrying connections.
 
 # Example: