From 86aedd07545a21fba758cabc5109167070994f44 Mon Sep 17 00:00:00 2001
From: "kirby@puppetlabs.com" <mk.kirby@gmail.com>
Date: Mon, 28 Nov 2016 23:09:53 -0800
Subject: [PATCH] Add retry logic with a delay for vsphere connections

This commit adds retry logic and configurable delays to vsphere helper.
Without this change vmpooler instances that have large numbers of pools
can create enough connections in a short period of time to cause vcenter
issues.
---
 lib/vmpooler/pool_manager.rb   |  6 +++---
 lib/vmpooler/vsphere_helper.rb | 25 +++++++++++++++++++------
 vmpooler.yaml.example          | 11 +++++++++++
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
index 2c3daab..bed2289 100644
--- a/lib/vmpooler/pool_manager.rb
+++ b/lib/vmpooler/pool_manager.rb
@@ -413,7 +413,7 @@ module Vmpooler
     def check_disk_queue
       $logger.log('d', "[*] [disk_manager] starting worker thread")
 
-      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['disk_manager'] = Thread.new do
         loop do
@@ -439,7 +439,7 @@ module Vmpooler
     def check_snapshot_queue
       $logger.log('d', "[*] [snapshot_manager] starting worker thread")
 
-      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads['snapshot_manager'] = Thread.new do
         loop do
@@ -547,7 +547,7 @@ module Vmpooler
     def check_pool(pool)
       $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
 
-      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config[:vsphere], $metrics
+      $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics
 
       $threads[pool['name']] = Thread.new do
         loop do
diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb
index 946f9a8..cc93250 100644
--- a/lib/vmpooler/vsphere_helper.rb
+++ b/lib/vmpooler/vsphere_helper.rb
@@ -6,8 +6,9 @@ module Vmpooler
     DISK_TYPE = 'thin'
     DISK_MODE = 'persistent'
 
-    def initialize(credentials, metrics)
-      $credentials = credentials
+    def initialize(config, metrics)
+      $credentials = config[:vsphere]
+      $conf = config[:config]
       $metrics = metrics
     end
 
@@ -19,10 +20,22 @@ module Vmpooler
     end
 
     def connect_to_vsphere(credentials)
-      @connection = RbVmomi::VIM.connect host: credentials['server'],
-                                         user: credentials['username'],
-                                         password: credentials['password'],
-                                         insecure: credentials['insecure'] || true
+      max_tries = $conf['max_tries'] || 3
+      retry_factor = $conf['retry_factor'] || 10
+      try = 1
+      begin
+        @connection = RbVmomi::VIM.connect host: credentials['server'],
+                                           user: credentials['username'],
+                                           password: credentials['password'],
+                                           insecure: credentials['insecure'] || true
+        $metrics.increment("connect.open")
+      rescue => err
+        try += 1
+        $metrics.increment("connect.fail")
+        raise err if try == max_tries
+        sleep(try * retry_factor)
+        retry
+      end
     end
 
     def add_disk(vm, size, datastore)
diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example
index c83808c..76dcb25 100644
--- a/vmpooler.yaml.example
+++ b/vmpooler.yaml.example
@@ -232,6 +232,17 @@
 #     in an effort to maintain a more even distribution of load across compute resources.
 #     The migration_limit ensures that no more than n migrations will be evaluated at any one time
 #     and greatly reduces the possibilty of VMs ending up bunched together on a particular host.
+#
+#  - max_tries
+#    Set the max number of times a connection should retry in vsphere helper.
+#    This optional setting allows a user to dial in retry limits to
+#    suit your environment.
+#
+#  - retry_factor
+#    When retrying, each attempt sleeps for the try count * retry_factor.
+#    Increase this number to lengthen the delay between retry attempts.
+#    This is particularly useful for instances with a large number of pools
+#    to prevent a thundering herd when retrying connections.
 
 # Example: