(POOLER-156) Detect redis connection failures

This commit adds detection for redis connection failures to pool_manager. When a connection fails the error will be raised to executeforcing the connection to be re-established. Without this change, when a redis connection fails, it generates a redis connection error, which is swallowed by a rescue for StandardError, preventing the manager application component from recovering in the case of a redis connection failure.

lib/vmpooler/pool_manager.rb

@@ -822,6 +822,8 @@ module Vmpooler
               loop_count += 1
             end
           end
+        rescue Redis::CannotConnectError => e
+          raise
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{e}")
           raise
@@ -1310,6 +1312,9 @@ module Vmpooler
           loop_count += 1
         end
       end
+    rescue Redis::CannotConnectError => e
+      $logger.log('s', "Cannot connect to the redis server: #{e}")
+      raise
     end
   end
 end

spec/unit/pool_manager_spec.rb

@@ -2792,6 +2792,16 @@ EOT
         subject.execute!(maxloop,0)
       end
     end
+
+    context 'when redis server connection is not available' do
+      let(:maxloop) { 2 }
+      it 'should log a failure and raise the error' do
+        expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).and_raise(Redis::CannotConnectError)
+        expect(logger).to receive(:log).with('s', 'Cannot connect to the redis server: Redis::CannotConnectError')
+
+        expect{subject.execute!(maxloop,0)}.to raise_error Redis::CannotConnectError
+      end
+    end
   end
 
   describe "#sleep_with_wakeup_events" do
@@ -2987,6 +2997,18 @@ EOT
       end
     end
 
+    context 'when redis connection fails' do
+      let(:maxloop) { 2 }
+      let(:loop_delay) { 1 }
+
+      it 'should raise the error' do
+        allow(subject).to receive(:_check_pool).and_raise(Redis::CannotConnectError)
+        expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread")
+
+        expect{subject.check_pool(pool_object,maxloop,loop_delay,loop_delay)}.to raise_error Redis::CannotConnectError
+      end
+    end
+
     context 'delays between loops configured in the pool configuration' do
       let(:maxloop) { 2 }
       let(:loop_delay) { 1 }