(POOLER-52) Add a generic connection pool

Previously VMPooler had no concept of a connection pooler.  While there is an
up to date connection pooler Gem (connection_pool), that supports MRI and jRuby,
it lacked metrics which are useful to diagnose errors and judge pool size.
This commit:

- Brings in the connection_pool gem
- Creates a new class called generic_connection_pool which inherits from the
  ConnectionPool class in the connection_pool gem.
- Extends the connection pool object with a new function called `with_metrics`
  This copies the code from the original `with` method but emits metrics for
  how long it took to get an object from the pool, and then how many objects
  are left in the pool.  This is sent using VMPooler's metrics object.
  Extending the object was used instead of overriding as it was not possible to
  inject into the existing function and monkey patching did not seem the correct
  way.

  In order use the metics, the GenericConnectionPool object modifies the
  initialize method to use :metrics and :metrics_prefix options
- Also added tests for the GenericConnectionPool class to ensure the new
  functions are tested.  Note that the functionality that was not extended is
  not tested in VMPooler.

Gemfile

@@ -16,6 +16,7 @@ gem 'redis', '>= 3.2'
 gem 'sinatra', '>= 1.4'
 gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3
 gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd'
+gem 'connection_pool', '>= 2.2.1'
 
 # Test deps
 group :test do

lib/vmpooler.rb

@@ -12,7 +12,7 @@ module Vmpooler
   require 'yaml'
   require 'set'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd providers].each do |lib|
+  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib|
     begin
       require "vmpooler/#{lib}"
     rescue LoadError

lib/vmpooler/generic_connection_pool.rb

@@ -0,0 +1,53 @@
+require 'connection_pool'
+
+module Vmpooler
+  class PoolManager
+    class GenericConnectionPool < ConnectionPool
+      # Extend the ConnectionPool class with instrumentation
+      # https://github.com/mperham/connection_pool/blob/master/lib/connection_pool.rb
+
+      def initialize(options = {}, &block)
+        super(options, &block)
+        @metrics = options[:metrics]
+        @metric_prefix = options[:metric_prefix]
+        @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
+      end
+
+      if Thread.respond_to?(:handle_interrupt)
+        # MRI
+        def with_metrics(options = {})
+          Thread.handle_interrupt(Exception => :never) do
+            start = Time.now
+            conn = checkout(options)
+            timespan_ms = ((Time.now - start) * 1000).to_i
+            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+            @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
+            begin
+              Thread.handle_interrupt(Exception => :immediate) do
+                yield conn
+              end
+            ensure
+              checkin
+              @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+            end
+          end
+        end
+      else
+        # jruby 1.7.x
+        def with_metrics(options = {})
+          start = Time.now
+          conn = checkout(options)
+          timespan_ms = ((Time.now - start) * 1000).to_i
+          @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+          @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil?
+          begin
+            yield conn
+          ensure
+            checkin
+            @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil?
+          end
+        end
+      end
+    end
+  end
+end

spec/unit/generic_connection_pool_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe 'GenericConnectionPool' do
+  let(:metrics) { Vmpooler::DummyStatsd.new }
+  let(:metric_prefix) { 'prefix' }
+  let(:default_metric_prefix) { 'connectionpool' }
+  let(:connection_object) { double('connection') }
+  let(:pool_size) { 1 }
+  let(:pool_timeout) { 1 }
+
+  subject { Vmpooler::PoolManager::GenericConnectionPool.new(
+              metrics: metrics,
+              metric_prefix: metric_prefix,
+              size: pool_size,
+              timeout: pool_timeout
+            ) { connection_object }
+  }
+
+
+  describe "#with_metrics" do
+    before(:each) do
+      expect(subject).not_to be_nil
+    end
+
+    context 'When metrics are configured' do
+      it 'should emit a gauge metric when the connection is grabbed and released' do
+        expect(metrics).to receive(:gauge).with(/\.available/,Integer).exactly(2).times
+
+        subject.with_metrics do |conn1|
+          # do nothing
+        end
+      end
+
+      it 'should emit a timing metric when the connection is grabbed' do
+        expect(metrics).to receive(:timing).with(/\.waited/,Integer).exactly(1).times
+
+        subject.with_metrics do |conn1|
+          # do nothing
+        end
+      end
+
+      it 'should emit metrics with the specified prefix' do
+        expect(metrics).to receive(:gauge).with(/#{metric_prefix}\./,Integer).at_least(1).times
+        expect(metrics).to receive(:timing).with(/#{metric_prefix}\./,Integer).at_least(1).times
+
+        subject.with_metrics do |conn1|
+          # do nothing
+        end
+      end
+
+      context 'Metrix prefix is missing' do
+        let(:metric_prefix) { nil }
+
+        it 'should emit metrics with default prefix' do
+          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+
+          subject.with_metrics do |conn1|
+            # do nothing
+          end
+        end
+      end
+
+      context 'Metrix prefix is empty' do
+        let(:metric_prefix) { '' }
+
+        it 'should emit metrics with default prefix' do
+          expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+          expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times
+
+          subject.with_metrics do |conn1|
+            # do nothing
+          end
+        end
+      end
+    end
+
+    context 'When metrics are not configured' do
+      let(:metrics) { nil }
+
+      it 'should not emit any metrics' do
+        # if any metrics are called it would result in a method error on Nil.
+
+        subject.with_metrics do |conn1|
+          # do nothing
+        end
+      end
+    end
+
+  end
+end