Adds a new mechanism to load providers from any gem or file path. (#263)

* Adds ability to load only providers used in config file

CHANGELOG.md

@@ -14,14 +14,15 @@ git logs & PR history.
 # [Unreleased](https://github.com/puppetlabs/vmpooler/compare/0.1.0...master)
 
 ### Fixed
-
 - (POOLER-128) VM specific mutex objects are not dereferenced when a VM is destroyed
 - A VM that is being destroyed is reported as discovered
 
+### Added
+- Adds a new mechanism to load providers from any gem or file path 
+
 # [0.1.0](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...master)
 
 ### Fixed
-
 - Remove unused method `find_pool` and related pending tests
 - Setting `max_tries` results in an infinite loop (POOLER-124)
 - Do not evaluate folders as VMs in `get_pool_vms` (POOLER-40)

Gemfile

@@ -18,6 +18,8 @@ end
 
 # Test deps
 group :test do
+  # required in order for the providers auto detect mechanism to work
+  gem 'vmpooler', path: './'
   gem 'mock_redis', '>= 0.17.0'
   gem 'rack-test', '>= 0.6'
   gem 'rspec', '>= 3.2'

PROVIDER_API.md

@@ -0,0 +1,83 @@
+# Provider API
+
+## Create a new provider gem from scratch
+
+### Requirements
+1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name
+2. the main provider code file should be named the same at the name of the provider. ie. (vpshere == lib/vmpooler/providers/vsphere.rb)
+3. The gem must be installed on the same machine as vmpooler
+4. The provider name must be referenced in the vmpooler config file in order for it to be loaded.
+5. Your gem name or repository name should contain vmpooler-<name>-provider so the community can easily search provider plugins
+   for vmpooler.
+### 1. Use bundler to create the provider scaffolding
+
+```
+bundler gem --test=rspec --no-exe --no-ext vmpooler-spoof-provider
+cd vmpooler-providers-spoof/
+mkdir -p ./lib/vmpooler/providers
+cd ./lib/vmpooler/providers
+touch spoof.rb
+
+```
+
+There may be some boilerplate files there were generated, just delete those.
+
+### 2. Create the main provider file
+Ensure the main provider file uses the following code.
+
+
+```ruby
+# lib/vmpooler/providers/spoof.rb
+require 'yaml'
+require 'vmpooler/providers/base'
+
+module Vmpooler
+  class PoolManager
+    class Provider
+      class Spoof < Vmpooler::PoolManager::Provider::Base
+      
+      # at this time it is not documented which methods should be implemented
+      # have a look at the vmpooler/providers/vpshere provider for examples
+
+      end
+
+    end
+  end
+end
+  
+
+```
+
+### 3. Fill out your gemspec
+Ensure you fill out your gemspec file to your specifications.  If you need a dependency please make sure you require them.
+
+`spec.add_dependency "vmware", "~> 1.15"`.
+
+At a minimum you may want to add the vmpooler gem as a dev dependency so you can use it during testing.
+
+`spec.add_dev_dependency "vmpooler", "~> 1.15"`
+
+or in your Gemfile
+
+```ruby
+
+gem 'vmpooler', github: 'puppetlabs/vmpooler'
+```
+
+Also make sure this dependency can be loaded by jruby.  If the dependency cannot be used by jruby don't use it.
+
+### 4. Create some tests
+Your provider code should be tested before releasing.  Copy and refactor some tests from the vmpooler gem under 
+`spec/unit/providers/dummy_spec.rb`
+
+### 5. Publish
+Think your provider gem is good enough for others?  Publish it and tell us on Slack or update this doc with a link to your gem.
+
+
+## Available Third Party Providers
+Be the first to update this list.  Create a provider today!
+
+
+## Example provider
+You can use the following [repo as an example](https://github.com/logicminds/vmpooler-vsphere-provider) of how to setup your provider gem.
+

lib/vmpooler.rb

@@ -11,7 +11,7 @@ module Vmpooler
   require 'yaml'
   require 'set'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib|
+  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
 

lib/vmpooler/pool_manager.rb

@@ -1,3 +1,5 @@
+require 'vmpooler/providers'
+
 module Vmpooler
   class PoolManager
     CHECK_LOOP_DELAY_MIN_DEFAULT = 5
@@ -26,6 +28,9 @@ module Vmpooler
       @reconfigure_pool = {}
 
       @vm_mutex = {}
+
+      # load specified providers from config file
+      load_used_providers
     end
 
     def config
@@ -457,19 +462,36 @@ module Vmpooler
       result
     end
 
+    # load only providers used in config file
+    def load_used_providers
+      Vmpooler::Providers.load_by_name(used_providers)
+    end
+
+    # @return [Array] - a list of used providers from the config file, defaults to the default providers
+    # ie. ["vsphere", "dummy"]
+    def used_providers
+      pools = config[:pools] || []
+      @used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers ).uniq
+    end
+
+    # @return [Array] - returns a list of providers that should always be loaded
+    # note: vsphere is the default if user does not specify although this should not be
+    # if vsphere is to no longer be loaded by default please remove
+    def default_providers
+      @default_providers ||= %w( vsphere dummy )
+    end
+
     def get_pool_name_for_vm(vm_name)
       # the 'template' is a bad name.  Should really be 'poolname'
       $redis.hget('vmpooler__vm__' + vm_name, 'template')
     end
 
+    # @param pool_name [String] - the name of the pool
+    # @return [Provider] - returns the provider class Object
     def get_provider_for_pool(pool_name)
-      provider_name = nil
+      pool = $config[:pools].find { |pool| pool['name'] == pool_name }
-      $config[:pools].each do |pool|
+      return nil unless pool
-        next unless pool['name'] == pool_name
+      provider_name = pool.fetch('provider', nil)
-        provider_name = pool['provider']
-      end
-      return nil if provider_name.nil?
-
       $providers[provider_name]
     end
 

lib/vmpooler/providers.rb

@@ -1,7 +1,119 @@
-%w[base dummy vsphere].each do |lib|
+require 'pathname'
-  begin
+
-    require "vmpooler/providers/#{lib}"
+module Vmpooler
-  rescue LoadError
+  class Providers
-    require File.expand_path(File.join(File.dirname(__FILE__), 'providers', lib))
+
+    # @param names [Array] - an array of names or string name of a provider
+    # @return [Array] - list of provider files loaded
+    # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
+    def self.load_by_name(names)
+      names = Array(names)
+      instance = self.new
+      names.map {|name| instance.load_from_gems(name)}.flatten
+    end
+
+    # @return [Array] - array of provider files
+    # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
+    # although these files can come from any gem
+    def self.load_all_providers
+      self.new.load_from_gems
+    end
+
+    # @return [Array] - returns an array of gem names that contain a provider
+    def self.installed_providers
+      self.new.vmpooler_provider_gem_list.map(&:name)
+    end
+
+    # @return [Array] returns a list of vmpooler providers gem plugin specs
+    def vmpooler_provider_gem_list
+      gemspecs.find_all { |spec| File.directory?(File.join(spec.full_gem_path, provider_path)) } + included_lib_dirs
+    end
+
+    # Internal: Find any gems containing vmpooler provider plugins and load the main file in them.
+    #
+    # @return [Array[String]] - a array of provider files
+    # @param name [String] - the name of the provider to load
+    def load_from_gems(name = nil)
+      paths = gem_directories.map do |gem_path|
+        # we don't exactly know if the provider name matches the main file name that should be loaded
+        # so we use globs to get everything like the name
+        # this could mean that vsphere5 and vsphere6 are loaded when only vsphere5 is used
+        Dir.glob(File.join(gem_path, "*#{name}*.rb")).each do |file|
+          require file
+        end
+      end
+      paths.flatten
+    end
+
+    private
+
+    # @return [String] - the relative path to the vmpooler provider dir
+    # this is used when searching gems for this path
+    def provider_path
+      File.join('lib','vmpooler','providers')
+    end
+
+    # Add constants to array to skip over classes, ie. Vmpooler::PoolManager::Provider::Dummy
+    def excluded_classes
+      []
+    end
+
+    # paths to include in the search path
+    def included_lib_dirs
+      []
+    end
+
+    # returns an array of plugin classes by looking in the object space for all loaded classes
+    # that start with Vmpooler::PoolManager::Provider
+    def plugin_classes
+      unless @plugin_classes
+        load_plugins
+        # weed out any subclasses in the formatter
+        klasses = ObjectSpace.each_object(Class).find_all do |c|
+          c.name && c.name.split('::').count == 3 && c.name =~ /Vmpooler::PoolManager::Provider/
+        end
+        @plugin_classes = klasses - excluded_classes || []
+      end
+      @plugin_classes
+    end
+
+    def plugin_map
+      @plugin_map ||= Hash[plugin_classes.map { |gem| [gem.send(:name), gem] }]
+    end
+
+
+
+    # Internal: Retrieve a list of available gem paths from RubyGems.
+    #
+    # Returns an Array of Pathname objects.
+    def gem_directories
+      dirs = []
+      if has_rubygems?
+        dirs = gemspecs.map do |spec|
+          lib_path = File.expand_path(File.join(spec.full_gem_path,provider_path))
+          lib_path if File.exists? lib_path
+        end + included_lib_dirs
+      end
+      dirs.reject { |dir| dir.nil? }.uniq
+    end
+
+    # Internal: Check if RubyGems is loaded and available.
+    #
+    # Returns true if RubyGems is available, false if not.
+    def has_rubygems?
+      defined? ::Gem
+    end
+
+    # Internal: Retrieve a list of available gemspecs.
+    #
+    # Returns an Array of Gem::Specification objects.
+    def gemspecs
+      @gemspecs ||= if Gem::Specification.respond_to?(:latest_specs)
+                      Gem::Specification.latest_specs
+                    else
+                      Gem.searcher.init_gemspecs
+                    end
+    end
+
   end
 end

lib/vmpooler/providers/dummy.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'vmpooler/providers/base'
 
 module Vmpooler
   class PoolManager

lib/vmpooler/providers/vsphere.rb

@@ -1,3 +1,5 @@
+require 'vmpooler/providers/base'
+
 module Vmpooler
   class PoolManager
     class Provider

spec/unit/pool_manager_spec.rb

@@ -46,6 +46,30 @@ EOT
     end
   end
 
+  describe '#load_used_providers' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    size: 1
+    provider: 'spoof'
+    EOT
+    )
+    }
+    it do
+      files = ["#{project_root_dir}/lib/vmpooler/providers/vsphere.rb",
+               "#{project_root_dir}/lib/vmpooler/providers/dummy.rb"]
+      expect(subject.load_used_providers).to eq(files)
+    end
+  end
+
+  it '#default_providers' do
+    expect(subject.default_providers).to eq(['vsphere', 'dummy'])
+  end
+
   describe '#check_pending_vm' do
     before do
       expect(subject).not_to be_nil

spec/unit/providers/base_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'vmpooler/providers/base'
 
 # This spec does not really exercise code paths but is merely used
 # to enforce that certain methods are defined in the base classes

spec/unit/providers/dummy_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'vmpooler/providers/dummy'
 
 describe 'Vmpooler::PoolManager::Provider::Dummy' do
   let(:logger) { MockLogger.new }

spec/unit/providers/vsphere_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'mock_redis'
+require 'vmpooler/providers/vsphere'
 
 RSpec::Matchers.define :relocation_spec_with_host do |value|
   match { |actual| actual[:spec].host == value }

spec/unit/providers_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'vmpooler/providers'
+
+describe 'providers' do
+
+  let(:providers) do
+    Vmpooler::Providers.new
+  end
+
+  it '#correct class' do
+    expect(providers).to be_a Vmpooler::Providers
+  end
+
+  it '#load_all_providers' do
+    p = [
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
+    ]
+    expect(Vmpooler::Providers.load_all_providers).to eq(p)
+  end
+
+  it '#installed_providers' do
+    expect(Vmpooler::Providers.installed_providers).to eq(['vmpooler'])
+  end
+
+  it '#vmpooler_provider_gem_list' do
+    expect(providers.vmpooler_provider_gem_list).to be_a Array
+    expect(providers.vmpooler_provider_gem_list.first).to be_a Gem::Specification
+  end
+
+  it '#load_by_name' do
+    expect(Vmpooler::Providers.load_by_name('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+  end
+
+  it '#load only vpshere' do
+    expect(providers.load_from_gems('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')])
+  end
+
+  it '#load all providers from gems' do
+    p = [
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'),
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'),
+        File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')
+    ]
+    expect(providers.load_from_gems).to eq(p)
+
+  end
+
+
+end