diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a063fb..1930370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,14 +14,15 @@ git logs & PR history. # [Unreleased](https://github.com/puppetlabs/vmpooler/compare/0.1.0...master) ### Fixed - - (POOLER-128) VM specific mutex objects are not dereferenced when a VM is destroyed - A VM that is being destroyed is reported as discovered +### Added +- Adds a new mechanism to load providers from any gem or file path + # [0.1.0](https://github.com/puppetlabs/vmpooler/compare/4c858d012a262093383e57ea6db790521886d8d4...master) ### Fixed - - Remove unused method `find_pool` and related pending tests - Setting `max_tries` results in an infinite loop (POOLER-124) - Do not evaluate folders as VMs in `get_pool_vms` (POOLER-40) diff --git a/Gemfile b/Gemfile index 3a74d05..2b06870 100644 --- a/Gemfile +++ b/Gemfile @@ -18,6 +18,8 @@ end # Test deps group :test do + # required in order for the providers auto detect mechanism to work + gem 'vmpooler', path: './' gem 'mock_redis', '>= 0.17.0' gem 'rack-test', '>= 0.6' gem 'rspec', '>= 3.2' diff --git a/PROVIDER_API.md b/PROVIDER_API.md new file mode 100644 index 0000000..0210c3b --- /dev/null +++ b/PROVIDER_API.md @@ -0,0 +1,83 @@ +# Provider API + +## Create a new provider gem from scratch + +### Requirements +1. the provider code will need to be in lib/vmpooler/providers directory of your gem regardless of your gem name +2. the main provider code file should be named the same at the name of the provider. ie. (vpshere == lib/vmpooler/providers/vsphere.rb) +3. The gem must be installed on the same machine as vmpooler +4. The provider name must be referenced in the vmpooler config file in order for it to be loaded. +5. Your gem name or repository name should contain vmpooler--provider so the community can easily search provider plugins + for vmpooler. +### 1. Use bundler to create the provider scaffolding + +``` +bundler gem --test=rspec --no-exe --no-ext vmpooler-spoof-provider +cd vmpooler-providers-spoof/ +mkdir -p ./lib/vmpooler/providers +cd ./lib/vmpooler/providers +touch spoof.rb + +``` + +There may be some boilerplate files there were generated, just delete those. + +### 2. Create the main provider file +Ensure the main provider file uses the following code. + + +```ruby +# lib/vmpooler/providers/spoof.rb +require 'yaml' +require 'vmpooler/providers/base' + +module Vmpooler + class PoolManager + class Provider + class Spoof < Vmpooler::PoolManager::Provider::Base + + # at this time it is not documented which methods should be implemented + # have a look at the vmpooler/providers/vpshere provider for examples + + end + + end + end +end + + +``` + +### 3. Fill out your gemspec +Ensure you fill out your gemspec file to your specifications. If you need a dependency please make sure you require them. + +`spec.add_dependency "vmware", "~> 1.15"`. + +At a minimum you may want to add the vmpooler gem as a dev dependency so you can use it during testing. + +`spec.add_dev_dependency "vmpooler", "~> 1.15"` + +or in your Gemfile + +```ruby + +gem 'vmpooler', github: 'puppetlabs/vmpooler' +``` + +Also make sure this dependency can be loaded by jruby. If the dependency cannot be used by jruby don't use it. + +### 4. Create some tests +Your provider code should be tested before releasing. Copy and refactor some tests from the vmpooler gem under +`spec/unit/providers/dummy_spec.rb` + +### 5. Publish +Think your provider gem is good enough for others? Publish it and tell us on Slack or update this doc with a link to your gem. + + +## Available Third Party Providers +Be the first to update this list. Create a provider today! + + +## Example provider +You can use the following [repo as an example](https://github.com/logicminds/vmpooler-vsphere-provider) of how to setup your provider gem. + diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 1bbdd3d..fe1480b 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -11,7 +11,7 @@ module Vmpooler require 'yaml' require 'set' - %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib| + %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib| require "vmpooler/#{lib}" end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 1e489cb..14c9d7d 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -1,3 +1,5 @@ +require 'vmpooler/providers' + module Vmpooler class PoolManager CHECK_LOOP_DELAY_MIN_DEFAULT = 5 @@ -26,6 +28,9 @@ module Vmpooler @reconfigure_pool = {} @vm_mutex = {} + + # load specified providers from config file + load_used_providers end def config @@ -457,19 +462,36 @@ module Vmpooler result end + # load only providers used in config file + def load_used_providers + Vmpooler::Providers.load_by_name(used_providers) + end + + # @return [Array] - a list of used providers from the config file, defaults to the default providers + # ie. ["vsphere", "dummy"] + def used_providers + pools = config[:pools] || [] + @used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers ).uniq + end + + # @return [Array] - returns a list of providers that should always be loaded + # note: vsphere is the default if user does not specify although this should not be + # if vsphere is to no longer be loaded by default please remove + def default_providers + @default_providers ||= %w( vsphere dummy ) + end + def get_pool_name_for_vm(vm_name) # the 'template' is a bad name. Should really be 'poolname' $redis.hget('vmpooler__vm__' + vm_name, 'template') end + # @param pool_name [String] - the name of the pool + # @return [Provider] - returns the provider class Object def get_provider_for_pool(pool_name) - provider_name = nil - $config[:pools].each do |pool| - next unless pool['name'] == pool_name - provider_name = pool['provider'] - end - return nil if provider_name.nil? - + pool = $config[:pools].find { |pool| pool['name'] == pool_name } + return nil unless pool + provider_name = pool.fetch('provider', nil) $providers[provider_name] end diff --git a/lib/vmpooler/providers.rb b/lib/vmpooler/providers.rb index 9d2f0ae..9a5d955 100644 --- a/lib/vmpooler/providers.rb +++ b/lib/vmpooler/providers.rb @@ -1,7 +1,119 @@ -%w[base dummy vsphere].each do |lib| - begin - require "vmpooler/providers/#{lib}" - rescue LoadError - require File.expand_path(File.join(File.dirname(__FILE__), 'providers', lib)) +require 'pathname' + +module Vmpooler + class Providers + + # @param names [Array] - an array of names or string name of a provider + # @return [Array] - list of provider files loaded + # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"] + def self.load_by_name(names) + names = Array(names) + instance = self.new + names.map {|name| instance.load_from_gems(name)}.flatten + end + + # @return [Array] - array of provider files + # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"] + # although these files can come from any gem + def self.load_all_providers + self.new.load_from_gems + end + + # @return [Array] - returns an array of gem names that contain a provider + def self.installed_providers + self.new.vmpooler_provider_gem_list.map(&:name) + end + + # @return [Array] returns a list of vmpooler providers gem plugin specs + def vmpooler_provider_gem_list + gemspecs.find_all { |spec| File.directory?(File.join(spec.full_gem_path, provider_path)) } + included_lib_dirs + end + + # Internal: Find any gems containing vmpooler provider plugins and load the main file in them. + # + # @return [Array[String]] - a array of provider files + # @param name [String] - the name of the provider to load + def load_from_gems(name = nil) + paths = gem_directories.map do |gem_path| + # we don't exactly know if the provider name matches the main file name that should be loaded + # so we use globs to get everything like the name + # this could mean that vsphere5 and vsphere6 are loaded when only vsphere5 is used + Dir.glob(File.join(gem_path, "*#{name}*.rb")).each do |file| + require file + end + end + paths.flatten + end + + private + + # @return [String] - the relative path to the vmpooler provider dir + # this is used when searching gems for this path + def provider_path + File.join('lib','vmpooler','providers') + end + + # Add constants to array to skip over classes, ie. Vmpooler::PoolManager::Provider::Dummy + def excluded_classes + [] + end + + # paths to include in the search path + def included_lib_dirs + [] + end + + # returns an array of plugin classes by looking in the object space for all loaded classes + # that start with Vmpooler::PoolManager::Provider + def plugin_classes + unless @plugin_classes + load_plugins + # weed out any subclasses in the formatter + klasses = ObjectSpace.each_object(Class).find_all do |c| + c.name && c.name.split('::').count == 3 && c.name =~ /Vmpooler::PoolManager::Provider/ + end + @plugin_classes = klasses - excluded_classes || [] + end + @plugin_classes + end + + def plugin_map + @plugin_map ||= Hash[plugin_classes.map { |gem| [gem.send(:name), gem] }] + end + + + + # Internal: Retrieve a list of available gem paths from RubyGems. + # + # Returns an Array of Pathname objects. + def gem_directories + dirs = [] + if has_rubygems? + dirs = gemspecs.map do |spec| + lib_path = File.expand_path(File.join(spec.full_gem_path,provider_path)) + lib_path if File.exists? lib_path + end + included_lib_dirs + end + dirs.reject { |dir| dir.nil? }.uniq + end + + # Internal: Check if RubyGems is loaded and available. + # + # Returns true if RubyGems is available, false if not. + def has_rubygems? + defined? ::Gem + end + + # Internal: Retrieve a list of available gemspecs. + # + # Returns an Array of Gem::Specification objects. + def gemspecs + @gemspecs ||= if Gem::Specification.respond_to?(:latest_specs) + Gem::Specification.latest_specs + else + Gem.searcher.init_gemspecs + end + end + end end diff --git a/lib/vmpooler/providers/dummy.rb b/lib/vmpooler/providers/dummy.rb index 1173511..bc924a0 100644 --- a/lib/vmpooler/providers/dummy.rb +++ b/lib/vmpooler/providers/dummy.rb @@ -1,4 +1,5 @@ require 'yaml' +require 'vmpooler/providers/base' module Vmpooler class PoolManager diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb index 5fa9ad0..e139459 100644 --- a/lib/vmpooler/providers/vsphere.rb +++ b/lib/vmpooler/providers/vsphere.rb @@ -1,3 +1,5 @@ +require 'vmpooler/providers/base' + module Vmpooler class PoolManager class Provider diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 2033c2e..1f5d5da 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -46,6 +46,30 @@ EOT end end + describe '#load_used_providers' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 + provider: 'spoof' + EOT + ) + } + it do + files = ["#{project_root_dir}/lib/vmpooler/providers/vsphere.rb", + "#{project_root_dir}/lib/vmpooler/providers/dummy.rb"] + expect(subject.load_used_providers).to eq(files) + end + end + + it '#default_providers' do + expect(subject.default_providers).to eq(['vsphere', 'dummy']) + end + describe '#check_pending_vm' do before do expect(subject).not_to be_nil diff --git a/spec/unit/providers/base_spec.rb b/spec/unit/providers/base_spec.rb index 5e7feb6..5dc71c5 100644 --- a/spec/unit/providers/base_spec.rb +++ b/spec/unit/providers/base_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'vmpooler/providers/base' # This spec does not really exercise code paths but is merely used # to enforce that certain methods are defined in the base classes diff --git a/spec/unit/providers/dummy_spec.rb b/spec/unit/providers/dummy_spec.rb index f0156f9..13006ce 100644 --- a/spec/unit/providers/dummy_spec.rb +++ b/spec/unit/providers/dummy_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'vmpooler/providers/dummy' describe 'Vmpooler::PoolManager::Provider::Dummy' do let(:logger) { MockLogger.new } diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb index 4a9e90d..d09f7f4 100644 --- a/spec/unit/providers/vsphere_spec.rb +++ b/spec/unit/providers/vsphere_spec.rb @@ -1,5 +1,6 @@ require 'spec_helper' require 'mock_redis' +require 'vmpooler/providers/vsphere' RSpec::Matchers.define :relocation_spec_with_host do |value| match { |actual| actual[:spec].host == value } diff --git a/spec/unit/providers_spec.rb b/spec/unit/providers_spec.rb new file mode 100644 index 0000000..6c3b35a --- /dev/null +++ b/spec/unit/providers_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' +require 'vmpooler/providers' + +describe 'providers' do + + let(:providers) do + Vmpooler::Providers.new + end + + it '#correct class' do + expect(providers).to be_a Vmpooler::Providers + end + + it '#load_all_providers' do + p = [ + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'), + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'), + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb') + ] + expect(Vmpooler::Providers.load_all_providers).to eq(p) + end + + it '#installed_providers' do + expect(Vmpooler::Providers.installed_providers).to eq(['vmpooler']) + end + + it '#vmpooler_provider_gem_list' do + expect(providers.vmpooler_provider_gem_list).to be_a Array + expect(providers.vmpooler_provider_gem_list.first).to be_a Gem::Specification + end + + it '#load_by_name' do + expect(Vmpooler::Providers.load_by_name('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')]) + end + + it '#load only vpshere' do + expect(providers.load_from_gems('vsphere')).to eq([File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb')]) + end + + it '#load all providers from gems' do + p = [ + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'base.rb'), + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'dummy.rb'), + File.join(project_root_dir, 'lib', 'vmpooler', 'providers', 'vsphere.rb') + ] + expect(providers.load_from_gems).to eq(p) + + end + + +end