Merge pull request #281 from mattkirby/pooler_109

(POOLER-109) Allow API to run independently

Dockerfile

@@ -1,28 +0,0 @@
-# Run vmpooler in a Docker container!  Configuration can either be embedded
-# and built within the current working directory, or stored in a
-# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
-#
-# BUILD:
-#   docker build -t vmpooler .
-#
-# RUN:
-#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
-
-FROM jruby:9.1-jdk
-
-RUN mkdir -p /var/lib/vmpooler
-
-WORKDIR /var/lib/vmpooler
-
-ADD Gemfile* /var/lib/vmpooler/
-RUN bundle install --system
-
-RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby
-
-COPY . /var/lib/vmpooler
-
-ENV VMPOOLER_LOG /var/log/vmpooler.log
-CMD \
-    /var/lib/vmpooler/scripts/vmpooler_init.sh start \
-    && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \
-    tail -f ${VMPOOLER_LOG}

Gemfile

@@ -1,29 +1,16 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
-gem 'puma', '>= 3.6.0'
-# Rack 2.x requires ruby 2.2 or above.
-# As VMPooler should work in older jruby, we need to be Ruby 1.9.3 compatible.
-gem 'rack', '~> 1.6'
-gem 'rake', '>= 10.4'
-gem 'rbvmomi', '>= 1.8'
-gem 'sinatra', '>= 1.4'
-gem 'net-ldap', '>= 0.16.1'
-gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd'
-gem 'connection_pool', '>= 2.2.1'
-gem 'nokogiri', '>= 1.8.2'
-gem 'vmpooler', path: './'
-# Pin gems against Ruby version
-# Note we can't use platform restrictions easily so use
-# lowest version range any platform
-# ----
-# redis
-if RUBY_VERSION =~ /^2\.[1]/
-  gem 'redis', '~> 3.0'
-elsif RUBY_VERSION =~ /^2\.2\.[01]/
-  gem 'redis', '~> 3.0'
-else
-  gem 'redis', '>= 3.2'
-end
+gem 'json', '>= 1.8'
+gem 'puma', '~> 3.11'
+gem 'rack', '~> 2.0'
+gem 'rake', '~> 12.3'
+gem 'redis', '~> 4.0'
+gem 'rbvmomi', '~> 1.13'
+gem 'sinatra', '~> 2.0'
+gem 'net-ldap', '~> 0.16'
+gem 'statsd-ruby', '~> 1.4.0', :require => 'statsd'
+gem 'connection_pool', '~> 2.2'
+gem 'nokogiri', '~> 1.8'
 
 group :development do
   gem 'pry'

bin/vmpooler

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rubygems' unless defined?(Gem)
+require 'lib/vmpooler'
+
+config = Vmpooler.config
+redis_host = config[:redis]['server']
+redis_port = config[:redis]['port']
+redis_password = config[:redis]['password']
+logger_file = config[:config]['logfile']
+api_logger_file = config[:config]['api_logfile']
+
+metrics = Vmpooler.new_metrics(config)
+
+torun_threads = []
+if ARGV.count == 0
+  torun = ['api', 'manager']
+else
+  torun = []
+  torun << 'api' if ARGV.include? 'api'
+  torun << 'manager' if ARGV.include? 'manager'
+  exit(2) if torun.empty?
+end
+
+if torun.include? 'api'
+  api = Thread.new do
+    thr = Vmpooler::API.new
+    redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
+    thr.helpers.configure(config, redis, metrics)
+    thr.helpers.execute!
+  end
+  torun_threads << api
+end
+
+if torun.include? 'manager'
+  manager = Thread.new do
+    Vmpooler::PoolManager.new(
+      config,
+      Vmpooler.new_logger(logger_file),
+      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      metrics
+    ).execute!
+  end
+  torun_threads << manager
+end
+
+if ENV['VMPOOLER_DEBUG']
+  trap('INT') do
+    puts 'Shutting down.'
+    torun_threads.each(&:exit)
+  end
+end
+
+torun_threads.each do |th|
+  th.join
+end

docker/Dockerfile

@@ -0,0 +1,21 @@
+# Run vmpooler in a Docker container!  Configuration can either be embedded
+# and built within the current working directory, or stored in a
+# VMPOOLER_CONFIG environment value and passed to the Docker daemon.
+#
+# BUILD:
+#   docker build -t vmpooler .
+#
+# RUN:
+#   docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
+
+FROM jruby:9.1-jdk
+
+COPY ./docker/docker-entrypoint.sh /usr/local/bin/
+
+ENV LOGFILE=/dev/stdout \
+      RACK_ENV=production
+
+RUN gem install vmpooler && \
+      chmod +x /usr/local/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["docker-entrypoint.sh"]

docker/docker-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+set -- /var/lib/vmpooler/vmpooler "$@"
+
+exec "$@"

lib/vmpooler.rb

@@ -1,5 +1,3 @@
-require 'rubygems' unless defined?(Gem)
-
 module Vmpooler
   require 'date'
   require 'json'
@@ -14,11 +12,7 @@ module Vmpooler
   require 'set'
 
   %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib|
-    begin
-      require "vmpooler/#{lib}"
-    rescue LoadError
-      require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
-    end
+    require "vmpooler/#{lib}"
   end
 
   def self.config(filepath = 'vmpooler.yaml')
@@ -31,36 +25,78 @@ module Vmpooler
     else
       # Take the name of the config file either from an ENV variable or from the filepath argument
       config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
-      parsed_config = YAML.load_file(config_file)
+      parsed_config = YAML.load_file(config_file) if File.exist? config_file
     end
 
-    exit unless parsed_config
+    parsed_config ||= { config: {} }
 
     # Bail out if someone attempts to start vmpooler with dummy authentication
     # without enbaling debug mode.
-    if parsed_config[:auth]['provider'] == 'dummy'
-      unless ENV['VMPOOLER_DEBUG']
-        warning = [
-          'Dummy authentication should not be used outside of debug mode',
-          'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
-        ]
+    if parsed_config.has_key? :auth
+      if parsed_config[:auth]['provider'] == 'dummy'
+        unless ENV['VMPOOLER_DEBUG']
+          warning = [
+            'Dummy authentication should not be used outside of debug mode',
+            'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
+          ]
 
-        raise warning.join(";\s")
+          raise warning.join(";\s")
+        end
       end
     end
 
     # Set some configuration defaults
-    parsed_config[:redis]             ||= {}
-    parsed_config[:redis]['server']   ||= 'localhost'
-    parsed_config[:redis]['data_ttl'] ||= 168
+    parsed_config[:config]['task_limit'] = ENV['TASK_LIMIT'] || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['migration_limit'] = ENV['MIGRATION_LIMIT'] if ENV['MIGRATION_LIMIT']
+    parsed_config[:config]['vm_checktime'] = ENV['VM_CHECKTIME'] || parsed_config[:config]['vm_checktime'] || 15
+    parsed_config[:config]['vm_lifetime'] = ENV['VM_LIFETIME'] || parsed_config[:config]['vm_lifetime']  || 24
+    parsed_config[:config]['prefix'] = ENV['VM_PREFIX'] || parsed_config[:config]['prefix'] || ''
 
-    parsed_config[:config]['task_limit']   ||= 10
-    parsed_config[:config]['vm_checktime'] ||= 15
-    parsed_config[:config]['vm_lifetime']  ||= 24
-    parsed_config[:config]['prefix']       ||= ''
+    parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
+
+    parsed_config[:config]['site_name'] = ENV['SITE_NAME'] if ENV['SITE_NAME']
+    parsed_config[:config]['domain'] = ENV['DOMAIN_NAME'] if ENV['DOMAIN_NAME']
+    parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
+    parsed_config[:config]['timeout'] = ENV['TIMEOUT'] if ENV['TIMEOUT']
+    parsed_config[:config]['vm_lifetime_auth'] = ENV['VM_LIFETIME_AUTH'] if ENV['VM_LIFETIME_AUTH']
+    parsed_config[:config]['ssh_key'] = ENV['SSH_KEY'] if ENV['SSH_KEY']
+    parsed_config[:config]['max_tries'] = ENV['MAX_TRIES'] if ENV['MAX_TRIES']
+    parsed_config[:config]['retry_factor'] = ENV['RETRY_FACTOR'] if ENV['RETRY_FACTOR']
+    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+
+    parsed_config[:redis] = parsed_config[:redis] || {}
+    parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
+    parsed_config[:redis]['port'] = ENV['REDIS_PORT'] if ENV['REDIS_PORT']
+    parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
+    parsed_config[:redis]['data_ttl'] = ENV['REDIS_DATA_TTL'] || parsed_config[:redis]['data_ttl'] || 168
+
+    parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
+    parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
+    parsed_config[:statsd]['prefix'] = ENV['STATSD_PREFIX'] if ENV['STATSD_PREFIX']
+    parsed_config[:statsd]['port'] = ENV['STATSD_PORT'] if ENV['STATSD_PORT']
+
+    parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
+
+    parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
+    if parsed_config.has_key? :auth
+      parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
+      parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
+      parsed_config[:auth][:ldap]['server'] = ENV['LDAP_SERVER'] if ENV['LDAP_SERVER']
+      parsed_config[:auth][:ldap]['port'] = ENV['LDAP_PORT'] if ENV['LDAP_PORT']
+      parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
+      parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
+    end
 
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
+    unless parsed_config[:pools]
+      redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
+      parsed_config[:pools] = load_pools_from_redis(redis)
+    end
+
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
       if pool['alias']
@@ -84,9 +120,23 @@ module Vmpooler
     end
 
     parsed_config[:uptime] = Time.now
+
     parsed_config
   end
 
+  def self.load_pools_from_redis(redis)
+    pools = []
+    redis.smembers('vmpooler__pools').each do |pool|
+      pool_hash = {}
+      redis.hgetall("vmpooler__pool__#{pool}").each do |k, v|
+        pool_hash[k] = v
+      end
+      pool_hash['alias'] = pool_hash['alias'].split(',')
+      pools << pool_hash
+    end
+    pools
+  end
+
   def self.new_redis(host = 'localhost', port = nil, password = nil)
     Redis.new(host: host, port: port, password: password)
   end

lib/vmpooler/api.rb

@@ -4,8 +4,6 @@ module Vmpooler
       super
     end
 
-    set :environment, :production
-
     not_found do
       content_type :json
 
@@ -42,11 +40,10 @@ module Vmpooler
     use Vmpooler::API::Reroute
     use Vmpooler::API::V1
 
-    def configure(config, redis, metrics, environment = :production)
+    def configure(config, redis, metrics)
       self.settings.set :config, config
       self.settings.set :redis, redis
       self.settings.set :metrics, metrics
-      self.settings.set :environment, environment
     end
 
     def execute!

lib/vmpooler/dashboard.rb

@@ -1,8 +1,13 @@
 module Vmpooler
   class Dashboard < Sinatra::Base
+
+    def config
+      Vmpooler.config
+    end
+
     get '/dashboard/?' do
       erb :dashboard, locals: {
-        site_name: $config[:config]['site_name'] || '<b>vmpooler</b>'
+        site_name: ENV['SITE_NAME'] || config[:config]['site_name'] || '<b>vmpooler</b>'
       }
     end
   end

lib/vmpooler/pool_manager.rb

@@ -32,6 +32,31 @@ module Vmpooler
       $config
     end
 
+    # Place pool configuration in redis so an API instance can discover running pool configuration
+    def load_pools_to_redis
+      previously_configured_pools = $redis.smembers('vmpooler__pools')
+      currently_configured_pools = []
+      config[:pools].each do |pool|
+        currently_configured_pools << pool['name']
+        $redis.sadd('vmpooler__pools', pool['name'])
+        pool_keys = pool.keys
+        pool_keys.delete('alias')
+        to_set = {}
+        pool_keys.each do |k|
+          to_set[k] = pool[k]
+        end
+        to_set['alias'] = pool['alias'].join(',') if to_set.has_key?('alias')
+        $redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
+      end
+      previously_configured_pools.each do |pool|
+        unless currently_configured_pools.include? pool
+          $redis.srem('vmpooler__pools', pool)
+          $redis.del("vmpooler__pool__#{pool}")
+        end
+      end
+      return
+    end
+
     # Check the state of a VM
     def check_pending_vm(vm, pool, timeout, provider)
       Thread.new do
@@ -927,6 +952,9 @@ module Vmpooler
         end
       end
 
+      # Load running pool configuration into redis so API server can retrieve it
+      load_pools_to_redis
+
       # Get pool loop settings
       $config[:config] = {} if $config[:config].nil?
       check_loop_delay_min = $config[:config]['check_loop_delay_min'] || CHECK_LOOP_DELAY_MIN_DEFAULT

spec/integration/dashboard_spec.rb

@@ -22,12 +22,10 @@ describe Vmpooler::API do
     end
 
     context '/dashboard/' do
-      let(:config) { {
-          config: {'site_name' => 'test pooler'}
-      } }
+      ENV['SITE_NAME'] = 'test pooler'
+      ENV['VMPOOLER_CONFIG'] = 'thing'
 
       before do
-        $config = config
         get '/dashboard/'
       end
 

vmpooler

@@ -1,38 +0,0 @@
-#!/usr/bin/env ruby
-
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'rubygems' unless defined?(Gem)
-require 'lib/vmpooler'
-
-config = Vmpooler.config
-redis_host = config[:redis]['server']
-redis_port = config[:redis]['port']
-redis_password = config[:redis]['password']
-logger_file = config[:config]['logfile']
-
-metrics = Vmpooler.new_metrics(config)
-
-api = Thread.new do
-  thr = Vmpooler::API.new
-  thr.helpers.configure(config, Vmpooler.new_redis(redis_host, redis_port, redis_password), metrics)
-  thr.helpers.execute!
-end
-
-manager = Thread.new do
-  Vmpooler::PoolManager.new(
-    config,
-    Vmpooler.new_logger(logger_file),
-    Vmpooler.new_redis(redis_host, redis_port, redis_password),
-    metrics
-  ).execute!
-end
-
-if ENV['VMPOOLER_DEBUG']
-  trap('INT') do
-    puts 'Shutting down.'
-    [api, manager].each(&:exit)
-  end
-end
-
-[api, manager].each(&:join)

vmpooler.gemspec

@@ -2,33 +2,29 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'vmpooler/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = 'vmpooler'
-  spec.version       = Vmpooler::VERSION
-  spec.authors       = ['Puppet']
-  spec.email         = ['support@puppet.com']
+Gem::Specification.new do |s|
+  s.name          = 'vmpooler'
+  s.version       = Vmpooler::VERSION
+  s.authors       = ['Puppet']
+  s.email         = ['support@puppet.com']
 
-  spec.summary       = 'vmpooler provides configurable pools of instantly-available (running) virtual machines'
-  spec.description   = 'vmpooler provides configurable pools of instantly-available (running) virtual machines'
-  spec.homepage      = 'https://github.com/puppetlabs/vmpooler'
-  spec.license       = 'MIT'
-
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
-  spec.add_dependency 'puma', '>= 3.6.0'
-  spec.add_dependency 'rack', '~> 1.6'
-  spec.add_dependency 'rake', '>= 10.4'
-  spec.add_dependency 'rbvmomi', '>= 1.8'
-  spec.add_dependency 'sinatra', '>= 1.4'
-  spec.add_dependency 'net-ldap', '>= 0.16.1'
-  spec.add_dependency 'statsd-ruby', '>= 1.3.0'
-  spec.add_dependency 'connection_pool', '>= 2.2.1'
-  spec.add_dependency 'nokogiri', '>= 1.8.2'
-  # we should lock ruby support down to 2.2.2+ and update redis version 3.2
-  spec.add_dependency 'redis', '>= 3.0'
+  s.summary       = 'vmpooler provides configurable pools of instantly-available (running) virtual machines'
+  s.homepage      = 'https://github.com/puppetlabs/vmpooler'
+  s.license       = 'Apache-2.0'
+  s.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
+  s.files         = Dir[ "bin/*", "lib/**/*" ]
+  s.bindir        = 'bin'
+  s.executables   = 'vmpooler'
+  s.require_paths = ["lib"]
+  s.add_dependency 'puma', '~> 3.11'
+  s.add_dependency 'rack', '~> 2.0'
+  s.add_dependency 'rake', '~> 12.3'
+  s.add_dependency 'redis', '~> 4.0'
+  s.add_dependency 'rbvmomi', '~> 1.13'
+  s.add_dependency 'sinatra', '~> 2.0'
+  s.add_dependency 'net-ldap', '~> 0.16'
+  s.add_dependency 'statsd-ruby', '~> 1.4'
+  s.add_dependency 'connection_pool', '~> 2.2'
+  s.add_dependency 'nokogiri', '~> 1.8'
 end