diff --git a/Gemfile b/Gemfile index 2b06870..1ce5e86 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,7 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' gem 'json', '>= 1.8' +gem 'pickup', '~> 0.0.11' gem 'puma', '~> 3.11' gem 'rack', '~> 2.0' gem 'rake', '~> 12.3' diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index fe1480b..0b05599 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -1,15 +1,16 @@ module Vmpooler require 'date' require 'json' - require 'open-uri' require 'net/ldap' + require 'open-uri' + require 'pickup' require 'rbvmomi' require 'redis' + require 'set' require 'sinatra/base' require 'time' require 'timeout' require 'yaml' - require 'set' %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib| require "vmpooler/#{lib}" diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 9915a60..694a05e 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -36,16 +36,46 @@ module Vmpooler validate_token(backend) end - def fetch_single_vm(template) - vm = backend.spop('vmpooler__ready__' + template) - return [vm, template] if vm + def pool_backend(pool) + pool_index = pool_index(pools) + pool_config = pools[pool_index[pool]] + backend = pool_config['clone_target'] || config['clone_target'] || 'default' + return backend + end + def fetch_single_vm(template) + template_backends = [template] aliases = Vmpooler::API.settings.config[:alias] - if aliases && aliased_template = aliases[template] - vm = backend.spop('vmpooler__ready__' + aliased_template) - return [vm, aliased_template] if vm + if aliases + template_backends << aliases[template] if aliases[template] + + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + + if weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + selection = pickup.pick + template_backends.delete(selection) + template_backends.unshift(selection) + else + template_backends = template_backends.sample(template_backends.count) + end end + template_backends.each do |t| + vm = backend.spop('vmpooler__ready__' + t) + return [vm, t] if vm + end [nil, nil] end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index d42bbb2..66ac8b6 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -149,25 +149,34 @@ module Vmpooler # Periodically check that the VM is available mutex = vm_mutex(vm) return if mutex.locked? - mutex.synchronize do - check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') - return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) + begin + mutex.synchronize do + stage = 'stamp check' + check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) - $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) - # Check if the hosts TTL has expired - if ttl > 0 - # host['boottime'] may be nil if host is not powered on - if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl - $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) + $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the hosts TTL has expired + stage = 'ttl' + if ttl > 0 + # host['boottime'] may be nil if host is not powered on + if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl + $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) - $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - return + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + return + end end + + stage = 'hostname mismatch' + return if has_mismatched_hostname?(vm, pool, provider) + + stage = 'still ready' + vm_still_ready?(pool['name'], vm, provider) end - - return if has_mismatched_hostname?(vm, pool, provider) - - vm_still_ready?(pool['name'], vm, provider) + rescue => err + $logger.log('s', "Failed at stage #{stage} for #{vm}") + raise end end @@ -190,6 +199,7 @@ module Vmpooler vm_hash = provider.get_vm(pool['name'], vm) hostname = vm_hash['hostname'] + return if hostname.nil? return if hostname.empty? return if hostname == vm $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) @@ -865,12 +875,12 @@ module Vmpooler end end - def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0) + def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl) $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm| if inventory[vm] begin pool_check_response[:checked_ready_vms] += 1 - check_ready_vm(vm, pool_name, pool_ttl, provider) + check_ready_vm(vm, pool_name, pool_ttl || 0, provider) rescue => err $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}") end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb index e139459..fd95ccf 100644 --- a/lib/vmpooler/providers/vsphere.rb +++ b/lib/vmpooler/providers/vsphere.rb @@ -428,7 +428,7 @@ module Vmpooler def vm_ready?(_pool_name, vm_name) begin - open_socket(vm_name) + open_socket(vm_name, global_config[:config]['domain']) rescue => _err return false end diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 4df9609..c8b0433 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -17,6 +17,7 @@ Gem::Specification.new do |s| s.bindir = 'bin' s.executables = 'vmpooler' s.require_paths = ["lib"] + s.add_dependency 'pickup', '~> 0.0.11' s.add_dependency 'puma', '~> 3.11' s.add_dependency 'rack', '~> 2.0' s.add_dependency 'rake', '~> 12.3'