From cd73f53561acbdfd3883186fe8b603775a0d8673 Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Tue, 11 Sep 2018 11:14:51 -0700 Subject: [PATCH] (POOLER-129) Allow setting weights for backends This commit updates get_vm in the vmpooler API to allow for setting weights for backends. Additionally, when an alias for a pool exists, and the backend configured is not weighted, then the selection of the pool based on alias will be randomly sampled. Without this change any pool with the title of the alias is exhausted before an alternate pool with the configured alias is used, which results in an uneven distribution of VMs. When all backends involved are configured with weighted values the VM selection will be based on probability using those weights. A bug is fixed when setting the default ttl for check_ready_vm. Pickup is added to handle weighted VM selection. A dockerfile is added that allows for building and installing vmpooler from the current HEAD in docker to make for easy testing. --- Gemfile | 1 + lib/vmpooler.rb | 5 ++-- lib/vmpooler/api/v1.rb | 42 ++++++++++++++++++++++++----- lib/vmpooler/pool_manager.rb | 44 +++++++++++++++++++------------ lib/vmpooler/providers/vsphere.rb | 2 +- vmpooler.gemspec | 1 + 6 files changed, 69 insertions(+), 26 deletions(-) diff --git a/Gemfile b/Gemfile index 2b06870..1ce5e86 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,7 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' gem 'json', '>= 1.8' +gem 'pickup', '~> 0.0.11' gem 'puma', '~> 3.11' gem 'rack', '~> 2.0' gem 'rake', '~> 12.3' diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index fe1480b..0b05599 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -1,15 +1,16 @@ module Vmpooler require 'date' require 'json' - require 'open-uri' require 'net/ldap' + require 'open-uri' + require 'pickup' require 'rbvmomi' require 'redis' + require 'set' require 'sinatra/base' require 'time' require 'timeout' require 'yaml' - require 'set' %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib| require "vmpooler/#{lib}" diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index 9915a60..694a05e 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -36,16 +36,46 @@ module Vmpooler validate_token(backend) end - def fetch_single_vm(template) - vm = backend.spop('vmpooler__ready__' + template) - return [vm, template] if vm + def pool_backend(pool) + pool_index = pool_index(pools) + pool_config = pools[pool_index[pool]] + backend = pool_config['clone_target'] || config['clone_target'] || 'default' + return backend + end + def fetch_single_vm(template) + template_backends = [template] aliases = Vmpooler::API.settings.config[:alias] - if aliases && aliased_template = aliases[template] - vm = backend.spop('vmpooler__ready__' + aliased_template) - return [vm, aliased_template] if vm + if aliases + template_backends << aliases[template] if aliases[template] + + pool_index = pool_index(pools) + weighted_pools = {} + template_backends.each do |t| + next unless pool_index.key? t + index = pool_index[t] + clone_target = pools[index]['clone_target'] || config['clone_target'] + next unless config.key?('backend_weight') + weight = config['backend_weight'][clone_target] + if weight + weighted_pools[t] = weight + end + end + + if weighted_pools.count == template_backends.count + pickup = Pickup.new(weighted_pools) + selection = pickup.pick + template_backends.delete(selection) + template_backends.unshift(selection) + else + template_backends = template_backends.sample(template_backends.count) + end end + template_backends.each do |t| + vm = backend.spop('vmpooler__ready__' + t) + return [vm, t] if vm + end [nil, nil] end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index d42bbb2..66ac8b6 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -149,25 +149,34 @@ module Vmpooler # Periodically check that the VM is available mutex = vm_mutex(vm) return if mutex.locked? - mutex.synchronize do - check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') - return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) + begin + mutex.synchronize do + stage = 'stamp check' + check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) - $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) - # Check if the hosts TTL has expired - if ttl > 0 - # host['boottime'] may be nil if host is not powered on - if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl - $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) + $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the hosts TTL has expired + stage = 'ttl' + if ttl > 0 + # host['boottime'] may be nil if host is not powered on + if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl + $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) - $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - return + $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + return + end end + + stage = 'hostname mismatch' + return if has_mismatched_hostname?(vm, pool, provider) + + stage = 'still ready' + vm_still_ready?(pool['name'], vm, provider) end - - return if has_mismatched_hostname?(vm, pool, provider) - - vm_still_ready?(pool['name'], vm, provider) + rescue => err + $logger.log('s', "Failed at stage #{stage} for #{vm}") + raise end end @@ -190,6 +199,7 @@ module Vmpooler vm_hash = provider.get_vm(pool['name'], vm) hostname = vm_hash['hostname'] + return if hostname.nil? return if hostname.empty? return if hostname == vm $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) @@ -865,12 +875,12 @@ module Vmpooler end end - def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0) + def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl) $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm| if inventory[vm] begin pool_check_response[:checked_ready_vms] += 1 - check_ready_vm(vm, pool_name, pool_ttl, provider) + check_ready_vm(vm, pool_name, pool_ttl || 0, provider) rescue => err $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}") end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb index e139459..fd95ccf 100644 --- a/lib/vmpooler/providers/vsphere.rb +++ b/lib/vmpooler/providers/vsphere.rb @@ -428,7 +428,7 @@ module Vmpooler def vm_ready?(_pool_name, vm_name) begin - open_socket(vm_name) + open_socket(vm_name, global_config[:config]['domain']) rescue => _err return false end diff --git a/vmpooler.gemspec b/vmpooler.gemspec index 4df9609..c8b0433 100644 --- a/vmpooler.gemspec +++ b/vmpooler.gemspec @@ -17,6 +17,7 @@ Gem::Specification.new do |s| s.bindir = 'bin' s.executables = 'vmpooler' s.require_paths = ["lib"] + s.add_dependency 'pickup', '~> 0.0.11' s.add_dependency 'puma', '~> 3.11' s.add_dependency 'rack', '~> 2.0' s.add_dependency 'rake', '~> 12.3'