(POOLER-129) Allow setting weights for backends

This commit updates get_vm in the vmpooler API to allow for setting weights for backends. Additionally, when an alias for a pool exists, and the backend configured is not weighted, then the selection of the pool based on alias will be randomly sampled. Without this change any pool with the title of the alias is exhausted before an alternate pool with the configured alias is used, which results in an uneven distribution of VMs. When all backends involved are configured with weighted values the VM selection will be based on probability using those weights.

A bug is fixed when setting the default ttl for check_ready_vm.

Pickup is added to handle weighted VM selection.

A dockerfile is added that allows for building and installing vmpooler
from the current HEAD in docker to make for easy testing.

Gemfile

@@ -1,6 +1,7 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
 gem 'json', '>= 1.8'
+gem 'pickup', '~> 0.0.11'
 gem 'puma', '~> 3.11'
 gem 'rack', '~> 2.0'
 gem 'rake', '~> 12.3'

lib/vmpooler.rb

@@ -1,15 +1,16 @@
 module Vmpooler
   require 'date'
   require 'json'
-  require 'open-uri'
   require 'net/ldap'
+  require 'open-uri'
+  require 'pickup'
   require 'rbvmomi'
   require 'redis'
+  require 'set'
   require 'sinatra/base'
   require 'time'
   require 'timeout'
   require 'yaml'
-  require 'set'
 
   %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"

lib/vmpooler/api/v1.rb

@@ -36,16 +36,46 @@ module Vmpooler
       validate_token(backend)
     end
 
-    def fetch_single_vm(template)
-      vm = backend.spop('vmpooler__ready__' + template)
-      return [vm, template] if vm
-
-      aliases = Vmpooler::API.settings.config[:alias]
-      if aliases && aliased_template = aliases[template]
-        vm = backend.spop('vmpooler__ready__' + aliased_template)
-        return [vm, aliased_template] if vm
+    def pool_backend(pool)
+      pool_index = pool_index(pools)
+      pool_config = pools[pool_index[pool]]
+      backend = pool_config['clone_target'] || config['clone_target'] || 'default'
+      return backend
     end
 
+    def fetch_single_vm(template)
+      template_backends = [template]
+      aliases = Vmpooler::API.settings.config[:alias]
+      if aliases
+        template_backends << aliases[template] if aliases[template]
+
+        pool_index = pool_index(pools)
+        weighted_pools = {}
+        template_backends.each do |t|
+          next unless pool_index.key? t
+          index = pool_index[t]
+          clone_target = pools[index]['clone_target'] || config['clone_target']
+          next unless config.key?('backend_weight')
+          weight = config['backend_weight'][clone_target]
+          if weight
+            weighted_pools[t] = weight
+          end
+        end
+
+        if weighted_pools.count == template_backends.count
+          pickup = Pickup.new(weighted_pools)
+          selection = pickup.pick
+          template_backends.delete(selection)
+          template_backends.unshift(selection)
+        else
+          template_backends = template_backends.sample(template_backends.count)
+        end
+      end
+
+      template_backends.each do |t|
+        vm = backend.spop('vmpooler__ready__' + t)
+        return [vm, t] if vm
+      end
       [nil, nil]
     end
 

lib/vmpooler/pool_manager.rb

@@ -149,12 +149,15 @@ module Vmpooler
       # Periodically check that the VM is available
       mutex = vm_mutex(vm)
       return if mutex.locked?
+      begin
         mutex.synchronize do
+          stage = 'stamp check'
           check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
           return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
 
           $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
           # Check if the hosts TTL has expired
+          stage = 'ttl'
           if ttl > 0
             # host['boottime'] may be nil if host is not powered on
             if ((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
@@ -165,10 +168,16 @@ module Vmpooler
             end
           end
 
+          stage = 'hostname mismatch'
           return if has_mismatched_hostname?(vm, pool, provider)
 
+          stage = 'still ready'
           vm_still_ready?(pool['name'], vm, provider)
         end
+      rescue => err
+        $logger.log('s', "Failed at stage #{stage} for #{vm}")
+        raise
+      end
     end
 
     def has_mismatched_hostname?(vm, pool, provider)
@@ -190,6 +199,7 @@ module Vmpooler
       vm_hash = provider.get_vm(pool['name'], vm)
       hostname = vm_hash['hostname']
 
+      return if hostname.nil?
       return if hostname.empty?
       return if hostname == vm
       $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
@@ -865,12 +875,12 @@ module Vmpooler
       end
     end
 
-    def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
+    def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
       $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             pool_check_response[:checked_ready_vms] += 1
-            check_ready_vm(vm, pool_name, pool_ttl, provider)
+            check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
           rescue => err
             $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}")
           end

lib/vmpooler/providers/vsphere.rb

@@ -428,7 +428,7 @@ module Vmpooler
 
         def vm_ready?(_pool_name, vm_name)
           begin
-            open_socket(vm_name)
+            open_socket(vm_name, global_config[:config]['domain'])
           rescue => _err
             return false
           end

vmpooler.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.bindir        = 'bin'
   s.executables   = 'vmpooler'
   s.require_paths = ["lib"]
+  s.add_dependency 'pickup', '~> 0.0.11'
   s.add_dependency 'puma', '~> 3.11'
   s.add_dependency 'rack', '~> 2.0'
   s.add_dependency 'rake', '~> 12.3'