From 58774b9d7e28b883768749a71222ea8903859b28 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Tue, 24 Jan 2017 14:37:36 -0800 Subject: [PATCH 01/23] (maint) Update Gemfile and gitignore Previously, a bundle install would not pull in gems from Gemfile.local or ~/.gemfile which are common development workflows in Puppet. This commit modifies the Gemfile to pull in these additional gemfiles if they exist. This commit also adds common files and folders to gitignore which should not be committed to this repository. --- .gitignore | 4 ++++ Gemfile | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index 835dcd0..99789c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .ruby-version Gemfile.lock +Gemfile.local vendor +vmpooler.yaml +.bundle +coverage diff --git a/Gemfile b/Gemfile index 4fef816..975f8fd 100644 --- a/Gemfile +++ b/Gemfile @@ -23,3 +23,13 @@ group :test do gem 'simplecov', '>= 0.11.2' gem 'yarjuf', '>= 2.0' end + +# Evaluate Gemfile.local if it exists +if File.exists? "#{__FILE__}.local" + eval(File.read("#{__FILE__}.local"), binding) +end + +# Evaluate ~/.gemfile if it exists +if File.exists?(File.join(Dir.home, '.gemfile')) + eval(File.read(File.join(Dir.home, '.gemfile')), binding) +end From 380d4bd39daae5261097f57256968019dc0eed7e Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Mon, 28 Nov 2016 15:14:39 -0800 Subject: [PATCH 02/23] (maint) Enable Ctrl-C to kill all threads in developer environment Previously, if you ran the vpooler via ruby, pressing Ctrl-C would terminate the Webserver however the PoolManager does not have a handler and would instead just keep executing. This commit adds a global Ctrl-C hook which terminates both the api and manager threads. This behaviour will only be enabled if the `VMPOOLER_DEBUG` environment variable exists so that it does not affect VMPooler when running in production environments. --- vmpooler | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vmpooler b/vmpooler index 20eba53..fd8d557 100755 --- a/vmpooler +++ b/vmpooler @@ -26,4 +26,11 @@ manager = Thread.new { ).execute! } +if ENV['VMPOOLER_DEBUG'] + trap("INT") { + puts "Shutting down." + [api, manager].each { |t| t.exit } + } +end + [api, manager].each { |t| t.join } From 48ed24a0de9b9d2a4aa1c6b2f02c7710d463fa1d Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Mon, 28 Nov 2016 15:29:32 -0800 Subject: [PATCH 03/23] (maint) Add VMPOOLER_CONFIG environment variable to change config file Previously there was no way to use a different configuration file when using the `./vmpooler` ruby file. This commit will use the content of the `VMPOOLER_CONFIG` environment variable, or default to `vmpooler.yaml` when loading the vmpooler configuration. --- vmpooler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmpooler b/vmpooler index fd8d557..30fe321 100755 --- a/vmpooler +++ b/vmpooler @@ -5,7 +5,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) require 'rubygems' unless defined?(Gem) require 'lib/vmpooler' -config = Vmpooler.config +config = Vmpooler.config(ENV['VMPOOLER_CONFIG'] || 'vmpooler.yaml') redis_host = config[:redis]['server'] logger_file = config[:config]['logfile'] From d962886cf810ec11553b22c80c6ba5fa8df7c609 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 15:05:38 -0800 Subject: [PATCH 04/23] (POOLER-71) Add dummy authentication provider Previously it was difficult to do local development as VMPooler requires an LDAP service for authentication. This commit adds a dummy authentication provider. The provider has passes authentication if the username and password are different, and fails if the username and password are the same. This commit also updates the documentation in the config YML file. --- lib/vmpooler/api/helpers.rb | 2 ++ vmpooler.yaml.example | 27 ++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 47c9062..0bd6275 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -56,6 +56,8 @@ module Vmpooler def authenticate(auth, username_str, password_str) case auth['provider'] + when 'dummy' + return (username_str != password_str) when 'ldap' require 'rubygems' require 'net/ldap' diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 76dcb25..c128b5d 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -137,8 +137,22 @@ # This section contains information related to authenticating users # for token operations. # -# Currently the only supported provider is LDAP; the following parameters -# will all be under an ':ldap:' subsection (see example below). +# Supported Auth Providers: +# - Dummy +# - LDAP +# +# - Dummy Auth Provider +# The Dummy Authentication provider should only be used during development or testing +# If the Username and Password are different then validation succeeds +# If the Username and Password are the same then validation fails +# +# Example: +# :auth: +# provider: 'dummy' +# +# - LDAP Auth Provider +# The LDAP Authentication provider will validate usernames and passwords against an +# existing LDAP service # # Available configuration parameters: # @@ -154,8 +168,15 @@ # # - user_object # The LDAP object-type used to designate a user object. - +# # Example: +# :auth: +# provider: 'ldap' +# :ldap: +# host: 'localhost' +# port: 389 +# base: 'ou=users,dc=company,dc=com' +# user_object: 'uid' :auth: provider: 'ldap' From 626195685fbde6021d86740251e6daf555c010da Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 15:09:00 -0800 Subject: [PATCH 05/23] (maint) Output logging to STDOUT when debugging Previously log messages would only go to the log file. This commit adds additional functionality to the logger by output to STDOUT if the environment variable VMPOOLER_DEBUG is set. --- lib/vmpooler/logger.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/vmpooler/logger.rb b/lib/vmpooler/logger.rb index 2c70756..8d1d52e 100644 --- a/lib/vmpooler/logger.rb +++ b/lib/vmpooler/logger.rb @@ -11,9 +11,11 @@ module Vmpooler def log(_level, string) time = Time.new stamp = time.strftime('%Y-%m-%d %H:%M:%S') - open(@file, 'a') do |f| f.puts "[#{stamp}] #{string}" + if ENV['VMPOOLER_DEBUG'] + puts "[#{stamp}] #{string}" + end end end end From 05f781ab69aed21ceddfc60f9f3bef4a03f78264 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 15:25:06 -0800 Subject: [PATCH 06/23] WIP (POOLER-70) Refactor out VSphere to separate class WIP --- lib/vmpooler.rb | 4 +- lib/vmpooler/backingservice.rb | 8 + lib/vmpooler/backingservice/base.rb | 102 +++ lib/vmpooler/backingservice/vsphere.rb | 622 ++++++++++++++++++ lib/vmpooler/pool_manager.rb | 526 +++++++++------- lib/vmpooler/vsphere_helper.rb | 834 ++++++++++++------------- 6 files changed, 1443 insertions(+), 653 deletions(-) create mode 100644 lib/vmpooler/backingservice.rb create mode 100644 lib/vmpooler/backingservice/base.rb create mode 100644 lib/vmpooler/backingservice/vsphere.rb diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 844aa6e..f3b565f 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -12,7 +12,7 @@ module Vmpooler require 'yaml' require 'set' - %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib| + %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd backingservice ).each do |lib| begin require "vmpooler/#{lib}" rescue LoadError @@ -21,7 +21,7 @@ module Vmpooler end def self.config(filepath='vmpooler.yaml') - parsed_config = {} + parsed_config = {} if ENV['VMPOOLER_CONFIG'] # Load configuration from ENV diff --git a/lib/vmpooler/backingservice.rb b/lib/vmpooler/backingservice.rb new file mode 100644 index 0000000..75645c3 --- /dev/null +++ b/lib/vmpooler/backingservice.rb @@ -0,0 +1,8 @@ +# TODO remove dummy for commit history +%w( base vsphere dummy ).each do |lib| + begin + require "vmpooler/backingservice/#{lib}" + rescue LoadError + require File.expand_path(File.join(File.dirname(__FILE__), 'backingservice', lib)) + end +end diff --git a/lib/vmpooler/backingservice/base.rb b/lib/vmpooler/backingservice/base.rb new file mode 100644 index 0000000..a977a31 --- /dev/null +++ b/lib/vmpooler/backingservice/base.rb @@ -0,0 +1,102 @@ +module Vmpooler + class PoolManager + class BackingService + class Base + # These defs must be overidden in child classes + + def initialize(options) + end + + #def validate_config(config) + # false + #end + + # inputs + # pool : hashtable from config file + # returns + # hashtable + # name : name of the device + def vms_in_pool(pool) + fail "#{self.class.name} does not implement vms_in_pool" + end + + # inputs + # vm_name: string + # returns + # [String] hostname = Name of the host computer running the vm. If this is not a Virtual Machine, it returns the vm_name + def get_vm_host(vm_name) + fail "#{self.class.name} does not implement get_vm_host" + end + + # inputs + # vm_name: string + # returns + # [String] hostname = Name of the most appropriate host computer to run this VM. Useful for load balancing VMs in a cluster + # If this is not a Virtual Machine, it returns the vm_name + def find_least_used_compatible_host(vm_name) + fail "#{self.class.name} does not implement find_least_used_compatible_host" + end + + # inputs + # vm_name: string + # dest_host_name: string (Name of the host to migrate `vm_name` to) + # returns + # [Boolean] Returns true on success or false on failure + def migrate_vm_to_host(vm_name, dest_host_name) + fail "#{self.class.name} does not implement migrate_vm_to_host" + end + + # inputs + # vm_name: string + # returns + # nil if it doesn't exist + # Hastable of the VM + # [String] hostname = Name reported by Vmware tools (host.summary.guest.hostName) + # [String] template = This is the name of template exposed by the API. It must _match_ the poolname + # [String] poolname = Name of the pool the VM is located + # [Time] boottime = Time when the VM was created/booted + # [String] powerstate = Current power state of a VM. Valid values (as per vCenter API) + # - 'PoweredOn','PoweredOff' + def get_vm(vm_name) + fail "#{self.class.name} does not implement get_vm" + end + + # inputs + # pool: string + # returns + # vm name: string + def create_vm(pool) + fail "#{self.class.name} does not implement create_vm" + end + + # inputs + # vm_name: string + # pool: string + # returns + # boolean : true if success, false on error + def destroy_vm(vm_name,pool) + fail "#{self.class.name} does not implement destroy_vm" + end + + # inputs + # vm : string + # pool: string + # timeout: int (Seconds) + # returns + # result: boolean + def is_vm_ready?(vm,pool,timeout) + fail "#{self.class.name} does not implement is_vm_ready?" + end + + # inputs + # vm : string + # returns + # result: boolean + def vm_exists?(vm) + fail "#{self.class.name} does not implement vm_exists?" + end + + end + end + end +end \ No newline at end of file diff --git a/lib/vmpooler/backingservice/vsphere.rb b/lib/vmpooler/backingservice/vsphere.rb new file mode 100644 index 0000000..ee69e13 --- /dev/null +++ b/lib/vmpooler/backingservice/vsphere.rb @@ -0,0 +1,622 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + class PoolManager + class BackingService + class Vsphere < Vmpooler::PoolManager::BackingService::Base + #--------------- Public methods + + def initialize(options) + $credentials = options['credentials'] + $metrics = options['metrics'] + end + + def devices_in_pool(pool) + base = find_folder(pool['folder']) + + base.childEntity.each do |vm| + vm + end + end + +# def destroy_vm() +# # Destroy a VM +# def _destroy_vm(vm, pool, vsphere) +# $redis.srem('vmpooler__completed__' + pool, vm) +# $redis.hdel('vmpooler__active__' + pool, vm) +# $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + +# # Auto-expire metadata key +# $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + +# # TODO This is all vSphere specific + +# host = vsphere.find_vm(vm) + +# if host +# start = Time.now + +# if +# (host.runtime) && +# (host.runtime.powerState) && +# (host.runtime.powerState == 'poweredOn') + +# $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") +# host.PowerOffVM_Task.wait_for_completion +# end + +# host.Destroy_Task.wait_for_completion +# finish = '%.2f' % (Time.now - start) + +# $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") +# $metrics.timing("destroy.#{pool}", finish) +# end +# end +# end + + def create_device(pool) +'12345' + # clone_vm( + # pool['template'], + # pool['folder'], + # pool['datastore'], + # pool['clone_target'], + # vsphere + # ) + + # Thread.new do + # begin + # vm = {} + + # if template =~ /\// + # templatefolders = template.split('/') + # vm['template'] = templatefolders.pop + # end + + # if templatefolders + # vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) + # else + # fail 'Please provide a full path to the template' + # end + + # if vm['template'].length == 0 + # fail "Unable to find template '#{vm['template']}'!" + # end + + # # Generate a randomized hostname + # o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + # vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + + # # Add VM to Redis inventory ('pending' pool) + # $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) + + # # Annotate with creation time, origin template, etc. + # # Add extraconfig options that can be queried by vmtools + # configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( + # annotation: JSON.pretty_generate( + # name: vm['hostname'], + # created_by: $config[:vsphere]['username'], + # base_template: vm['template'], + # creation_timestamp: Time.now.utc + # ), + # extraConfig: [ + # { key: 'guestinfo.hostname', + # value: vm['hostname'] + # } + # ] + # ) + + # # Choose a clone target + # if target + # $clone_target = vsphere.find_least_used_host(target) + # elsif $config[:config]['clone_target'] + # $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) + # end + + # # Put the VM in the specified folder and resource pool + # relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( + # datastore: vsphere.find_datastore(datastore), + # host: $clone_target, + # diskMoveType: :moveChildMostDiskBacking + # ) + + # # Create a clone spec + # spec = RbVmomi::VIM.VirtualMachineCloneSpec( + # location: relocateSpec, + # config: configSpec, + # powerOn: true, + # template: false + # ) + + # # Clone the VM + # $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + + # begin + # start = Time.now + # vm[vm['template']].CloneVM_Task( + # folder: vsphere.find_folder(folder), + # name: vm['hostname'], + # spec: spec + # ).wait_for_completion + # finish = '%.2f' % (Time.now - start) + + # $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + + # $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") + # $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + # raise + # end + + # $redis.decr('vmpooler__tasks__clone') + + # $metrics.timing("clone.#{vm['template']}", finish) + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") + # raise + # end + # end + + + end + +#**** When getting the VM details + # if (host.summary) && + # (host.summary.guest) && + # (host.summary.guest.hostName) && + # (host.summary.guest.hostName == vm) +# + + def is_vm_ready?(vm,pool,timeout) + fail "!!!!" + + + # def open_socket(host, domain=nil, timeout=5, port=22, &block) + # Timeout.timeout(timeout) do + # target_host = host + # target_host = "#{host}.#{domain}" if domain + # sock = TCPSocket.new target_host, port + # begin + # yield sock if block_given? + # ensure + # sock.close + # end + # end + # end + + # def _check_pending_vm(vm, pool, timeout, vsphere) + # host = vsphere.find_vm(vm) + + # if ! host + # fail_pending_vm(vm, pool, timeout, false) + # return + # end + # open_socket vm + # move_pending_vm_to_ready(vm, pool, host) + # rescue + # fail_pending_vm(vm, pool, timeout) + # end + + + end + + + + #--------------- Private methods + private + ADAPTER_TYPE = 'lsiLogic' + DISK_TYPE = 'thin' + DISK_MODE = 'persistent' + + def ensure_connected(connection, credentials) + connection.serviceInstance.CurrentTime + rescue + $metrics.increment("connect.open") + connect_to_vsphere $credentials + end + + def connect_to_vsphere(credentials) + @connection = RbVmomi::VIM.connect host: credentials['server'], + user: credentials['username'], + password: credentials['password'], + insecure: credentials['insecure'] || true + end + + def add_disk(vm, size, datastore) + ensure_connected @connection, $credentials + + return false unless size.to_i > 0 + + vmdk_datastore = find_datastore(datastore) + vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" + + controller = find_disk_controller(vm) + + vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( + capacityKb: size.to_i * 1024 * 1024, + adapterType: ADAPTER_TYPE, + diskType: DISK_TYPE + ) + + vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( + datastore: vmdk_datastore, + diskMode: DISK_MODE, + fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" + ) + + device = RbVmomi::VIM::VirtualDisk( + backing: vmdk_backing, + capacityInKB: size.to_i * 1024 * 1024, + controllerKey: controller.key, + key: -1, + unitNumber: find_disk_unit_number(vm, controller) + ) + + device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( + device: device, + operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') + ) + + vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( + deviceChange: [device_config_spec] + ) + + @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( + datacenter: @connection.serviceInstance.find_datacenter, + name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", + spec: vmdk_spec + ).wait_for_completion + + vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion + + true + end + + def find_datastore(datastorename) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + datacenter.find_datastore(datastorename) + end + + def find_device(vm, deviceName) + ensure_connected @connection, $credentials + + vm.config.hardware.device.each do |device| + return device if device.deviceInfo.label == deviceName + end + + nil + end + + def find_disk_controller(vm) + ensure_connected @connection, $credentials + + devices = find_disk_devices(vm) + + devices.keys.sort.each do |device| + if devices[device]['children'].length < 15 + return find_device(vm, devices[device]['device'].deviceInfo.label) + end + end + + nil + end + + def find_disk_devices(vm) + ensure_connected @connection, $credentials + + devices = {} + + vm.config.hardware.device.each do |device| + if device.is_a? RbVmomi::VIM::VirtualSCSIController + if devices[device.controllerKey].nil? + devices[device.key] = {} + devices[device.key]['children'] = [] + end + + devices[device.key]['device'] = device + end + + if device.is_a? RbVmomi::VIM::VirtualDisk + if devices[device.controllerKey].nil? + devices[device.controllerKey] = {} + devices[device.controllerKey]['children'] = [] + end + + devices[device.controllerKey]['children'].push(device) + end + end + + devices + end + + def find_disk_unit_number(vm, controller) + ensure_connected @connection, $credentials + + used_unit_numbers = [] + available_unit_numbers = [] + + devices = find_disk_devices(vm) + + devices.keys.sort.each do |c| + next unless controller.key == devices[c]['device'].key + used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) + devices[c]['children'].each do |disk| + used_unit_numbers.push(disk.unitNumber) + end + end + + (0..15).each do |scsi_id| + if used_unit_numbers.grep(scsi_id).length <= 0 + available_unit_numbers.push(scsi_id) + end + end + + available_unit_numbers.sort[0] + end + + def find_folder(foldername) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.vmFolder + folders = foldername.split('/') + folders.each do |folder| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == folder } + else + abort "Unexpected object type encountered (#{base.class}) while finding folder" + end + end + + base + end + + # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference + # Params: + # +model+:: CPU arch version to match on + # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments + def get_host_utilization(host, model=nil, limit=90) + if model + return nil unless host_has_cpu_model? host, model + end + return nil if host.runtime.inMaintenanceMode + return nil unless host.overallStatus == 'green' + + cpu_utilization = cpu_utilization_for host + memory_utilization = memory_utilization_for host + + return nil if cpu_utilization > limit + return nil if memory_utilization > limit + + [ cpu_utilization + memory_utilization, host ] + end + + def host_has_cpu_model?(host, model) + get_host_cpu_arch_version(host) == model + end + + def get_host_cpu_arch_version(host) + cpu_model = host.hardware.cpuPkg[0].description + cpu_model_parts = cpu_model.split() + arch_version = cpu_model_parts[4] + arch_version + end + + def cpu_utilization_for(host) + cpu_usage = host.summary.quickStats.overallCpuUsage + cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores + (cpu_usage.to_f / cpu_size.to_f) * 100 + end + + def memory_utilization_for(host) + memory_usage = host.summary.quickStats.overallMemoryUsage + memory_size = host.summary.hardware.memorySize / 1024 / 1024 + (memory_usage.to_f / memory_size.to_f) * 100 + end + + def find_least_used_host(cluster) + ensure_connected @connection, $credentials + + cluster_object = find_cluster(cluster) + target_hosts = get_cluster_host_utilization(cluster_object) + least_used_host = target_hosts.sort[0][1] + least_used_host + end + + def find_cluster(cluster) + datacenter = @connection.serviceInstance.find_datacenter + datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } + end + + def get_cluster_host_utilization(cluster) + cluster_hosts = [] + cluster.host.each do |host| + host_usage = get_host_utilization(host) + cluster_hosts << host_usage if host_usage + end + cluster_hosts + end + + def find_least_used_compatible_host(vm) + ensure_connected @connection, $credentials + + source_host = vm.summary.runtime.host + model = get_host_cpu_arch_version(source_host) + cluster = source_host.parent + target_hosts = [] + cluster.host.each do |host| + host_usage = get_host_utilization(host, model) + target_hosts << host_usage if host_usage + end + target_host = target_hosts.sort[0][1] + [target_host, target_host.name] + end + + def find_pool(poolname) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.hostFolder + pools = poolname.split('/') + pools.each do |pool| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == pool } + when RbVmomi::VIM::ClusterComputeResource + base = base.resourcePool.resourcePool.find { |f| f.name == pool } + when RbVmomi::VIM::ResourcePool + base = base.resourcePool.find { |f| f.name == pool } + else + abort "Unexpected object type encountered (#{base.class}) while finding resource pool" + end + end + + base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) + base + end + + def find_snapshot(vm, snapshotname) + if vm.snapshot + get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) + end + end + + def find_vm(vmname) + ensure_connected @connection, $credentials + find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] + end + + def find_vm_light(vmname) + ensure_connected @connection, $credentials + + @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) + end + + def find_vm_heavy(vmname) + ensure_connected @connection, $credentials + + vmname = vmname.is_a?(Array) ? vmname : [vmname] + containerView = get_base_vm_container_from @connection + propertyCollector = @connection.propertyCollector + + objectSet = [{ + obj: containerView, + skip: true, + selectSet: [RbVmomi::VIM::TraversalSpec.new( + name: 'gettingTheVMs', + path: 'view', + skip: false, + type: 'ContainerView' + )] + }] + + propSet = [{ + pathSet: ['name'], + type: 'VirtualMachine' + }] + + results = propertyCollector.RetrievePropertiesEx( + specSet: [{ + objectSet: objectSet, + propSet: propSet + }], + options: { maxObjects: nil } + ) + + vms = {} + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + + while results.token + results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + end + + vms + end + + def find_vmdks(vmname, datastore) + ensure_connected @connection, $credentials + + disks = [] + + vmdk_datastore = find_datastore(datastore) + + vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' + vm_files.keys.each do |f| + vm_files[f]['layoutEx.file'].each do |l| + if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) + disks.push(l) + end + end + end + + disks + end + + def get_base_vm_container_from(connection) + ensure_connected @connection, $credentials + + viewManager = connection.serviceContent.viewManager + viewManager.CreateContainerView( + container: connection.serviceContent.rootFolder, + recursive: true, + type: ['VirtualMachine'] + ) + end + + def get_snapshot_list(tree, snapshotname) + snapshot = nil + + tree.each do |child| + if child.name == snapshotname + snapshot ||= child.snapshot + else + snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) + end + end + + snapshot + end + + def migrate_vm_host(vm, host) + relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) + vm.RelocateVM_Task(spec: relospec).wait_for_completion + end + + def close + @connection.close + end + + end + end + end +end + + + + + + + + + +module Vmpooler + class VsphereHelper + + end +end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 4f55711..94918b0 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -12,51 +12,76 @@ module Vmpooler # Connect to Redis $redis = redis - # vSphere object - $vsphere = {} + # per pool VM Backing Services + $backing_services = {} # Our thread-tracker object $threads = {} + + # WARNING DEBUG + $logger.log('d',"Flushing REDIS WARNING!!!") + $redis.flushdb end # Check the state of a VM - def check_pending_vm(vm, pool, timeout, vsphere) + # DONE + def check_pending_vm(vm, pool, timeout, backingservice) Thread.new do - _check_pending_vm(vm, pool, timeout, vsphere) + _check_pending_vm(vm, pool, timeout, backingservice) end end - def open_socket(host, domain=nil, timeout=5, port=22, &block) - Timeout.timeout(timeout) do - target_host = host - target_host = "#{host}.#{domain}" if domain - sock = TCPSocket.new target_host, port - begin - yield sock if block_given? - ensure - sock.close - end - end - end - - def _check_pending_vm(vm, pool, timeout, vsphere) - host = vsphere.find_vm(vm) - + # DONE + def _check_pending_vm(vm, pool, timeout, backingservice) + host = backingservice.get_vm(vm) if ! host fail_pending_vm(vm, pool, timeout, false) return end - open_socket vm - move_pending_vm_to_ready(vm, pool, host) - rescue + if backingservice.is_vm_ready?(vm,pool,timeout) + move_pending_vm_to_ready(vm, pool, host) + else + fail "VM is not ready" + end + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}") fail_pending_vm(vm, pool, timeout) + raise end + # def open_socket(host, domain=nil, timeout=5, port=22, &block) + # Timeout.timeout(timeout) do + # target_host = host + # target_host = "#{host}.#{domain}" if domain + # sock = TCPSocket.new target_host, port + # begin + # yield sock if block_given? + # ensure + # sock.close + # end + # end + # end + + # def _check_pending_vm(vm, pool, timeout, vsphere) + # host = vsphere.find_vm(vm) + + # if ! host + # fail_pending_vm(vm, pool, timeout, false) + # return + # end + # open_socket vm + # move_pending_vm_to_ready(vm, pool, host) + # rescue + # fail_pending_vm(vm, pool, timeout) + # end + + # DONE def remove_nonexistent_vm(vm, pool) $redis.srem("vmpooler__pending__#{pool}", vm) $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") end + # DONE def fail_pending_vm(vm, pool, timeout, exists=true) clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone') return if ! clone_stamp @@ -74,14 +99,11 @@ module Vmpooler $logger.log('d', "Fail pending VM failed with an error: #{err}") end + # DONE def move_pending_vm_to_ready(vm, pool, host) - if (host.summary) && - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName == vm) - + if host['hostname'] == vm begin - Socket.getaddrinfo(vm, nil) # WTF? + Socket.getaddrinfo(vm, nil) # WTF? I assume this is just priming the local DNS resolver cache?!?! rescue end @@ -91,77 +113,83 @@ module Vmpooler $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm) $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) - $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") + $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") end end - def check_ready_vm(vm, pool, ttl, vsphere) + # DONE + def check_ready_vm(vm, pool, ttl, backingservice) Thread.new do - if ttl > 0 - if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + _check_ready_vm(vm, pool, ttl, backingservice) + end + end - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - end + # DONE + def _check_ready_vm(vm, pool, ttl, backingservice) + host = backingservice.get_vm(vm) + # Check if the host even exists + if !host + $redis.srem('vmpooler__ready__' + pool, vm) + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue") + return + end + + # Check if the hosts TTL has expired + if ttl > 0 + if (((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") end + end - check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + # Periodically check that the VM is available + check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + if + (!check_stamp) || + (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime']) + $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + + # Check if the VM is not powered on if - (!check_stamp) || - (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime']) + (host['powerstate'] != 'PoweredOn') + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end - $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the hostname has magically changed from underneath Pooler + if (host['hostname'] != vm) + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") + end - host = vsphere.find_vm(vm) - - if host - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState != 'poweredOn') - - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") - end - - if - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName != vm) - - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") - end + # Check if the VM is still ready/available + begin + fail "VM #{vm} is not ready" unless backingservice.is_vm_ready?(vm,pool,5) + rescue + if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") else - $redis.srem('vmpooler__ready__' + pool, vm) - - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - open_socket vm - rescue - if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") - else - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") - end + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") end end end + rescue => err + $logger.log('s', "[!] [#{vm['poolname']}] '#{vm['hostname']}' failed while checking a ready vm : #{err}") + raise end - def check_running_vm(vm, pool, ttl, vsphere) + # DONE + def check_running_vm(vm, pool, ttl, backingservice) Thread.new do - _check_running_vm(vm, pool, ttl, vsphere) + _check_running_vm(vm, pool, ttl, backingservice) end end - def _check_running_vm(vm, pool, ttl, vsphere) - host = vsphere.find_vm(vm) + # DONE + def _check_running_vm(vm, pool, ttl, backingservice) + host = backingservice.get_vm(vm) if host queue_from, queue_to = 'running', 'completed' @@ -179,151 +207,153 @@ module Vmpooler end end + # DONE def move_vm_queue(pool, vm, queue_from, queue_to, msg) $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm) $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") end - # Clone a VM - def clone_vm(template, folder, datastore, target, vsphere) + # DONE + def clone_vm(pool, backingservice) Thread.new do - begin - vm = {} + backingservice.create_vm(pool) + end + end - if template =~ /\// - templatefolders = template.split('/') - vm['template'] = templatefolders.pop - end + # Clone a VM + # def clone_vm(template, folder, datastore, target, vsphere) + # Thread.new do + # begin + # vm = {} - if templatefolders - vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) - else - fail 'Please provide a full path to the template' - end + # if template =~ /\// + # templatefolders = template.split('/') + # vm['template'] = templatefolders.pop + # end - if vm['template'].length == 0 - fail "Unable to find template '#{vm['template']}'!" - end + # if templatefolders + # vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) + # else + # fail 'Please provide a full path to the template' + # end - # Generate a randomized hostname - o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + # if vm['template'].length == 0 + # fail "Unable to find template '#{vm['template']}'!" + # end - # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) + # # Generate a randomized hostname + # o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + # vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join - # Annotate with creation time, origin template, etc. - # Add extraconfig options that can be queried by vmtools - configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - annotation: JSON.pretty_generate( - name: vm['hostname'], - created_by: $config[:vsphere]['username'], - base_template: vm['template'], - creation_timestamp: Time.now.utc - ), - extraConfig: [ - { key: 'guestinfo.hostname', - value: vm['hostname'] - } - ] - ) + # # Add VM to Redis inventory ('pending' pool) + # $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - # Choose a clone target - if target - $clone_target = vsphere.find_least_used_host(target) - elsif $config[:config]['clone_target'] - $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) - end + # # Annotate with creation time, origin template, etc. + # # Add extraconfig options that can be queried by vmtools + # configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( + # annotation: JSON.pretty_generate( + # name: vm['hostname'], + # created_by: $config[:vsphere]['username'], + # base_template: vm['template'], + # creation_timestamp: Time.now.utc + # ), + # extraConfig: [ + # { key: 'guestinfo.hostname', + # value: vm['hostname'] + # } + # ] + # ) - # Put the VM in the specified folder and resource pool - relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - datastore: vsphere.find_datastore(datastore), - host: $clone_target, - diskMoveType: :moveChildMostDiskBacking - ) + # # Choose a clone target + # if target + # $clone_target = vsphere.find_least_used_host(target) + # elsif $config[:config]['clone_target'] + # $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) + # end - # Create a clone spec - spec = RbVmomi::VIM.VirtualMachineCloneSpec( - location: relocateSpec, - config: configSpec, - powerOn: true, - template: false - ) + # # Put the VM in the specified folder and resource pool + # relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( + # datastore: vsphere.find_datastore(datastore), + # host: $clone_target, + # diskMoveType: :moveChildMostDiskBacking + # ) - # Clone the VM - $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + # # Create a clone spec + # spec = RbVmomi::VIM.VirtualMachineCloneSpec( + # location: relocateSpec, + # config: configSpec, + # powerOn: true, + # template: false + # ) - begin - start = Time.now - vm[vm['template']].CloneVM_Task( - folder: vsphere.find_folder(folder), - name: vm['hostname'], - spec: spec - ).wait_for_completion - finish = '%.2f' % (Time.now - start) + # # Clone the VM + # $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") - $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + # begin + # start = Time.now + # vm[vm['template']].CloneVM_Task( + # folder: vsphere.find_folder(folder), + # name: vm['hostname'], + # spec: spec + # ).wait_for_completion + # finish = '%.2f' % (Time.now - start) - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") - rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") - $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) - raise - end + # $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) - $redis.decr('vmpooler__tasks__clone') + # $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") + # $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + # raise + # end - $metrics.timing("clone.#{vm['template']}", finish) - rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") - raise - end + # $redis.decr('vmpooler__tasks__clone') + + # $metrics.timing("clone.#{vm['template']}", finish) + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") + # raise + # end + # end + # end + + # Destroy a VM + # DONE + # TODO These calls should wrap the rescue block, not inside. This traps bad functions. Need to modify all functions + def destroy_vm(vm, pool, backingservice) + Thread.new do + _destroy_vm(vm, pool, backingservice) end end # Destroy a VM - def destroy_vm(vm, pool, vsphere) - Thread.new do - $redis.srem('vmpooler__completed__' + pool, vm) - $redis.hdel('vmpooler__active__' + pool, vm) - $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + # DONE + def _destroy_vm(vm, pool, backingservice) + $redis.srem('vmpooler__completed__' + pool, vm) + $redis.hdel('vmpooler__active__' + pool, vm) + $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) - # Auto-expire metadata key - $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + # Auto-expire metadata key + $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) - host = vsphere.find_vm(vm) - - if host - start = Time.now - - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState == 'poweredOn') - - $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") - host.PowerOffVM_Task.wait_for_completion - end - - host.Destroy_Task.wait_for_completion - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") - $metrics.timing("destroy.#{pool}", finish) - end - end + backingservice.destroy_vm(vm,pool) + rescue => err + $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}") + raise end def create_vm_disk(vm, disk_size, vsphere) +# TODO This is all vSphere specific Thread.new do _create_vm_disk(vm, disk_size, vsphere) end end def _create_vm_disk(vm, disk_size, vsphere) +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0)) @@ -358,12 +388,14 @@ module Vmpooler end def create_vm_snapshot(vm, snapshot_name, vsphere) +# TODO This is all vSphere specific Thread.new do _create_vm_snapshot(vm, snapshot_name, vsphere) end end def _create_vm_snapshot(vm, snapshot_name, vsphere) +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?)) @@ -387,12 +419,14 @@ module Vmpooler end def revert_vm_snapshot(vm, snapshot_name, vsphere) +# TODO This is all vSphere specific Thread.new do _revert_vm_snapshot(vm, snapshot_name, vsphere) end end def _revert_vm_snapshot(vm, snapshot_name, vsphere) +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if host @@ -413,6 +447,7 @@ module Vmpooler end def check_disk_queue +# TODO This is all vSphere specific $logger.log('d', "[*] [disk_manager] starting worker thread") $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics @@ -426,6 +461,7 @@ module Vmpooler end def _check_disk_queue(vsphere) +# TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__disk') unless vm.nil? @@ -439,6 +475,7 @@ module Vmpooler end def check_snapshot_queue +# TODO This is all vSphere specific $logger.log('d', "[*] [snapshot_manager] starting worker thread") $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics @@ -452,6 +489,7 @@ module Vmpooler end def _check_snapshot_queue(vsphere) +# TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__snapshot') unless vm.nil? @@ -475,23 +513,26 @@ module Vmpooler end end + # DONE def migration_limit(migration_limit) # Returns migration_limit setting when enabled return false if migration_limit == 0 || ! migration_limit migration_limit if migration_limit >= 1 end - def migrate_vm(vm, pool, vsphere) + # DONE + def migrate_vm(vm, pool, backingservice) Thread.new do - _migrate_vm(vm, pool, vsphere) + _migrate_vm(vm, pool, backingservice) end end - def _migrate_vm(vm, pool, vsphere) + # DONE + def _migrate_vm(vm, pool, backingservice) begin $redis.srem('vmpooler__migrating__' + pool, vm) - vm_object = vsphere.find_vm(vm) - parent_host, parent_host_name = get_vm_host_info(vm_object) + + parent_host_name = backingservice.get_vm_host(vm) migration_limit = migration_limit $config[:config]['migration_limit'] migration_count = $redis.scard('vmpooler__migration') @@ -504,11 +545,11 @@ module Vmpooler return else $redis.sadd('vmpooler__migration', vm) - host, host_name = vsphere.find_least_used_compatible_host(vm_object) - if host == parent_host + host_name = backingservice.find_least_used_compatible_host(vm) + if host_name == parent_host_name $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") else - finish = migrate_vm_and_record_timing(vm_object, vm, pool, host, parent_host_name, host_name, vsphere) + finish = migrate_vm_and_record_timing(vm, pool, parent_host_name, host_name, backingservice) $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") end remove_vmpooler_migration_vm(pool, vm) @@ -520,11 +561,13 @@ module Vmpooler end end - def get_vm_host_info(vm_object) - parent_host = vm_object.summary.runtime.host - [parent_host, parent_host.name] - end +# TODO This is all vSphere specific + # def get_vm_host_info(vm_object) + # parent_host = vm_object.summary.runtime.host + # [parent_host, parent_host.name] + # end + # DONE def remove_vmpooler_migration_vm(pool, vm) begin $redis.srem('vmpooler__migration', vm) @@ -533,9 +576,10 @@ module Vmpooler end end - def migrate_vm_and_record_timing(vm_object, vm_name, pool, host, source_host_name, dest_host_name, vsphere) + # DONE + def migrate_vm_and_record_timing(vm_name, pool, source_host_name, dest_host_name, backingservice) start = Time.now - vsphere.migrate_vm_host(vm_object, host) + backingservice.migrate_vm_to_host(vm_name, dest_host_name) finish = '%.2f' % (Time.now - start) $metrics.timing("migrate.#{pool}", finish) $metrics.increment("migrate_from.#{source_host_name}") @@ -546,26 +590,35 @@ module Vmpooler finish end + # DONE def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") - $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics + case pool['backingservice'] + when 'vsphere' + $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Vsphere.new({ 'metrics' => $metrics}) # TODO Vmpooler::VsphereHelper.new $config[:vsphere], $metrics + when 'dummy' + $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Dummy.new($config[:backingservice][:dummy]) + else + $logger.log('s', "[!] backing service #{pool['backingservice']} is unknown for pool [#{pool['name']}]") + end $threads[pool['name']] = Thread.new do loop do - _check_pool(pool, $vsphere[pool['name']]) - sleep(5) + _check_pool(pool, $backing_services[pool['name']]) +# TODO Should this be configurable? + sleep(2) # Should be 5 end end end - def _check_pool(pool, vsphere) + def _check_pool(pool,backingservice) +puts "CHECK POOL STARTING" # INVENTORY + # DONE!! inventory = {} begin - base = vsphere.find_folder(pool['folder']) - - base.childEntity.each do |vm| + backingservice.vms_in_pool(pool).each do |vm| if (! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) && @@ -586,11 +639,12 @@ module Vmpooler end # RUNNING + # DONE!! $redis.smembers("vmpooler__running__#{pool['name']}").each do |vm| if inventory[vm] begin vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12 - check_running_vm(vm, pool['name'], vm_lifetime, vsphere) + check_running_vm(vm, pool['name'], vm_lifetime, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool with an error while evaluating running VMs: #{err}") end @@ -598,10 +652,11 @@ module Vmpooler end # READY + # DONE!! $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm| if inventory[vm] begin - check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere) + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}") end @@ -609,11 +664,12 @@ module Vmpooler end # PENDING + # DONE!! $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm| pool_timeout = pool['timeout'] || $config[:config]['timeout'] || 15 if inventory[vm] begin - check_pending_vm(vm, pool['name'], pool_timeout, vsphere) + check_pending_vm(vm, pool['name'], pool_timeout, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating pending VMs: #{err}") end @@ -623,10 +679,11 @@ module Vmpooler end # COMPLETED + # DONE!! $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm| if inventory[vm] begin - destroy_vm(vm, pool['name'], vsphere) + destroy_vm(vm, pool['name'], backingservice) rescue => err $redis.srem("vmpooler__completed__#{pool['name']}", vm) $redis.hdel("vmpooler__active__#{pool['name']}", vm) @@ -642,6 +699,7 @@ module Vmpooler end # DISCOVERED + # DONE begin $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm| %w(pending ready running completed).each do |queue| @@ -660,10 +718,11 @@ module Vmpooler end # MIGRATIONS + # DONE $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm| if inventory[vm] begin - migrate_vm(vm, pool['name'], vsphere) + migrate_vm(vm, pool['name'], backingservice) rescue => err $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") end @@ -671,6 +730,7 @@ module Vmpooler end # REPOPULATE + # DONE ready = $redis.scard("vmpooler__ready__#{pool['name']}") total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready @@ -693,14 +753,7 @@ module Vmpooler if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i begin $redis.incr('vmpooler__tasks__clone') - - clone_vm( - pool['template'], - pool['folder'], - pool['datastore'], - pool['clone_target'], - vsphere - ) + clone_vm(pool,backingservice) rescue => err $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}") $redis.decr('vmpooler__tasks__clone') @@ -721,21 +774,26 @@ module Vmpooler $redis.set('vmpooler__tasks__clone', 0) # Clear out vmpooler__migrations since stale entries may be left after a restart $redis.del('vmpooler__migration') + # Set default backingservice for all pools that do not have one defined + $config[:pools].each do |pool| + pool['backingservice'] = 'vsphere' if pool['backingservice'].nil? + end loop do - if ! $threads['disk_manager'] - check_disk_queue - elsif ! $threads['disk_manager'].alive? - $logger.log('d', "[!] [disk_manager] worker thread died, restarting") - check_disk_queue - end + # DEBUG TO DO + # if ! $threads['disk_manager'] + # check_disk_queue + # elsif ! $threads['disk_manager'].alive? + # $logger.log('d', "[!] [disk_manager] worker thread died, restarting") + # check_disk_queue + # end - if ! $threads['snapshot_manager'] - check_snapshot_queue - elsif ! $threads['snapshot_manager'].alive? - $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting") - check_snapshot_queue - end + # if ! $threads['snapshot_manager'] + # check_snapshot_queue + # elsif ! $threads['snapshot_manager'].alive? + # $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting") + # check_snapshot_queue + # end $config[:pools].each do |pool| if ! $threads[pool['name']] diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index cc93250..3b7f230 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -1,417 +1,417 @@ -require 'rubygems' unless defined?(Gem) - -module Vmpooler - class VsphereHelper - ADAPTER_TYPE = 'lsiLogic' - DISK_TYPE = 'thin' - DISK_MODE = 'persistent' - - def initialize(config, metrics) - $credentials = config[:vsphere] - $conf = config[:config] - $metrics = metrics - end - - def ensure_connected(connection, credentials) - connection.serviceInstance.CurrentTime - rescue - $metrics.increment("connect.open") - connect_to_vsphere $credentials - end - - def connect_to_vsphere(credentials) - max_tries = $conf['max_tries'] || 3 - retry_factor = $conf['retry_factor'] || 10 - try = 1 - begin - @connection = RbVmomi::VIM.connect host: credentials['server'], - user: credentials['username'], - password: credentials['password'], - insecure: credentials['insecure'] || true - $metrics.increment("connect.open") - rescue => err - try += 1 - $metrics.increment("connect.fail") - raise err if try == max_tries - sleep(try * retry_factor) - retry - end - end - - def add_disk(vm, size, datastore) - ensure_connected @connection, $credentials - - return false unless size.to_i > 0 - - vmdk_datastore = find_datastore(datastore) - vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" - - controller = find_disk_controller(vm) - - vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( - capacityKb: size.to_i * 1024 * 1024, - adapterType: ADAPTER_TYPE, - diskType: DISK_TYPE - ) - - vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( - datastore: vmdk_datastore, - diskMode: DISK_MODE, - fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" - ) - - device = RbVmomi::VIM::VirtualDisk( - backing: vmdk_backing, - capacityInKB: size.to_i * 1024 * 1024, - controllerKey: controller.key, - key: -1, - unitNumber: find_disk_unit_number(vm, controller) - ) - - device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( - device: device, - operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') - ) - - vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( - deviceChange: [device_config_spec] - ) - - @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( - datacenter: @connection.serviceInstance.find_datacenter, - name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", - spec: vmdk_spec - ).wait_for_completion - - vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion - - true - end - - def find_datastore(datastorename) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - datacenter.find_datastore(datastorename) - end - - def find_device(vm, deviceName) - ensure_connected @connection, $credentials - - vm.config.hardware.device.each do |device| - return device if device.deviceInfo.label == deviceName - end - - nil - end - - def find_disk_controller(vm) - ensure_connected @connection, $credentials - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |device| - if devices[device]['children'].length < 15 - return find_device(vm, devices[device]['device'].deviceInfo.label) - end - end - - nil - end - - def find_disk_devices(vm) - ensure_connected @connection, $credentials - - devices = {} - - vm.config.hardware.device.each do |device| - if device.is_a? RbVmomi::VIM::VirtualSCSIController - if devices[device.controllerKey].nil? - devices[device.key] = {} - devices[device.key]['children'] = [] - end - - devices[device.key]['device'] = device - end - - if device.is_a? RbVmomi::VIM::VirtualDisk - if devices[device.controllerKey].nil? - devices[device.controllerKey] = {} - devices[device.controllerKey]['children'] = [] - end - - devices[device.controllerKey]['children'].push(device) - end - end - - devices - end - - def find_disk_unit_number(vm, controller) - ensure_connected @connection, $credentials - - used_unit_numbers = [] - available_unit_numbers = [] - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |c| - next unless controller.key == devices[c]['device'].key - used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) - devices[c]['children'].each do |disk| - used_unit_numbers.push(disk.unitNumber) - end - end - - (0..15).each do |scsi_id| - if used_unit_numbers.grep(scsi_id).length <= 0 - available_unit_numbers.push(scsi_id) - end - end - - available_unit_numbers.sort[0] - end - - def find_folder(foldername) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.vmFolder - folders = foldername.split('/') - folders.each do |folder| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == folder } - else - abort "Unexpected object type encountered (#{base.class}) while finding folder" - end - end - - base - end - - # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference - # Params: - # +model+:: CPU arch version to match on - # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments - def get_host_utilization(host, model=nil, limit=90) - if model - return nil unless host_has_cpu_model? host, model - end - return nil if host.runtime.inMaintenanceMode - return nil unless host.overallStatus == 'green' - - cpu_utilization = cpu_utilization_for host - memory_utilization = memory_utilization_for host - - return nil if cpu_utilization > limit - return nil if memory_utilization > limit - - [ cpu_utilization + memory_utilization, host ] - end - - def host_has_cpu_model?(host, model) - get_host_cpu_arch_version(host) == model - end - - def get_host_cpu_arch_version(host) - cpu_model = host.hardware.cpuPkg[0].description - cpu_model_parts = cpu_model.split() - arch_version = cpu_model_parts[4] - arch_version - end - - def cpu_utilization_for(host) - cpu_usage = host.summary.quickStats.overallCpuUsage - cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores - (cpu_usage.to_f / cpu_size.to_f) * 100 - end - - def memory_utilization_for(host) - memory_usage = host.summary.quickStats.overallMemoryUsage - memory_size = host.summary.hardware.memorySize / 1024 / 1024 - (memory_usage.to_f / memory_size.to_f) * 100 - end - - def find_least_used_host(cluster) - ensure_connected @connection, $credentials - - cluster_object = find_cluster(cluster) - target_hosts = get_cluster_host_utilization(cluster_object) - least_used_host = target_hosts.sort[0][1] - least_used_host - end - - def find_cluster(cluster) - datacenter = @connection.serviceInstance.find_datacenter - datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } - end - - def get_cluster_host_utilization(cluster) - cluster_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host) - cluster_hosts << host_usage if host_usage - end - cluster_hosts - end - - def find_least_used_compatible_host(vm) - ensure_connected @connection, $credentials - - source_host = vm.summary.runtime.host - model = get_host_cpu_arch_version(source_host) - cluster = source_host.parent - target_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host, model) - target_hosts << host_usage if host_usage - end - target_host = target_hosts.sort[0][1] - [target_host, target_host.name] - end - - def find_pool(poolname) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.hostFolder - pools = poolname.split('/') - pools.each do |pool| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == pool } - when RbVmomi::VIM::ClusterComputeResource - base = base.resourcePool.resourcePool.find { |f| f.name == pool } - when RbVmomi::VIM::ResourcePool - base = base.resourcePool.find { |f| f.name == pool } - else - abort "Unexpected object type encountered (#{base.class}) while finding resource pool" - end - end - - base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) - base - end - - def find_snapshot(vm, snapshotname) - if vm.snapshot - get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) - end - end - - def find_vm(vmname) - ensure_connected @connection, $credentials - find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] - end - - def find_vm_light(vmname) - ensure_connected @connection, $credentials - - @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) - end - - def find_vm_heavy(vmname) - ensure_connected @connection, $credentials - - vmname = vmname.is_a?(Array) ? vmname : [vmname] - containerView = get_base_vm_container_from @connection - propertyCollector = @connection.propertyCollector - - objectSet = [{ - obj: containerView, - skip: true, - selectSet: [RbVmomi::VIM::TraversalSpec.new( - name: 'gettingTheVMs', - path: 'view', - skip: false, - type: 'ContainerView' - )] - }] - - propSet = [{ - pathSet: ['name'], - type: 'VirtualMachine' - }] - - results = propertyCollector.RetrievePropertiesEx( - specSet: [{ - objectSet: objectSet, - propSet: propSet - }], - options: { maxObjects: nil } - ) - - vms = {} - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - - while results.token - results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - end - - vms - end - - def find_vmdks(vmname, datastore) - ensure_connected @connection, $credentials - - disks = [] - - vmdk_datastore = find_datastore(datastore) - - vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' - vm_files.keys.each do |f| - vm_files[f]['layoutEx.file'].each do |l| - if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) - disks.push(l) - end - end - end - - disks - end - - def get_base_vm_container_from(connection) - ensure_connected @connection, $credentials - - viewManager = connection.serviceContent.viewManager - viewManager.CreateContainerView( - container: connection.serviceContent.rootFolder, - recursive: true, - type: ['VirtualMachine'] - ) - end - - def get_snapshot_list(tree, snapshotname) - snapshot = nil - - tree.each do |child| - if child.name == snapshotname - snapshot ||= child.snapshot - else - snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) - end - end - - snapshot - end - - def migrate_vm_host(vm, host) - relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) - vm.RelocateVM_Task(spec: relospec).wait_for_completion - end - - def close - @connection.close - end - end -end +# require 'rubygems' unless defined?(Gem) + +# module Vmpooler +# class VsphereHelper +# ADAPTER_TYPE = 'lsiLogic' +# DISK_TYPE = 'thin' +# DISK_MODE = 'persistent' + +# def initialize(config, metrics) +# $credentials = config[:vsphere] +# $conf = config[:config] +# $metrics = metrics +# end + +# def ensure_connected(connection, credentials) +# connection.serviceInstance.CurrentTime +# rescue +# $metrics.increment("connect.open") +# connect_to_vsphere $credentials +# end + +# def connect_to_vsphere(credentials) +# max_tries = $conf['max_tries'] || 3 +# retry_factor = $conf['retry_factor'] || 10 +# try = 1 +# begin +# @connection = RbVmomi::VIM.connect host: credentials['server'], +# user: credentials['username'], +# password: credentials['password'], +# insecure: credentials['insecure'] || true +# $metrics.increment("connect.open") +# rescue => err +# try += 1 +# $metrics.increment("connect.fail") +# raise err if try == max_tries +# sleep(try * retry_factor) +# retry +# end +# end + +# def add_disk(vm, size, datastore) +# ensure_connected @connection, $credentials + +# return false unless size.to_i > 0 + +# vmdk_datastore = find_datastore(datastore) +# vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" + +# controller = find_disk_controller(vm) + +# vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( +# capacityKb: size.to_i * 1024 * 1024, +# adapterType: ADAPTER_TYPE, +# diskType: DISK_TYPE +# ) + +# vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( +# datastore: vmdk_datastore, +# diskMode: DISK_MODE, +# fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" +# ) + +# device = RbVmomi::VIM::VirtualDisk( +# backing: vmdk_backing, +# capacityInKB: size.to_i * 1024 * 1024, +# controllerKey: controller.key, +# key: -1, +# unitNumber: find_disk_unit_number(vm, controller) +# ) + +# device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( +# device: device, +# operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') +# ) + +# vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( +# deviceChange: [device_config_spec] +# ) + +# @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( +# datacenter: @connection.serviceInstance.find_datacenter, +# name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", +# spec: vmdk_spec +# ).wait_for_completion + +# vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion + +# true +# end + +# def find_datastore(datastorename) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# datacenter.find_datastore(datastorename) +# end + +# def find_device(vm, deviceName) +# ensure_connected @connection, $credentials + +# vm.config.hardware.device.each do |device| +# return device if device.deviceInfo.label == deviceName +# end + +# nil +# end + +# def find_disk_controller(vm) +# ensure_connected @connection, $credentials + +# devices = find_disk_devices(vm) + +# devices.keys.sort.each do |device| +# if devices[device]['children'].length < 15 +# return find_device(vm, devices[device]['device'].deviceInfo.label) +# end +# end + +# nil +# end + +# def find_disk_devices(vm) +# ensure_connected @connection, $credentials + +# devices = {} + +# vm.config.hardware.device.each do |device| +# if device.is_a? RbVmomi::VIM::VirtualSCSIController +# if devices[device.controllerKey].nil? +# devices[device.key] = {} +# devices[device.key]['children'] = [] +# end + +# devices[device.key]['device'] = device +# end + +# if device.is_a? RbVmomi::VIM::VirtualDisk +# if devices[device.controllerKey].nil? +# devices[device.controllerKey] = {} +# devices[device.controllerKey]['children'] = [] +# end + +# devices[device.controllerKey]['children'].push(device) +# end +# end + +# devices +# end + +# def find_disk_unit_number(vm, controller) +# ensure_connected @connection, $credentials + +# used_unit_numbers = [] +# available_unit_numbers = [] + +# devices = find_disk_devices(vm) + +# devices.keys.sort.each do |c| +# next unless controller.key == devices[c]['device'].key +# used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) +# devices[c]['children'].each do |disk| +# used_unit_numbers.push(disk.unitNumber) +# end +# end + +# (0..15).each do |scsi_id| +# if used_unit_numbers.grep(scsi_id).length <= 0 +# available_unit_numbers.push(scsi_id) +# end +# end + +# available_unit_numbers.sort[0] +# end + +# def find_folder(foldername) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# base = datacenter.vmFolder +# folders = foldername.split('/') +# folders.each do |folder| +# case base +# when RbVmomi::VIM::Folder +# base = base.childEntity.find { |f| f.name == folder } +# else +# abort "Unexpected object type encountered (#{base.class}) while finding folder" +# end +# end + +# base +# end + +# # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference +# # Params: +# # +model+:: CPU arch version to match on +# # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments +# def get_host_utilization(host, model=nil, limit=90) +# if model +# return nil unless host_has_cpu_model? host, model +# end +# return nil if host.runtime.inMaintenanceMode +# return nil unless host.overallStatus == 'green' + +# cpu_utilization = cpu_utilization_for host +# memory_utilization = memory_utilization_for host + +# return nil if cpu_utilization > limit +# return nil if memory_utilization > limit + +# [ cpu_utilization + memory_utilization, host ] +# end + +# def host_has_cpu_model?(host, model) +# get_host_cpu_arch_version(host) == model +# end + +# def get_host_cpu_arch_version(host) +# cpu_model = host.hardware.cpuPkg[0].description +# cpu_model_parts = cpu_model.split() +# arch_version = cpu_model_parts[4] +# arch_version +# end + +# def cpu_utilization_for(host) +# cpu_usage = host.summary.quickStats.overallCpuUsage +# cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores +# (cpu_usage.to_f / cpu_size.to_f) * 100 +# end + +# def memory_utilization_for(host) +# memory_usage = host.summary.quickStats.overallMemoryUsage +# memory_size = host.summary.hardware.memorySize / 1024 / 1024 +# (memory_usage.to_f / memory_size.to_f) * 100 +# end + +# def find_least_used_host(cluster) +# ensure_connected @connection, $credentials + +# cluster_object = find_cluster(cluster) +# target_hosts = get_cluster_host_utilization(cluster_object) +# least_used_host = target_hosts.sort[0][1] +# least_used_host +# end + +# def find_cluster(cluster) +# datacenter = @connection.serviceInstance.find_datacenter +# datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } +# end + +# def get_cluster_host_utilization(cluster) +# cluster_hosts = [] +# cluster.host.each do |host| +# host_usage = get_host_utilization(host) +# cluster_hosts << host_usage if host_usage +# end +# cluster_hosts +# end + +# def find_least_used_compatible_host(vm) +# ensure_connected @connection, $credentials + +# source_host = vm.summary.runtime.host +# model = get_host_cpu_arch_version(source_host) +# cluster = source_host.parent +# target_hosts = [] +# cluster.host.each do |host| +# host_usage = get_host_utilization(host, model) +# target_hosts << host_usage if host_usage +# end +# target_host = target_hosts.sort[0][1] +# [target_host, target_host.name] +# end + +# def find_pool(poolname) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# base = datacenter.hostFolder +# pools = poolname.split('/') +# pools.each do |pool| +# case base +# when RbVmomi::VIM::Folder +# base = base.childEntity.find { |f| f.name == pool } +# when RbVmomi::VIM::ClusterComputeResource +# base = base.resourcePool.resourcePool.find { |f| f.name == pool } +# when RbVmomi::VIM::ResourcePool +# base = base.resourcePool.find { |f| f.name == pool } +# else +# abort "Unexpected object type encountered (#{base.class}) while finding resource pool" +# end +# end + +# base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) +# base +# end + +# def find_snapshot(vm, snapshotname) +# if vm.snapshot +# get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) +# end +# end + +# def find_vm(vmname) +# ensure_connected @connection, $credentials +# find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] +# end + +# def find_vm_light(vmname) +# ensure_connected @connection, $credentials + +# @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) +# end + +# def find_vm_heavy(vmname) +# ensure_connected @connection, $credentials + +# vmname = vmname.is_a?(Array) ? vmname : [vmname] +# containerView = get_base_vm_container_from @connection +# propertyCollector = @connection.propertyCollector + +# objectSet = [{ +# obj: containerView, +# skip: true, +# selectSet: [RbVmomi::VIM::TraversalSpec.new( +# name: 'gettingTheVMs', +# path: 'view', +# skip: false, +# type: 'ContainerView' +# )] +# }] + +# propSet = [{ +# pathSet: ['name'], +# type: 'VirtualMachine' +# }] + +# results = propertyCollector.RetrievePropertiesEx( +# specSet: [{ +# objectSet: objectSet, +# propSet: propSet +# }], +# options: { maxObjects: nil } +# ) + +# vms = {} +# results.objects.each do |result| +# name = result.propSet.first.val +# next unless vmname.include? name +# vms[name] = result.obj +# end + +# while results.token +# results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) +# results.objects.each do |result| +# name = result.propSet.first.val +# next unless vmname.include? name +# vms[name] = result.obj +# end +# end + +# vms +# end + +# def find_vmdks(vmname, datastore) +# ensure_connected @connection, $credentials + +# disks = [] + +# vmdk_datastore = find_datastore(datastore) + +# vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' +# vm_files.keys.each do |f| +# vm_files[f]['layoutEx.file'].each do |l| +# if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) +# disks.push(l) +# end +# end +# end + +# disks +# end + +# def get_base_vm_container_from(connection) +# ensure_connected @connection, $credentials + +# viewManager = connection.serviceContent.viewManager +# viewManager.CreateContainerView( +# container: connection.serviceContent.rootFolder, +# recursive: true, +# type: ['VirtualMachine'] +# ) +# end + +# def get_snapshot_list(tree, snapshotname) +# snapshot = nil + +# tree.each do |child| +# if child.name == snapshotname +# snapshot ||= child.snapshot +# else +# snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) +# end +# end + +# snapshot +# end + +# def migrate_vm_host(vm, host) +# relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) +# vm.RelocateVM_Task(spec: relospec).wait_for_completion +# end + +# def close +# @connection.close +# end +# end +# end From d0c10104d17b660f25082faa4a7f2cc048ccd9a0 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 15:26:21 -0800 Subject: [PATCH 07/23] (POOLER-72) Add Dummy VM Backing Service Previosuly, It is difficult to do development on VM Pooler as it requires a VSphere environment for VM creation etc.. This commit implements a Dummy backing service which behaves like VSphere but will just keep a VM registry in memory. This backing service can also inject failure into operations for testing how VM pooler behaves. --- lib/vmpooler/backingservice/dummy.rb | 200 +++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 lib/vmpooler/backingservice/dummy.rb diff --git a/lib/vmpooler/backingservice/dummy.rb b/lib/vmpooler/backingservice/dummy.rb new file mode 100644 index 0000000..0355f96 --- /dev/null +++ b/lib/vmpooler/backingservice/dummy.rb @@ -0,0 +1,200 @@ +require 'yaml' + +module Vmpooler + class PoolManager + class BackingService + class Dummy < Vmpooler::PoolManager::BackingService::Base + + # Fake VM backing service for testing, with initial configuration set in a simple text YAML filename + def initialize(options) + dummyfilename = options['filename'] + + # TODO Accessing @dummylist is not thread safe :-( Mutexes? + @dummylist = {} + + if !dummyfilename.nil? && File.exists?(dummyfilename) + @dummylist ||= YAML.load_file(dummyfilename) + end + end + + def vms_in_pool(pool) + get_pool_object(pool['name']).each do |vm| + vm + end + end + + def get_vm(vm) + dummy = get_dummy_vm(vm) + return nil if dummy.nil? + + obj = {} + # TODO Randomly power off the vm? + # TODO Randomly change the hostname of the vm? + obj['hostname'] = dummy['name'] + obj['boottime'] = dummy['boottime'] + obj['template'] = dummy['template'] + obj['poolname'] = dummy['poolname'] + obj['powerstate'] = dummy['powerstate'] + + obj + end + + def vm_exists?(vm) + !get_vm(vm).nil? + end + + def find_least_used_compatible_host(vm_name) + current_vm = get_dummy_vm(vm_name) + + # TODO parameterise this (75% chance it will not migrate) + return current_vm['vm_host'] if 1 + rand(100) < 75 + + # TODO paramtise this (Simulates a 10 node cluster) + (1 + rand(10)).to_s + end + + def get_vm_host(vm_name) + current_vm = get_dummy_vm(vm_name) + + current_vm['vm_host'] + end + + def migrate_vm_to_host(vm_name, dest_host_name) + current_vm = get_dummy_vm(vm_name) + + # TODO do I need fake a random sleep for ready? + # TODO Should I inject a random error? + + sleep(1) + current_vm['vm_host'] = dest_host_name + + true + end + + def is_vm_ready?(vm,pool,timeout) + host = get_dummy_vm(vm) + if !host then return false end + if host['poolname'] != pool then return false end + if vm['ready'] then return true end + # TODO do I need fake a random sleep for ready? + # TODO Should I inject a random error? + sleep(2) + host['ready'] = true + + true + end + + def create_vm(pool) + # This is an async operation + # This code just clones a VM and starts it + # Later checking will move it from the pending to ready queue + Thread.new do + begin + template_name = pool['template'] + pool_name = pool['name'] + + # Generate a randomized hostname + o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + dummy_hostname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + + vm = {} + vm['name'] = dummy_hostname + vm['hostname'] = dummy_hostname + vm['domain'] = 'dummy.local' + vm['vm_template'] = template_name + # 'template' is the Template in API, not the template to create the VM ('vm_template') + vm['template'] = pool_name + vm['poolname'] = pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = '1' + get_pool_object(pool_name) + @dummylist['pool'][pool_name] << vm + + # Add VM to Redis inventory ('pending' pool) + $redis.sadd('vmpooler__pending__' + pool_name, vm['hostname']) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) + + $logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'") + begin + start = Time.now + + # TODO do I need fake a random sleep to clone + sleep(2) + + # TODO Inject random clone failure + finish = '%.2f' % (Time.now - start) + + $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) + $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + + $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + rescue => err + $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") + $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + raise + end + + $redis.decr('vmpooler__tasks__clone') + + $metrics.timing("clone.#{vm['template']}", finish) + dummy_hostname + rescue => err + $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") + raise + end + end + end + + def destroy_vm(vm_name,pool) + vm = get_dummy_vm(vm_name) + if !vm then return false end + if vm['poolname'] != pool then return false end + + start = Time.now + + # Shutdown down the VM if it's poweredOn + if vm['powerstate'] = 'PoweredOn' + $logger.log('d', "[ ] [#{pool}] '#{vm_name}' is being shut down") + # TODO Use random shutdown interval + sleep(2) + vm['powerstate'] = 'PoweredOff' + end + + # 'Destroy' the VM + new_poollist = @dummylist['pool'][pool].delete_if { |vm| vm['name'] == vm_name } + @dummylist['pool'][pool] = new_poollist + + # TODO Use random destruction interval + sleep(2) + + finish = '%.2f' % (Time.now - start) + + $logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds") + $metrics.timing("destroy.#{pool}", finish) + end + + private + # Get's the pool config safely from the in-memory hashtable + def get_pool_object(pool_name) + @dummylist['pool'] = {} if @dummylist['pool'].nil? + @dummylist['pool'][pool_name] = [] if @dummylist['pool'][pool_name].nil? + + return @dummylist['pool'][pool_name] + end + + def get_dummy_vm(vm) + @dummylist['pool'].keys.each do |poolname| + @dummylist['pool'][poolname].each do |poolvm| + return poolvm if poolvm['name'] == vm + end + end + + nil + end + end + end + end +end \ No newline at end of file From 9f4130d1b213074f0a771d0409a620a273eaf756 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 15:42:57 -0800 Subject: [PATCH 08/23] f new config file - killme --- vmpooler-new.yaml | 404 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 vmpooler-new.yaml diff --git a/vmpooler-new.yaml b/vmpooler-new.yaml new file mode 100644 index 0000000..43f2805 --- /dev/null +++ b/vmpooler-new.yaml @@ -0,0 +1,404 @@ +--- +# TODO KILL ME +:vsphere: + server: '127.0.0.1' + username: 'vmpooler' + password: 'swimsw1msw!m' + + +:backingservice: +# :backingservice: +# +# This section contains the backing services for VMs and Pools +# The currently supported backing services are: +# - vsphere +# - dummy + +# :vsphere: +# +# This section contains the server hostname and authentication credentials +# needed for vmpooler to connect to VMware vSphere. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the VMware vSphere server. +# (required) +# +# - username +# The username used to authenticate VMware vSphere. +# (required) +# +# - password +# The password used to authenticate VMware vSphere. +# (required) + +# Example: + + :vsphere: + server: '127.0.0.1' + username: 'vmpooler' + password: 'swimsw1msw!m' + +# :dummy: +# +# The dummy backing service is a simple text file service that can be used +# to test vmpooler operations in a development or test environment +# +# Available configuration parameters: +# +# - filename +# The filename used to store the backing text file +# (required) + +# Example: + + :dummy: + filename: 'C:/source/vmpooler/dummy-backing.yaml' + +# :redis: +# +# This section contains the server hostname and authentication credentials +# needed for vmpooler to connect to Redis. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the Redis server. +# (optional; default: 'localhost') +# +# - username +# The username used to authenticate Redis. +# (optional) +# +# - password +# The password used to authenticate Redis. +# (optional) +# +# - data_ttl +# How long (in hours) to retain metadata in Redis after VM destruction. +# (optional; default: '168') + +# Example: + +:redis: + server: 'localhost' + + + # :graphs: + # + # This section contains the server and prefix information for a graphite- + # compatible web front-end where graphs may be viewed. This is used by the + # vmpooler dashboard to retrieve statistics and graphs for a given instance. + # + # NOTE: This is not the endpoint for publishing metrics data. See `graphite:` + # and `statsd:` below. + # + # NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be + # consulted for `server`/`prefix` information to use in locating a + # graph server for our dashboard. `graphs:` is recommended over + # `graphite:` + # + # + # Available configuration parameters: + # + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + + # :statsd: + # + # This section contains the connection information required to store + # historical data via statsd. This is mutually exclusive with graphite + # and takes precedence. + # + # Available configuration parameters: + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + # + # - port + # The UDP port to communicate with the statsd daemon. + # (optional; default: 8125) + + # Example: + + :statsd: + server: 'localhost' + prefix: 'vmpooler' + port: 8125 + +# :graphite: +# +# This section contains the connection information required to store +# historical data in an external Graphite database. This is mutually exclusive +# with statsd. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the Graphite server. +# (required) +# +# - prefix +# The prefix to use while storing Graphite data. +# (optional; default: 'vmpooler') +# +# - port +# The TCP port to communicate with the graphite server. +# (optional; default: 2003) + +# Example: + +:graphite: + server: 'graphite.company.com' + +# :auth: +# +# This section contains information related to authenticating users +# for token operations. +# +# Supported Auth Providers: +# - Dummy +# - LDAP +# +# - Dummy Auth Provider +# The Dummy Authentication provider should only be used during development or testing +# If the Username and Password are different then validation succeeds +# If the Username and Password are the same then validation fails +# +# Example: +# :auth: +# provider: 'dummy' +# +# - LDAP Auth Provider +# The LDAP Authentication provider will validate usernames and passwords against an +# existing LDAP service +# +# Available configuration parameters: +# +# - host +# The FQDN hostname of the LDAP server. +# +# - port +# The port used to connect to the LDAP service. +# (optional; default: '389') +# +# - base +# The base DN used for LDAP searches. +# +# - user_object +# The LDAP object-type used to designate a user object. +# +# Example: +# :auth: +# provider: 'ldap' +# :ldap: +# host: 'localhost' +# port: 389 +# base: 'ou=users,dc=company,dc=com' +# user_object: 'uid' + +:auth: + provider: 'dummy' + +# :tagfilter: +# +# Filter tags by regular expression. + +# Example: +# +# This example demonstrates discarding everything after a '/' character for +# the 'url' tag, transforming 'foo.com/something.html' to 'foo.com'. + +:tagfilter: + url: '(.*)\/' + +# :config: +# +# This section contains global configuration information. +# +# Available configuration parameters: +# +# - site_name +# The name of your deployment. +# (optional; default: 'vmpooler') +# +# - logfile +# The path to vmpooler's log file. +# (optional; default: '/var/log/vmpooler.log') +# +# - clone_target TODO +# The target cluster VMs are cloned into (host with least VMs chosen) +# (optional; default: same cluster/host as origin template) +# +# - task_limit TODO +# The number of concurrent VMware vSphere tasks to perform. +# (optional; default: '10') +# +# - timeout +# How long (in minutes) before marking a clone as 'failed' and retrying. +# (optional; default: '15') +# +# - vm_checktime +# How often (in minutes) to check the sanity of VMs in 'ready' queues. +# (optional; default: '15') +# +# - vm_lifetime +# How long (in hours) to keep VMs in 'running' queues before destroying. +# (optional; default: '24') +# +# - vm_lifetime_auth +# Same as vm_lifetime, but applied if a valid authentication token is +# included during the request. +# +# - allowed_tags +# If set, restricts tags to those specified in this array. +# +# - domain +# If set, returns a top-level 'domain' JSON key in POST requests +# +# - prefix +# If set, prefixes all created VMs with this string. This should include +# a separator. +# (optional; default: '') +# +# - migration_limit +# When set to any value greater than 0 enable VM migration at checkout. +# When enabled this capability will evaluate a VM for migration when it is requested +# in an effort to maintain a more even distribution of load across compute resources. +# The migration_limit ensures that no more than n migrations will be evaluated at any one time +# and greatly reduces the possibilty of VMs ending up bunched together on a particular host. + +# Example: + +:config: + site_name: 'vmpooler' + logfile: 'c:/temp/vmpooler.log' + task_limit: 10 + timeout: 15 + vm_checktime: 15 + vm_lifetime: 12 + vm_lifetime_auth: 24 + # allowed_tags: + # - 'created_by' + # - 'project' + domain: 'company.com' + prefix: 'poolvm-' + +# :pools: +# +# This section contains a list of virtual machine 'pools' for vmpooler to +# create and maintain. +# +# Available configuration parameters (per-pool): +# +# - name +# The name of the pool. +# (required) +# +# - alias +# Other names this pool can be requested as. +# (optional) +# +# - backingservice +# The backing service which will be used to provision this pool. Default is vsphere +# (optional) +# +# - timeout +# How long (in minutes) before marking a clone as 'failed' and retrying. +# This setting overrides any globally-configured timeout setting. +# (optional; default: '15') +# +# - ready_ttl +# How long (in minutes) to keep VMs in 'ready' queues before destroying. +# (optional) +# +# - template +# The template or virtual machine target to spawn clones from. +# (required) +# +# - size +# The number of waiting VMs to keep in a pool. +# (required) + +# VSphere Backing Service configuration Options +# +# - folder +# The vSphere 'folder' destination for spawned clones. +# (required) +# +# - datastore +# The vSphere 'datastore' destination for spawned clones. +# (required) +# +# - clone_target +# Per-pool option to override the global 'clone_target' cluster. +# (optional) +# + +# Example: + +:pools: + # - name: 'debian-7-i386' + # alias: [ 'debian-7-32' ] + # template: 'Templates/debian-7-i386' + # folder: 'Pooled VMs/debian-7-i386' + # datastore: 'vmstorage' + # size: 2 + # timeout: 15 + # ready_ttl: 1440 + + # - name: 'debian-7-x86_64' + # alias: [ 'debian-7-64', 'debian-7-amd64' ] + # template: 'Templates/debian-7-x86_64' + # folder: 'Pooled VMs/debian-7-x86_64' + # datastore: 'vmstorage' + # size: 2 + # timeout: 15 + # ready_ttl: 1440 + + - name: 'debian-7-i386' + template: 'test1' + backingservice: dummy + size: 5 + timeout: 15 + ready_ttl: 1440 + + - name: 'debian-7-x86_64' + template: 'test1' + backingservice: dummy + size: 5 + timeout: 15 + ready_ttl: 1440 + + - name: 'win-2008r2-x86_64' + template: 'test1' + backingservice: dummy + size: 5 + timeout: 15 + ready_ttl: 1440 + + - name: 'win-2016-x86_64' + template: 'test1' + backingservice: dummy + size: 5 + timeout: 15 + ready_ttl: 1440 + + - name: 'win-2012r2-x86_64' + template: 'test1' + backingservice: dummy + size: 5 + timeout: 15 + ready_ttl: 1440 From 00971c8655700879f961d878ceaa260585ea7d15 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 19:36:56 -0800 Subject: [PATCH 09/23] f pool_manager --- lib/vmpooler/pool_manager.rb | 278 ++++++++++++----------------------- 1 file changed, 98 insertions(+), 180 deletions(-) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 94918b0..4d72478 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -27,7 +27,13 @@ module Vmpooler # DONE def check_pending_vm(vm, pool, timeout, backingservice) Thread.new do - _check_pending_vm(vm, pool, timeout, backingservice) + begin + _check_pending_vm(vm, pool, timeout, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}") + fail_pending_vm(vm, pool, timeout) + raise + end end end @@ -43,38 +49,8 @@ module Vmpooler else fail "VM is not ready" end - rescue => err - $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}") - fail_pending_vm(vm, pool, timeout) - raise end - # def open_socket(host, domain=nil, timeout=5, port=22, &block) - # Timeout.timeout(timeout) do - # target_host = host - # target_host = "#{host}.#{domain}" if domain - # sock = TCPSocket.new target_host, port - # begin - # yield sock if block_given? - # ensure - # sock.close - # end - # end - # end - - # def _check_pending_vm(vm, pool, timeout, vsphere) - # host = vsphere.find_vm(vm) - - # if ! host - # fail_pending_vm(vm, pool, timeout, false) - # return - # end - # open_socket vm - # move_pending_vm_to_ready(vm, pool, host) - # rescue - # fail_pending_vm(vm, pool, timeout) - # end - # DONE def remove_nonexistent_vm(vm, pool) $redis.srem("vmpooler__pending__#{pool}", vm) @@ -120,7 +96,12 @@ module Vmpooler # DONE def check_ready_vm(vm, pool, ttl, backingservice) Thread.new do - _check_ready_vm(vm, pool, ttl, backingservice) + begin + _check_ready_vm(vm, pool, ttl, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : #{err}") + raise + end end end @@ -175,15 +156,17 @@ module Vmpooler end end end - rescue => err - $logger.log('s', "[!] [#{vm['poolname']}] '#{vm['hostname']}' failed while checking a ready vm : #{err}") - raise end # DONE def check_running_vm(vm, pool, ttl, backingservice) Thread.new do - _check_running_vm(vm, pool, ttl, backingservice) + begin + _check_running_vm(vm, pool, ttl, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{err}") + raise + end end end @@ -216,116 +199,52 @@ module Vmpooler # DONE def clone_vm(pool, backingservice) Thread.new do - backingservice.create_vm(pool) + begin + pool_name = pool['name'] + + # Generate a randomized hostname + o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + + # Add VM to Redis inventory ('pending' pool) + $redis.sadd('vmpooler__pending__' + pool_name, new_vmname) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now) + $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) + + begin + start = Time.now + backingservice.create_vm(pool,new_vmname) + finish = '%.2f' % (Time.now - start) + + $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish) + $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned from '#{pool_name}' in #{finish} seconds") + + $metrics.timing("clone.#{pool_name}", finish) + rescue => err + $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed with an error: #{err}") + $redis.srem('vmpooler__pending__' + pool_name, new_vmname) + raise + ensure + $redis.decr('vmpooler__tasks__clone') + end + rescue => err + $logger.log('s', "[!] [#{pool['name']}] failed while preparing to clone with an error: #{err}") + raise + end end end - # Clone a VM - # def clone_vm(template, folder, datastore, target, vsphere) - # Thread.new do - # begin - # vm = {} - - # if template =~ /\// - # templatefolders = template.split('/') - # vm['template'] = templatefolders.pop - # end - - # if templatefolders - # vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) - # else - # fail 'Please provide a full path to the template' - # end - - # if vm['template'].length == 0 - # fail "Unable to find template '#{vm['template']}'!" - # end - - # # Generate a randomized hostname - # o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - # vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join - - # # Add VM to Redis inventory ('pending' pool) - # $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) - # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - # $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - - # # Annotate with creation time, origin template, etc. - # # Add extraconfig options that can be queried by vmtools - # configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - # annotation: JSON.pretty_generate( - # name: vm['hostname'], - # created_by: $config[:vsphere]['username'], - # base_template: vm['template'], - # creation_timestamp: Time.now.utc - # ), - # extraConfig: [ - # { key: 'guestinfo.hostname', - # value: vm['hostname'] - # } - # ] - # ) - - # # Choose a clone target - # if target - # $clone_target = vsphere.find_least_used_host(target) - # elsif $config[:config]['clone_target'] - # $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) - # end - - # # Put the VM in the specified folder and resource pool - # relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - # datastore: vsphere.find_datastore(datastore), - # host: $clone_target, - # diskMoveType: :moveChildMostDiskBacking - # ) - - # # Create a clone spec - # spec = RbVmomi::VIM.VirtualMachineCloneSpec( - # location: relocateSpec, - # config: configSpec, - # powerOn: true, - # template: false - # ) - - # # Clone the VM - # $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") - - # begin - # start = Time.now - # vm[vm['template']].CloneVM_Task( - # folder: vsphere.find_folder(folder), - # name: vm['hostname'], - # spec: spec - # ).wait_for_completion - # finish = '%.2f' % (Time.now - start) - - # $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) - - # $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") - # rescue => err - # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") - # $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) - # raise - # end - - # $redis.decr('vmpooler__tasks__clone') - - # $metrics.timing("clone.#{vm['template']}", finish) - # rescue => err - # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") - # raise - # end - # end - # end - # Destroy a VM # DONE - # TODO These calls should wrap the rescue block, not inside. This traps bad functions. Need to modify all functions def destroy_vm(vm, pool, backingservice) Thread.new do - _destroy_vm(vm, pool, backingservice) + begin + _destroy_vm(vm, pool, backingservice) + rescue => err + $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}") + raise + end end end @@ -339,13 +258,17 @@ module Vmpooler # Auto-expire metadata key $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + start = Time.now + backingservice.destroy_vm(vm,pool) - rescue => err - $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}") - raise + + finish = '%.2f' % (Time.now - start) + $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") + $metrics.timing("destroy.#{pool}", finish) end def create_vm_disk(vm, disk_size, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific Thread.new do _create_vm_disk(vm, disk_size, vsphere) @@ -353,6 +276,7 @@ module Vmpooler end def _create_vm_disk(vm, disk_size, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific host = vsphere.find_vm(vm) @@ -388,6 +312,7 @@ module Vmpooler end def create_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific Thread.new do _create_vm_snapshot(vm, snapshot_name, vsphere) @@ -395,6 +320,7 @@ module Vmpooler end def _create_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific host = vsphere.find_vm(vm) @@ -419,6 +345,7 @@ module Vmpooler end def revert_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific Thread.new do _revert_vm_snapshot(vm, snapshot_name, vsphere) @@ -426,6 +353,7 @@ module Vmpooler end def _revert_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific host = vsphere.find_vm(vm) @@ -447,6 +375,7 @@ module Vmpooler end def check_disk_queue +fail "NOT YET REFACTORED" # TODO This is all vSphere specific $logger.log('d', "[*] [disk_manager] starting worker thread") @@ -461,6 +390,7 @@ module Vmpooler end def _check_disk_queue(vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__disk') @@ -475,6 +405,7 @@ module Vmpooler end def check_snapshot_queue +fail "NOT YET REFACTORED" # TODO This is all vSphere specific $logger.log('d', "[*] [snapshot_manager] starting worker thread") @@ -489,6 +420,7 @@ module Vmpooler end def _check_snapshot_queue(vsphere) +fail "NOT YET REFACTORED" # TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__snapshot') @@ -523,50 +455,44 @@ module Vmpooler # DONE def migrate_vm(vm, pool, backingservice) Thread.new do - _migrate_vm(vm, pool, backingservice) + begin + _migrate_vm(vm, pool, backingservice) + rescue => err + $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") + remove_vmpooler_migration_vm(pool, vm) + end end end # DONE def _migrate_vm(vm, pool, backingservice) - begin - $redis.srem('vmpooler__migrating__' + pool, vm) + $redis.srem('vmpooler__migrating__' + pool, vm) - parent_host_name = backingservice.get_vm_host(vm) - migration_limit = migration_limit $config[:config]['migration_limit'] - migration_count = $redis.scard('vmpooler__migration') + parent_host_name = backingservice.get_vm_host(vm) + migration_limit = migration_limit $config[:config]['migration_limit'] + migration_count = $redis.scard('vmpooler__migration') - if ! migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + if ! migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + return + else + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") return else - if migration_count >= migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") - return + $redis.sadd('vmpooler__migration', vm) + host_name = backingservice.find_least_used_compatible_host(vm) + if host_name == parent_host_name + $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") else - $redis.sadd('vmpooler__migration', vm) - host_name = backingservice.find_least_used_compatible_host(vm) - if host_name == parent_host_name - $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") - else - finish = migrate_vm_and_record_timing(vm, pool, parent_host_name, host_name, backingservice) - $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") - end - remove_vmpooler_migration_vm(pool, vm) + finish = migrate_vm_and_record_timing(vm, pool, parent_host_name, host_name, backingservice) + $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") end + remove_vmpooler_migration_vm(pool, vm) end - rescue => err - $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") - remove_vmpooler_migration_vm(pool, vm) end end -# TODO This is all vSphere specific - # def get_vm_host_info(vm_object) - # parent_host = vm_object.summary.runtime.host - # [parent_host, parent_host.name] - # end - # DONE def remove_vmpooler_migration_vm(pool, vm) begin @@ -596,6 +522,7 @@ module Vmpooler case pool['backingservice'] when 'vsphere' + # TODO what about the helper $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Vsphere.new({ 'metrics' => $metrics}) # TODO Vmpooler::VsphereHelper.new $config[:vsphere], $metrics when 'dummy' $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Dummy.new($config[:backingservice][:dummy]) @@ -613,9 +540,7 @@ module Vmpooler end def _check_pool(pool,backingservice) -puts "CHECK POOL STARTING" # INVENTORY - # DONE!! inventory = {} begin backingservice.vms_in_pool(pool).each do |vm| @@ -639,7 +564,6 @@ puts "CHECK POOL STARTING" end # RUNNING - # DONE!! $redis.smembers("vmpooler__running__#{pool['name']}").each do |vm| if inventory[vm] begin @@ -652,7 +576,6 @@ puts "CHECK POOL STARTING" end # READY - # DONE!! $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm| if inventory[vm] begin @@ -664,7 +587,6 @@ puts "CHECK POOL STARTING" end # PENDING - # DONE!! $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm| pool_timeout = pool['timeout'] || $config[:config]['timeout'] || 15 if inventory[vm] @@ -679,7 +601,6 @@ puts "CHECK POOL STARTING" end # COMPLETED - # DONE!! $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm| if inventory[vm] begin @@ -699,7 +620,6 @@ puts "CHECK POOL STARTING" end # DISCOVERED - # DONE begin $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm| %w(pending ready running completed).each do |queue| @@ -718,7 +638,6 @@ puts "CHECK POOL STARTING" end # MIGRATIONS - # DONE $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm| if inventory[vm] begin @@ -730,7 +649,6 @@ puts "CHECK POOL STARTING" end # REPOPULATE - # DONE ready = $redis.scard("vmpooler__ready__#{pool['name']}") total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready From 24c043c506d539044d048638eee0a3e05adfaf07 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 19:37:14 -0800 Subject: [PATCH 10/23] f base.rb (backingservice) --- lib/vmpooler/backingservice/base.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/vmpooler/backingservice/base.rb b/lib/vmpooler/backingservice/base.rb index a977a31..a508b72 100644 --- a/lib/vmpooler/backingservice/base.rb +++ b/lib/vmpooler/backingservice/base.rb @@ -5,6 +5,13 @@ module Vmpooler # These defs must be overidden in child classes def initialize(options) + @options = options + end + + # returns + # [String] Name of the backing service + def name + 'base' end #def validate_config(config) @@ -15,7 +22,7 @@ module Vmpooler # pool : hashtable from config file # returns # hashtable - # name : name of the device + # name : name of the device <---- TODO is this all? def vms_in_pool(pool) fail "#{self.class.name} does not implement vms_in_pool" end @@ -51,6 +58,7 @@ module Vmpooler # returns # nil if it doesn't exist # Hastable of the VM + # [String] name = Name of the VM # [String] hostname = Name reported by Vmware tools (host.summary.guest.hostName) # [String] template = This is the name of template exposed by the API. It must _match_ the poolname # [String] poolname = Name of the pool the VM is located @@ -93,7 +101,7 @@ module Vmpooler # returns # result: boolean def vm_exists?(vm) - fail "#{self.class.name} does not implement vm_exists?" + !get_vm(vm).nil? end end From 3760463d9a1804eb214d950e6067e06f76ad74fe Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 2 Feb 2017 15:47:26 -0800 Subject: [PATCH 11/23] f base --- lib/vmpooler/backingservice/base.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/vmpooler/backingservice/base.rb b/lib/vmpooler/backingservice/base.rb index a508b72..015be93 100644 --- a/lib/vmpooler/backingservice/base.rb +++ b/lib/vmpooler/backingservice/base.rb @@ -70,10 +70,11 @@ module Vmpooler end # inputs - # pool: string + # pool : hashtable from config file + # new_vmname : string Name the new VM should use # returns - # vm name: string - def create_vm(pool) + # Hashtable of the VM as per get_vm + def create_vm(pool,new_vmname) fail "#{self.class.name} does not implement create_vm" end From 768001319ccba623af5222a56fc416d3655393c9 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 19:41:24 -0800 Subject: [PATCH 12/23] f vmpooler-new.yml --- vmpooler-new.yaml | 58 +++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/vmpooler-new.yaml b/vmpooler-new.yaml index 43f2805..2f4dd6c 100644 --- a/vmpooler-new.yaml +++ b/vmpooler-new.yaml @@ -49,13 +49,24 @@ # # - filename # The filename used to store the backing text file -# (required) # Example: :dummy: filename: 'C:/source/vmpooler/dummy-backing.yaml' + # createvm_max_time: 30 + # createvm_fail_percent: 5 + # migratevm_couldmove_percent: 25 + # migratevm_max_time: 10 + # migratevm_fail_percent: 5 + # vmready_fail_percent: 5 + # destroyvm_max_shutdown_time: 10 + # destroyvm_fail_percent: 50 + # destroyvm_max_time: 10 + # getvm_poweroff_percent: 10 + # getvm_rename_percent: 10 + # :redis: # # This section contains the server hostname and authentication credentials @@ -295,6 +306,7 @@ # - 'project' domain: 'company.com' prefix: 'poolvm-' + migration_limit: 5 # :pools: # @@ -350,12 +362,13 @@ # Example: :pools: + # VSPHERE # - name: 'debian-7-i386' # alias: [ 'debian-7-32' ] # template: 'Templates/debian-7-i386' # folder: 'Pooled VMs/debian-7-i386' # datastore: 'vmstorage' - # size: 2 + # size: 5 # timeout: 15 # ready_ttl: 1440 @@ -368,37 +381,38 @@ # timeout: 15 # ready_ttl: 1440 + # DUMMY - name: 'debian-7-i386' template: 'test1' backingservice: dummy - size: 5 + size: 1 timeout: 15 ready_ttl: 1440 - name: 'debian-7-x86_64' template: 'test1' backingservice: dummy - size: 5 + size: 1 timeout: 15 ready_ttl: 1440 - - name: 'win-2008r2-x86_64' - template: 'test1' - backingservice: dummy - size: 5 - timeout: 15 - ready_ttl: 1440 + # - name: 'win-2008r2-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 - - name: 'win-2016-x86_64' - template: 'test1' - backingservice: dummy - size: 5 - timeout: 15 - ready_ttl: 1440 + # - name: 'win-2016-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 - - name: 'win-2012r2-x86_64' - template: 'test1' - backingservice: dummy - size: 5 - timeout: 15 - ready_ttl: 1440 + # - name: 'win-2012r2-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 From e1576782eb619ca4b44ea363cd4c0f7587fa8a4c Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Wed, 1 Feb 2017 20:37:49 -0800 Subject: [PATCH 13/23] f pool_manager_spec --- spec/vmpooler/pool_manager_spec.rb | 158 +++++++---------------------- 1 file changed, 37 insertions(+), 121 deletions(-) diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index 6bf00bd..c8db82b 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -9,40 +9,41 @@ describe 'Pool Manager' do let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } - let(:host) { double('host') } + let(:host) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm + } subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } describe '#_check_pending_vm' do - let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { double('backingservice') } before do expect(subject).not_to be_nil - $vsphere = vsphere end context 'host not in pool' do it 'calls fail_pending_vm' do - allow(vsphere).to receive(:find_vm).and_return(nil) + allow(backingservice).to receive(:get_vm).and_return(nil) allow(redis).to receive(:hget) - subject._check_pending_vm(vm, pool, timeout, vsphere) + subject._check_pending_vm(vm, pool, timeout, backingservice) end end - context 'host is in pool' do - let(:vm_finder) { double('vm_finder') } - let(:tcpsocket) { double('TCPSocket') } - + context 'host is in pool and ready' do it 'calls move_pending_vm_to_ready' do - allow(subject).to receive(:open_socket).and_return(true) - allow(vsphere).to receive(:find_vm).and_return(vm_finder) - allow(vm_finder).to receive(:summary).and_return(nil) + allow(backingservice).to receive(:get_vm).with(vm).and_return(host) + allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(true) + allow(subject).to receive(:move_pending_vm_to_ready) - expect(vm_finder).to receive(:summary).once - expect(redis).not_to receive(:hget).with(String, 'clone') - - subject._check_pending_vm(vm, pool, timeout, vsphere) + subject._check_pending_vm(vm, pool, timeout, backingservice) end end end @@ -53,38 +54,21 @@ describe 'Pool Manager' do end context 'a host without correct summary' do - it 'does nothing when summary is nil' do - allow(host).to receive(:summary).and_return nil - subject.move_pending_vm_to_ready(vm, pool, host) - end - - it 'does nothing when guest is nil' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return nil - subject.move_pending_vm_to_ready(vm, pool, host) - end - it 'does nothing when hostName is nil' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return nil + host['hostname'] = nil + subject.move_pending_vm_to_ready(vm, pool, host) end it 'does nothing when hostName does not match vm' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return 'adifferentvm' + host['hostname'] = 'adifferentvm' + subject.move_pending_vm_to_ready(vm, pool, host) end end context 'a host with proper summary' do before do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return vm - allow(redis).to receive(:hget) allow(redis).to receive(:smove) allow(redis).to receive(:hset) @@ -145,24 +129,24 @@ describe 'Pool Manager' do describe '#_check_running_vm' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice end it 'does nothing with nil host' do - allow(vsphere).to receive(:find_vm).and_return(nil) + allow(backingservice).to receive(:get_vm).and_return(nil) expect(redis).not_to receive(:smove) - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end context 'valid host' do let(:vm_host) { double('vmhost') } it 'does not move vm when not poweredOn' do - allow(vsphere).to receive(:find_vm).and_return vm_host + allow(backingservice).to receive(:get_vm).and_return vm_host allow(vm_host).to receive(:runtime).and_return true allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff' @@ -170,11 +154,11 @@ describe 'Pool Manager' do expect(redis).not_to receive(:smove) expect(logger).not_to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end it 'moves vm when poweredOn, but past TTL' do - allow(vsphere).to receive(:find_vm).and_return vm_host + allow(backingservice).to receive(:get_vm).and_return vm_host allow(vm_host).to receive(:runtime).and_return true allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn' @@ -182,7 +166,7 @@ describe 'Pool Manager' do expect(redis).to receive(:smove) expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} hours") - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end end end @@ -209,7 +193,7 @@ describe 'Pool Manager' do describe '#_check_pool' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } let(:config) { { config: { task_limit: 10 }, pools: [ {'name' => 'pool1', 'size' => 5} ] @@ -217,7 +201,7 @@ describe 'Pool Manager' do before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice allow(logger).to receive(:log) allow(pool_helper).to receive(:find_folder) allow(redis).to receive(:smembers).with('vmpooler__pending__pool1').and_return([]) @@ -238,14 +222,14 @@ describe 'Pool Manager' do allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0) expect(logger).to receive(:log).with('s', "[!] [pool1] is empty") - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end end end describe '#_stats_running_ready' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { task_limit: 10 }, @@ -255,7 +239,7 @@ describe 'Pool Manager' do before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice allow(logger).to receive(:log) allow(pool_helper).to receive(:find_folder) allow(redis).to receive(:smembers).and_return([]) @@ -275,7 +259,7 @@ describe 'Pool Manager' do expect(metrics).to receive(:gauge).with('ready.pool1', 1) expect(metrics).to receive(:gauge).with('running.pool1', 5) - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end it 'increments metrics when ready with 0 when pool empty' do @@ -286,76 +270,8 @@ describe 'Pool Manager' do expect(metrics).to receive(:gauge).with('ready.pool1', 0) expect(metrics).to receive(:gauge).with('running.pool1', 5) - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end end end - - describe '#_create_vm_snapshot' do - let(:snapshot_manager) { 'snapshot_manager' } - let(:pool_helper) { double('snapshot_manager') } - let(:vsphere) { {snapshot_manager => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - context '(valid host)' do - let(:vm_host) { double('vmhost') } - - it 'creates a snapshot' do - expect(vsphere).to receive(:find_vm).and_return vm_host - expect(logger).to receive(:log) - expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion) - expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s) - expect(logger).to receive(:log) - - subject._create_vm_snapshot('testvm', 'testsnapshot', vsphere) - end - end - end - - describe '#_revert_vm_snapshot' do - let(:snapshot_manager) { 'snapshot_manager' } - let(:pool_helper) { double('snapshot_manager') } - let(:vsphere) { {snapshot_manager => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - context '(valid host)' do - let(:vm_host) { double('vmhost') } - let(:vm_snapshot) { double('vmsnapshot') } - - it 'reverts a snapshot' do - expect(vsphere).to receive(:find_vm).and_return vm_host - expect(vsphere).to receive(:find_snapshot).and_return vm_snapshot - expect(logger).to receive(:log) - expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion) - expect(logger).to receive(:log) - - subject._revert_vm_snapshot('testvm', 'testsnapshot', vsphere) - end - end - end - - describe '#_check_snapshot_queue' do - let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - it 'checks appropriate redis queues' do - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot') - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert') - - subject._check_snapshot_queue(vsphere) - end - end end From e9d2c974c068c008a3403c13d8903c4a773fbb83 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Wed, 1 Feb 2017 20:42:47 -0800 Subject: [PATCH 14/23] f pool_manager_migration_spec --- spec/vmpooler/pool_manager_migration_spec.rb | 90 +++++++++++--------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/spec/vmpooler/pool_manager_migration_spec.rb b/spec/vmpooler/pool_manager_migration_spec.rb index 9fd491b..a4904ab 100644 --- a/spec/vmpooler/pool_manager_migration_spec.rb +++ b/spec/vmpooler/pool_manager_migration_spec.rb @@ -3,84 +3,96 @@ require 'mock_redis' require 'time' describe 'Pool Manager' do - let(:logger) { double('logger') } let(:redis) { MockRedis.new } + let(:logger) { double('logger') } let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { - 'site_name' => 'test pooler', 'migration_limit' => 2, - vsphere: { - 'server' => 'vsphere.puppet.com', - 'username' => 'vmpooler@vsphere.local', - 'password' => '', - 'insecure' => true - }, - pools: [ {'name' => 'pool1', 'size' => 5, 'folder' => 'pool1_folder'} ], - statsd: { 'prefix' => 'stats_prefix'}, - pool_names: [ 'pool1' ] } } } - let(:pool) { config[:config][:pools][0]['name'] } - let(:vm) { - { - 'name' => 'vm1', - 'host' => 'host1', - 'template' => pool, - } + let(:backingservice) { double('backingservice') } + let(:pool) { 'pool1' } + let(:vm) { 'vm1' } + let(:timeout) { 5 } + let(:host) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm } + let(:vm_host_hostname) { 'host1' } + + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + + describe "#migration_limit" do + it 'return false if config is nil' do + expect(subject.migration_limit(nil)).to equal(false) + end + it 'return false if config is 0' do + expect(subject.migration_limit(0)).to equal(false) + end + it 'return nil if config is -1' do + expect(subject.migration_limit(-1)).to equal(nil) + end + it 'return 1 if config is 1' do + expect(subject.migration_limit(1)).to equal(1) + end + it 'return 100 if config is 100' do + expect(subject.migration_limit(100)).to equal(100) + end + end describe '#_migrate_vm' do - let(:vsphere) { double(pool) } - let(:pooler) { Vmpooler::PoolManager.new(config, logger, redis, metrics) } context 'evaluates VM for migration and logs host' do before do - create_migrating_vm vm['name'], pool, redis - allow(vsphere).to receive(:find_vm).and_return(vm) - allow(pooler).to receive(:get_vm_host_info).and_return([{'name' => 'host1'}, 'host1']) + create_migrating_vm vm, pool, redis + allow(backingservice).to receive(:get_vm_host).with(vm).and_return(vm_host_hostname) end it 'logs VM host when migration is disabled' do config[:config]['migration_limit'] = nil - expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be true - expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}") + expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be true + expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) - expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be false + expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be false end it 'verifies that migration_limit greater than or equal to migrations in progress and logs host' do - add_vm_to_migration_set vm['name'], redis + add_vm_to_migration_set vm, redis add_vm_to_migration_set 'vm2', redis - expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}. No migration will be evaluated since the migration_limit has been reached") + expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}. No migration will be evaluated since the migration_limit has been reached") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) end it 'verifies that migration_limit is less than migrations in progress and logs old host, new host and migration time' do - allow(vsphere).to receive(:find_least_used_compatible_host).and_return([{'name' => 'host2'}, 'host2']) - allow(vsphere).to receive(:migrate_vm_host) + allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2') + allow(backingservice).to receive(:migrate_vm_to_host).and_return(true) expect(redis.hget("vmpooler__vm__#{vm['name']}", 'migration_time')) expect(redis.hget("vmpooler__vm__#{vm['name']}", 'checkout_to_migration')) - expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm['name']}' migrated from #{vm['host']} to host2 in 0.00 seconds") + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' migrated from #{vm_host_hostname} to host2 in 0.00 seconds") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) end it 'fails when no suitable host can be found' do error = 'ArgumentError: No target host found' - allow(vsphere).to receive(:find_least_used_compatible_host) - allow(vsphere).to receive(:migrate_vm_host).and_raise(error) + allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2') + allow(backingservice).to receive(:migrate_vm_to_host).and_raise(error) - expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm['name']}' migration failed with an error: #{error}") - - pooler._migrate_vm(vm['name'], pool, vsphere) + expect{subject._migrate_vm(vm, pool, backingservice)}.to raise_error(error) end end end From f256ffac1f6058089ae2a94fc1f4e7380a25e0c7 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Wed, 1 Feb 2017 20:43:25 -0800 Subject: [PATCH 15/23] f pool_manager_snapshot_spec --- spec/vmpooler/pool_manager_snapshot_spec.rb | 83 +++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 spec/vmpooler/pool_manager_snapshot_spec.rb diff --git a/spec/vmpooler/pool_manager_snapshot_spec.rb b/spec/vmpooler/pool_manager_snapshot_spec.rb new file mode 100644 index 0000000..c55257d --- /dev/null +++ b/spec/vmpooler/pool_manager_snapshot_spec.rb @@ -0,0 +1,83 @@ +require 'spec_helper' +require 'time' + +describe 'Pool Manager' do + let(:logger) { double('logger') } + let(:redis) { double('redis') } + let(:metrics) { Vmpooler::DummyStatsd.new } + let(:config) { {} } + let(:pool) { 'pool1' } + let(:vm) { 'vm1' } + let(:timeout) { 5 } + let(:host) { double('host') } + + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + + describe '#_create_vm_snapshot' do + let(:snapshot_manager) { 'snapshot_manager' } + let(:pool_helper) { double('snapshot_manager') } + let(:backingservice) { {snapshot_manager => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + context '(valid host)' do + let(:vm_host) { double('vmhost') } + + it 'creates a snapshot' do + expect(backingservice).to receive(:get_vm).and_return vm_host + expect(logger).to receive(:log) + expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion) + expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s) + expect(logger).to receive(:log) + + subject._create_vm_snapshot('testvm', 'testsnapshot', backingservice) + end + end + end + + describe '#_revert_vm_snapshot' do + let(:snapshot_manager) { 'snapshot_manager' } + let(:pool_helper) { double('snapshot_manager') } + let(:backingservice) { {snapshot_manager => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + context '(valid host)' do + let(:vm_host) { double('vmhost') } + let(:vm_snapshot) { double('vmsnapshot') } + + it 'reverts a snapshot' do + expect(backingservice).to receive(:get_vm).and_return vm_host + expect(backingservice).to receive(:find_snapshot).and_return vm_snapshot + expect(logger).to receive(:log) + expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion) + expect(logger).to receive(:log) + + subject._revert_vm_snapshot('testvm', 'testsnapshot', backingservice) + end + end + end + + describe '#_check_snapshot_queue' do + let(:pool_helper) { double('pool') } + let(:backingservice) { {pool => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + it 'checks appropriate redis queues' do + expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot') + expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert') + + subject._check_snapshot_queue(backingservice) + end + end +end From 91a2207b3f21c5bf4ba56a03d4169831ac6dc9b4 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Wed, 1 Feb 2017 21:05:51 -0800 Subject: [PATCH 16/23] f base_spec --- spec/vmpooler/backingservice/base_spec.rb | 92 +++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 spec/vmpooler/backingservice/base_spec.rb diff --git a/spec/vmpooler/backingservice/base_spec.rb b/spec/vmpooler/backingservice/base_spec.rb new file mode 100644 index 0000000..5256c28 --- /dev/null +++ b/spec/vmpooler/backingservice/base_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' + +# This spec does not really exercise code paths but is merely used +# to enforce that certain methods are defined in the base classes + +describe 'Vmpooler::PoolManager::BackingService::Base' do + let(:config) { {} } + let(:fake_vm) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm + } + + subject { Vmpooler::PoolManager::BackingService::Base.new(config) } + + describe '#name' do + it 'should be base' do + expect(subject.name).to eq('base') + end + end + + describe '#vms_in_pool' do + it 'should raise error' do + expect{subject.vms_in_pool('pool')}.to raise_error(/does not implement vms_in_pool/) + end + end + + describe '#get_vm_host' do + it 'should raise error' do + expect{subject.get_vm_host('vm')}.to raise_error(/does not implement get_vm_host/) + end + end + + describe '#find_least_used_compatible_host' do + it 'should raise error' do + expect{subject.find_least_used_compatible_host('vm')}.to raise_error(/does not implement find_least_used_compatible_host/) + end + end + + describe '#migrate_vm_to_host' do + it 'should raise error' do + expect{subject.migrate_vm_to_host('vm','host')}.to raise_error(/does not implement migrate_vm_to_host/) + end + end + + describe '#get_vm' do + it 'should raise error' do + expect{subject.get_vm('vm')}.to raise_error(/does not implement get_vm/) + end + end + + describe '#create_vm' do + it 'should raise error' do + expect{subject.create_vm('pool','newname')}.to raise_error(/does not implement create_vm/) + end + end + + describe '#destroy_vm' do + it 'should raise error' do + expect{subject.destroy_vm('vm','pool')}.to raise_error(/does not implement destroy_vm/) + end + end + + describe '#is_vm_ready?' do + it 'should raise error' do + expect{subject.is_vm_ready?('vm','pool','timeout')}.to raise_error(/does not implement is_vm_ready?/) + end + end + + describe '#vm_exists?' do + it 'should raise error' do + expect{subject.vm_exists?('vm')}.to raise_error(/does not implement/) + end + + it 'should return true when get_vm is returns an object' do + allow(subject).to receive(:get_vm).with('vm').and_return(fake_vm) + + expect(subject.vm_exists?('vm')).to eq(true) + end + + it 'should return false when get_vm is returns nil' do + allow(subject).to receive(:get_vm).with('vm').and_return(nil) + + expect(subject.vm_exists?('vm')).to eq(false) + end + end +end From c806cfaf76f387584b59195fd1f586dad53dfb83 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 26 Jan 2017 19:36:17 -0800 Subject: [PATCH 17/23] f dummy.rb (backingservice) --- lib/vmpooler/backingservice/dummy.rb | 222 ++++++++++++++++----------- 1 file changed, 130 insertions(+), 92 deletions(-) diff --git a/lib/vmpooler/backingservice/dummy.rb b/lib/vmpooler/backingservice/dummy.rb index 0355f96..486abfd 100644 --- a/lib/vmpooler/backingservice/dummy.rb +++ b/lib/vmpooler/backingservice/dummy.rb @@ -6,11 +6,16 @@ module Vmpooler class Dummy < Vmpooler::PoolManager::BackingService::Base # Fake VM backing service for testing, with initial configuration set in a simple text YAML filename + # or via YAML in the config file def initialize(options) + super(options) + dummyfilename = options['filename'] # TODO Accessing @dummylist is not thread safe :-( Mutexes? - @dummylist = {} + + # This initial_state option is only intended to be used by spec tests + @dummylist = options['initial_state'].nil? ? {} : options['initial_state'] if !dummyfilename.nil? && File.exists?(dummyfilename) @dummylist ||= YAML.load_file(dummyfilename) @@ -23,14 +28,33 @@ module Vmpooler end end + def name + 'dummy' + end + def get_vm(vm) dummy = get_dummy_vm(vm) return nil if dummy.nil? + # Randomly power off the VM + unless dummy['powerstate'] != 'PoweredOn' || @options['getvm_poweroff_percent'].nil? + if 1 + rand(100) <= @options['getvm_poweroff_percent'] + dummy['powerstate'] = 'PoweredOff' + $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off") + end + end + + # Randomly rename the host + unless dummy['hostname'] != dummy['name'] || @options['getvm_rename_percent'].nil? + if 1 + rand(100) <= @options['getvm_rename_percent'] + dummy['hostname'] = 'DUMMY' + dummy['name'] + $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed") + end + end + obj = {} - # TODO Randomly power off the vm? - # TODO Randomly change the hostname of the vm? - obj['hostname'] = dummy['name'] + obj['name'] = dummy['name'] + obj['hostname'] = dummy['hostname'] obj['boottime'] = dummy['boottime'] obj['template'] = dummy['template'] obj['poolname'] = dummy['poolname'] @@ -39,33 +63,41 @@ module Vmpooler obj end - def vm_exists?(vm) - !get_vm(vm).nil? - end - def find_least_used_compatible_host(vm_name) current_vm = get_dummy_vm(vm_name) - # TODO parameterise this (75% chance it will not migrate) - return current_vm['vm_host'] if 1 + rand(100) < 75 + # Unless migratevm_couldmove_percent is specified, don't migrate + return current_vm['vm_host'] if @options['migratevm_couldmove_percent'].nil? - # TODO paramtise this (Simulates a 10 node cluster) - (1 + rand(10)).to_s + # Only migrate if migratevm_couldmove_percent is met + return current_vm['vm_host'] if 1 + rand(100) > @options['migratevm_couldmove_percent'] + + # Simulate a 10 node cluster and randomly pick a different one + new_host = "HOST" + (1 + rand(10)).to_s while new_host == current_vm['vm_host'] + + new_host end def get_vm_host(vm_name) current_vm = get_dummy_vm(vm_name) - current_vm['vm_host'] + current_vm.nil? ? fail("VM #{vm_name} does not exist") : current_vm['vm_host'] end def migrate_vm_to_host(vm_name, dest_host_name) current_vm = get_dummy_vm(vm_name) - # TODO do I need fake a random sleep for ready? - # TODO Should I inject a random error? + # Inject migration delay + unless @options['migratevm_max_time'].nil? + migrate_time = 1 + rand(@options['migratevm_max_time']) + sleep(migrate_time) + end + + # Inject clone failure + unless @options['migratevm_fail_percent'].nil? + fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['migratevm_fail_percent'] + end - sleep(1) current_vm['vm_host'] = dest_host_name true @@ -75,77 +107,70 @@ module Vmpooler host = get_dummy_vm(vm) if !host then return false end if host['poolname'] != pool then return false end - if vm['ready'] then return true end - # TODO do I need fake a random sleep for ready? - # TODO Should I inject a random error? - sleep(2) - host['ready'] = true + if host['ready'] then return true end + Timeout.timeout(timeout) do + while host['dummy_state'] != 'RUNNING' + sleep(2) + host = get_dummy_vm(vm) + end + end + + # Simulate how long it takes from a VM being powered on until + # it's ready to receive a connection + sleep(2) + + unless @options['vmready_fail_percent'].nil? + fail "Dummy Failure for vmready_fail_percent" if 1 + rand(100) <= @options['vmready_fail_percent'] + end + + host['ready'] = true true end - def create_vm(pool) - # This is an async operation - # This code just clones a VM and starts it - # Later checking will move it from the pending to ready queue - Thread.new do - begin - template_name = pool['template'] - pool_name = pool['name'] + def create_vm(pool,dummy_hostname) + template_name = pool['template'] + pool_name = pool['name'] - # Generate a randomized hostname - o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - dummy_hostname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + vm = {} + vm['name'] = dummy_hostname + vm['hostname'] = dummy_hostname + vm['domain'] = 'dummy.local' + # 'vm_template' is the name of the template to use to clone the VM from <----- Do we need this?!?!? + vm['vm_template'] = template_name + # 'template' is the Template name in VM Pooler API, in our case that's the poolname. + vm['template'] = pool_name + vm['poolname'] = pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'UNKNOWN' + get_pool_object(pool_name) + @dummylist['pool'][pool_name] << vm - vm = {} - vm['name'] = dummy_hostname - vm['hostname'] = dummy_hostname - vm['domain'] = 'dummy.local' - vm['vm_template'] = template_name - # 'template' is the Template in API, not the template to create the VM ('vm_template') - vm['template'] = pool_name - vm['poolname'] = pool_name - vm['ready'] = false - vm['boottime'] = Time.now - vm['powerstate'] = 'PoweredOn' - vm['vm_host'] = '1' - get_pool_object(pool_name) - @dummylist['pool'][pool_name] << vm - - # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__' + pool_name, vm['hostname']) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - - $logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'") - begin - start = Time.now - - # TODO do I need fake a random sleep to clone - sleep(2) - - # TODO Inject random clone failure - finish = '%.2f' % (Time.now - start) - - $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) - - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") - rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") - $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) - raise - end - - $redis.decr('vmpooler__tasks__clone') - - $metrics.timing("clone.#{vm['template']}", finish) - dummy_hostname - rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") - raise + $logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'") + begin + # Inject clone time delay + unless @options['createvm_max_time'].nil? + vm['dummy_state'] = 'CLONING' + clone_time = 1 + rand(@options['createvm_max_time']) + sleep(clone_time) end + + # Inject clone failure + unless @options['createvm_fail_percent'].nil? + fail "Dummy Failure for createvm_fail_percent" if 1 + rand(100) <= @options['createvm_fail_percent'] + end + + # Assert the VM is ready for use + vm['dummy_state'] = 'RUNNING' + rescue => err + remove_dummy_vm(dummy_hostname,pool_name) + raise end + + get_vm(dummy_hostname) end def destroy_vm(vm_name,pool) @@ -153,30 +178,43 @@ module Vmpooler if !vm then return false end if vm['poolname'] != pool then return false end - start = Time.now - # Shutdown down the VM if it's poweredOn if vm['powerstate'] = 'PoweredOn' $logger.log('d', "[ ] [#{pool}] '#{vm_name}' is being shut down") - # TODO Use random shutdown interval - sleep(2) + + # Inject shutdown delay time + unless @options['destroyvm_max_shutdown_time'].nil? + shutdown_time = 1 + rand(@options['destroyvm_max_shutdown_time']) + sleep(shutdown_time) + end + vm['powerstate'] = 'PoweredOff' end + # Inject destroy VM delay + unless @options['destroyvm_max_time'].nil? + destroy_time = 1 + rand(@options['destroyvm_max_time']) + sleep(destroy_time) + end + + # Inject destroy VM failure + unless @options['destroyvm_fail_percent'].nil? + fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['destroyvm_fail_percent'] + end + # 'Destroy' the VM - new_poollist = @dummylist['pool'][pool].delete_if { |vm| vm['name'] == vm_name } - @dummylist['pool'][pool] = new_poollist + remove_dummy_vm(vm_name,pool) - # TODO Use random destruction interval - sleep(2) - - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds") - $metrics.timing("destroy.#{pool}", finish) + true end private + def remove_dummy_vm(vm_name,pool) + return if @dummylist['pool'][pool].nil? + new_poollist = @dummylist['pool'][pool].delete_if { |vm| vm['name'] == vm_name } + @dummylist['pool'][pool] = new_poollist + end + # Get's the pool config safely from the in-memory hashtable def get_pool_object(pool_name) @dummylist['pool'] = {} if @dummylist['pool'].nil? From 7154b48e04b503f2205f6909c9f724a183170882 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Wed, 1 Feb 2017 22:06:55 -0800 Subject: [PATCH 18/23] f dummy_spec --- spec/vmpooler/backingservice/dummy_spec.rb | 408 +++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 spec/vmpooler/backingservice/dummy_spec.rb diff --git a/spec/vmpooler/backingservice/dummy_spec.rb b/spec/vmpooler/backingservice/dummy_spec.rb new file mode 100644 index 0000000..1d01d40 --- /dev/null +++ b/spec/vmpooler/backingservice/dummy_spec.rb @@ -0,0 +1,408 @@ +require 'spec_helper' + +describe 'Vmpooler::PoolManager::BackingService::Dummy' do + let(:logger) { double('logger') } + + let (:pool_hash) { + pool = {} + pool['name'] = 'pool1' + + pool + } + + let (:default_config) { + # Construct an initial state for testing + dummylist = {} + dummylist['pool'] = {} + # pool1 is a pool of "normal" VMs + dummylist['pool']['pool1'] = [] + # A normal running VM + vm = {} + vm['name'] = 'vm1' + vm['hostname'] = 'vm1' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool1' + vm['poolname'] = 'pool1' + vm['ready'] = true + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'RUNNING' + dummylist['pool']['pool1'] << vm + + # pool2 is a pool of "abnormal" VMs e.g. PoweredOff etc. + dummylist['pool']['pool2'] = [] + # A freshly provisioned VM that is not ready + vm = {} + vm['name'] = 'vm2' + vm['hostname'] = 'vm2' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool2' + vm['poolname'] = 'pool2' + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'UNKNOWN' + dummylist['pool']['pool2'] << vm + # A freshly provisioned VM that is running but not ready + vm = {} + vm['name'] = 'vm3' + vm['hostname'] = 'vm3' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool2' + vm['poolname'] = 'pool2' + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'RUNNING' + dummylist['pool']['pool2'] << vm + + config = {} + config['initial_state'] = dummylist + + config + } + + let (:config) { default_config } + + subject { Vmpooler::PoolManager::BackingService::Dummy.new(config) } + + before do + allow(logger).to receive(:log) + $logger = logger + end + + describe '#name' do + it 'should be dummy' do + expect(subject.name).to eq('dummy') + end + end + + describe '#vms_in_pool' do + it 'should return [] when pool does not exist' do + pool = pool_hash + pool['name'] = 'pool_does_not_exist' + + vm_list = subject.vms_in_pool(pool) + + expect(vm_list).to eq([]) + end + + it 'should return an array of VMs when pool exists' do + pool = pool_hash + pool['name'] = 'pool1' + + vm_list = subject.vms_in_pool(pool) + + expect(vm_list.count).to eq(1) + end + end + + describe '#get_vm_host' do + it 'should return the hostname when VM exists' do + expect(subject.get_vm_host('vm1')).to eq('HOST1') + end + + it 'should error when VM does not exist' do + expect{subject.get_vm_host('doesnotexist')}.to raise_error(RuntimeError) + end + end + + describe '#find_least_used_compatible_host' do + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to eq('HOST1') + end + + context 'using migratevm_couldmove_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['migratevm_couldmove_percent'] = 0 + config + } + + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to eq('HOST1') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['migratevm_couldmove_percent'] = 100 + config + } + + it 'should return a different host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to_not eq('HOST1') + end + end + + end + end + + describe '#migrate_vm_to_host' do + it 'should move to the new host' do + expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host('vm1')).to eq('NEWHOST') + end + + context 'using migratevm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['migratevm_fail_percent'] = 0 + config + } + + it 'should move to the new host' do + expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host('vm1')).to eq('NEWHOST') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['migratevm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.migrate_vm_to_host('vm1','NEWHOST')}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#get_vm' do + it 'should return the VM when VM exists' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOn') + expect(vm['hostname']).to eq(vm['name']) + end + + it 'should return nil when VM does not exist' do + expect(subject.get_vm('doesnotexist')).to eq(nil) + end + + context 'using getvm_poweroff_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['getvm_poweroff_percent'] = 0 + config + } + + it 'will not power off a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOn') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['getvm_poweroff_percent'] = 100 + config + } + + it 'will power off a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOff') + end + end + end + + context 'using getvm_rename_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['getvm_rename_percent'] = 0 + config + } + + it 'will not rename a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['hostname']).to eq(vm['name']) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['getvm_rename_percent'] = 100 + config + } + + it 'will rename a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['hostname']).to_not eq(vm['name']) + end + end + end + end + + describe '#create_vm' do + it 'should return a new VM' do + expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm') + end + + it 'should increase the number of VMs in the pool' do + pool = pool_hash + old_pool_count = subject.vms_in_pool(pool).count + new_vm = subject.create_vm(pool_hash,'newvm') + expect(subject.vms_in_pool(pool).count).to eq(old_pool_count + 1) + end + + context 'using createvm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['createvm_fail_percent'] = 0 + config + } + + it 'should return a new VM' do + expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['createvm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.create_vm(pool_hash,'newvm')}.to raise_error(/createvm_fail_percent/) + end + + it 'new VM should not exist' do + begin + subject.create_vm(pool_hash,'newvm') + rescue + end + expect(subject.get_vm('newvm')).to eq(nil) + end + end + end + end + + describe '#destroy_vm' do + it 'should return true when destroyed' do + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + + it 'should log if the VM is powered off' do + expect(logger).to receive(:log).with('d', "[ ] [pool1] 'vm1' is being shut down") + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.destroy_vm('doesnotexist','pool1')).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.destroy_vm('vm1','differentpool')).to eq(false) + end + + context 'using destroyvm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['destroyvm_fail_percent'] = 0 + config + } + + it 'should return true when destroyed' do + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['destroyvm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.destroy_vm('vm1','pool1')}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#is_vm_ready?' do + it 'should return true if ready' do + expect(subject.is_vm_ready?('vm1','pool1',0)).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.is_vm_ready?('doesnotexist','pool1',0)).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.is_vm_ready?('vm1','differentpool',0)).to eq(false) + end + + it 'should raise an error if timeout expires' do + expect{subject.is_vm_ready?('vm2','pool2',1)}.to raise_error(Timeout::Error) + end + + it 'should return true if VM becomes ready' do + expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true) + end + + context 'using vmready_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['vmready_fail_percent'] = 0 + config + } + + it 'should return true if VM becomes ready' do + expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['vmready_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.is_vm_ready?('vm3','pool2',1)}.to raise_error(/vmready_fail_percent/) + end + end + end + end + + describe '#vm_exists?' do + it 'should return true when VM exists' do + expect(subject.vm_exists?('vm1')).to eq(true) + end + + it 'should return true when VM does not exist' do + expect(subject.vm_exists?('doesnotexist')).to eq(false) + end + end +end From c067f850068932745c9c56228bc928ec626f756f Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 2 Feb 2017 17:17:53 -0800 Subject: [PATCH 19/23] f pool_manager --- lib/vmpooler/pool_manager.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 4d72478..384fac2 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -19,8 +19,8 @@ module Vmpooler $threads = {} # WARNING DEBUG - $logger.log('d',"Flushing REDIS WARNING!!!") - $redis.flushdb + #$logger.log('d',"Flushing REDIS WARNING!!!") + #$redis.flushdb end # Check the state of a VM @@ -60,7 +60,7 @@ module Vmpooler # DONE def fail_pending_vm(vm, pool, timeout, exists=true) clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone') - return if ! clone_stamp + return true if ! clone_stamp time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60 if time_since_clone > timeout @@ -71,8 +71,12 @@ module Vmpooler remove_nonexistent_vm(vm, pool) end end + + true rescue => err $logger.log('d', "Fail pending VM failed with an error: #{err}") + + false end # DONE From cd38df4eca6b499b5a0aa500b66659ad20fa8c62 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 2 Feb 2017 17:18:00 -0800 Subject: [PATCH 20/23] f pool_manager_spec --- spec/vmpooler/pool_manager_spec.rb | 126 ++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index c8db82b..1f75538 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require 'time' -describe 'Pool Manager' do +describe 'Vmpooler::PoolManager' do let(:logger) { double('logger') } let(:redis) { double('redis') } let(:metrics) { Vmpooler::DummyStatsd.new } @@ -22,6 +22,39 @@ describe 'Pool Manager' do subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + describe '#check_pending_vm' do + let(:backingservice) { double('backingservice') } + + before do + expect(subject).not_to be_nil + end + + it 'calls _check_pending_vm' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice) + + subject.check_pending_vm(vm, pool, timeout, backingservice) + end + + it 'calls fail_pending_vm if an error is raised' do + expect(Thread).to receive(:new).and_yield + allow(logger).to receive(:log) + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error') + expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout) + + expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/) + end + + it 'logs a message if an error is raised' do + expect(Thread).to receive(:new).and_yield + expect(logger).to receive(:log) + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error') + allow(subject).to receive(:fail_pending_vm).with(vm, pool, timeout) + + expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/) + end + end + describe '#_check_pending_vm' do let(:backingservice) { double('backingservice') } @@ -29,7 +62,7 @@ describe 'Pool Manager' do expect(subject).not_to be_nil end - context 'host not in pool' do + context 'VM does not exist' do it 'calls fail_pending_vm' do allow(backingservice).to receive(:get_vm).and_return(nil) allow(redis).to receive(:hget) @@ -37,7 +70,7 @@ describe 'Pool Manager' do end end - context 'host is in pool and ready' do + context 'VM is in pool and ready' do it 'calls move_pending_vm_to_ready' do allow(backingservice).to receive(:get_vm).with(vm).and_return(host) allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(true) @@ -46,6 +79,93 @@ describe 'Pool Manager' do subject._check_pending_vm(vm, pool, timeout, backingservice) end end + + context 'VM is in pool but not ready' do + it 'raises an error' do + allow(backingservice).to receive(:get_vm).with(vm).and_return(host) + allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(false) + allow(subject).to receive(:move_pending_vm_to_ready) + + expect{subject._check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/VM is not ready/) + end + end + end + + describe '#remove_nonexistent_vm' do + before do + expect(subject).not_to be_nil + end + + it 'removes VM from pending in redis' do + allow(logger).to receive(:log) + expect(redis).to receive(:srem).with("vmpooler__pending__#{pool}", vm) + + subject.remove_nonexistent_vm(vm, pool) + end + + it 'logs msg' do + allow(redis).to receive(:srem) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") + + subject.remove_nonexistent_vm(vm, pool) + end + end + + describe '#fail_pending_vm' do + before do + expect(subject).not_to be_nil + allow(logger).to receive(:log) + end + + it 'takes no action if VM is not cloning' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil) + + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) + end + + it 'takes no action if VM is within timeout' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s) + + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) + end + + it 'moves VM to completed queue if VM has exceeded timeout and exists' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm) + + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) + end + + it 'logs message if VM has exceeded timeout and exists' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") + + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) + end + + it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool) + + expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(true) + end + + it 'returns false and swallows error if an error is raised' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(false) + end + + it 'logs message if an error is raised' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + expect(logger).to receive(:log).with('d', String) + + subject.fail_pending_vm(vm, pool, timeout,true) + end + end + + describe 'move_pending_vm_to_ready' do + fail 'todo' end describe '#move_vm_to_ready' do From a094f2024d5e38624e477ad684ec55dc0ede4388 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 2 Feb 2017 22:01:58 -0800 Subject: [PATCH 21/23] f pool_manager_spec --- spec/vmpooler/pool_manager_spec.rb | 63 +++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index 1f75538..3183e09 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -165,7 +165,68 @@ describe 'Vmpooler::PoolManager' do end describe 'move_pending_vm_to_ready' do - fail 'todo' + before do + expect(subject).not_to be_nil + allow(logger).to receive(:log) + allow(Socket).to receive(:getaddrinfo) + end + + context 'when hostname does not match VM name' do + it 'should not take any action' do + expect(logger).to receive(:log).exactly(0).times + expect(Socket).to receive(:getaddrinfo).exactly(0).times + + bad_host = host + bad_host['hostname'] = 'different_name' + + subject.move_pending_vm_to_ready(vm, pool, bad_host) + end + end + + context 'when hostname matches VM name' do + it 'should use the pool in smove' do + allow(redis).to receive(:hget) + allow(redis).to receive(:hset) + expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm) + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should log a message' do + allow(redis).to receive(:hget) + allow(redis).to receive(:hset) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should use clone start time to determine boot timespan' do + allow(redis).to receive(:smove) + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s) + expect(redis).to receive(:hset).with(String,pool + ':' + vm,String) + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should not determine boot timespan if clone start time not set' do + allow(redis).to receive(:smove) + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil) + expect(redis).to receive(:hset).with(String,pool + ':' + vm,String).exactly(0).times + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should raise error if clone start time is not parsable' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + + expect{subject.move_pending_vm_to_ready(vm, pool, host)}.to raise_error(/iamnotparsable_asdate/) + end + end + end + + describe '#check_ready_vm' do + fail "todo" end describe '#move_vm_to_ready' do From 964b4c0d64dbdbd5757cf6026864858186f5d7fe Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Fri, 3 Feb 2017 17:30:34 -0800 Subject: [PATCH 22/23] f pool_manager --- lib/vmpooler/pool_manager.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 384fac2..809d547 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -125,6 +125,7 @@ module Vmpooler $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + return end end From 1e0b936d12760a8ac9183aeb59f4e8e0588a5d76 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Fri, 3 Feb 2017 17:30:42 -0800 Subject: [PATCH 23/23] f pool_manager_spec --- spec/vmpooler/pool_manager_spec.rb | 154 ++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index 3183e09..96b87fa 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -9,7 +9,7 @@ describe 'Vmpooler::PoolManager' do let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } - let(:host) { + let(:default_host) { fake_vm = {} fake_vm['name'] = 'vm1' fake_vm['hostname'] = 'vm1' @@ -19,6 +19,8 @@ describe 'Vmpooler::PoolManager' do fake_vm } + let(:host) { default_host } + let(:max_int) { 4611686018427387903 } # A really big integer (64bit) subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } @@ -226,9 +228,157 @@ describe 'Vmpooler::PoolManager' do end describe '#check_ready_vm' do - fail "todo" + let(:backingservice) { double('backingservice') } + let (:ttl) { 5 } + + before do + expect(subject).not_to be_nil + end + + it 'calls _check_pending_vm' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice) + + subject.check_ready_vm(vm, pool, ttl, backingservice) + end + + it 'logs a message if an error is raised' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice).and_raise('an_error') + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : an_error") + + expect{subject.check_ready_vm(vm, pool, ttl, backingservice)}.to raise_error(/an_error/) + end end + + + + + describe '#_check_ready_vm' do + let(:backingservice) { double('backingservice') } + let (:ttl) { 5 } + + let(:config) { + config = { + 'config': {} + } + # Use the configuration defaults + config[:config]['vm_checktime'] = 15 + + config + } + + before do + expect(subject).not_to be_nil + allow(backingservice).to receive(:get_vm).and_return(host) + allow(logger).to receive(:log) + end + + context 'a VM that does not need to be checked' do + it 'should do nothing' do + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + + context 'a VM that does not exist' do + it 'should log a message' do + allow(backingservice).to receive(:get_vm).and_return(nil) + allow(redis).to receive(:srem) + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue") + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + + it 'should remove the VM from the ready queue' do + allow(backingservice).to receive(:get_vm).and_return(nil) + allow(redis).to receive(:srem).with("vmpooler__ready__#{pool}", vm) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + + context 'an old VM' do + let (:host) { + fake_vm = default_host + fake_vm['boottime'] = Time.new(2001,1,1) + fake_vm + } + + context 'with a TTL of zero' do + it 'should do nothing' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, 0, backingservice) + end + end + + context 'with a TTL longer than the VM lifetime' do + it 'should do nothing' do + # allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, max_int, backingservice) + end + end + + context 'with a TTL shorter than the VM lifetime' do + it 'should move the VM to the completed queue' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) + + subject._check_ready_vm(vm, pool, 1, backingservice) + end + + it 'should log a message' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + end + + context 'a VM that has not previously been checked' do + before do + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(nil) + end + + it 'sets the last check time' do + expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'check', Time) + allow(backingservice).to receive(:is_vm_ready?).and_return(true) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + + end + + # TODO Need to test everything inside the check if statement + + end + + + + + + + + + + + + + + + + + + describe '#move_vm_to_ready' do before do expect(subject).not_to be_nil