diff --git a/.rubocop.yml b/.rubocop.yml index c5cf24d..ee900f8 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -9,6 +9,8 @@ AllCops: - 'scripts/**/*' - 'spec/**/*' - 'vendor/**/*' + - Gemfile + - Rakefile Style/Documentation: Enabled: false @@ -58,3 +60,7 @@ Style/ConditionalAssignment: Next: Enabled: false +# Enforce LF line endings, even when on Windows +Style/EndOfLine: + EnforcedStyle: lf + diff --git a/Gemfile b/Gemfile index 550b983..26f420f 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,7 @@ gem 'redis', '>= 3.2' gem 'sinatra', '>= 1.4' gem 'net-ldap', '<= 0.12.1' # keep compatibility w/ jruby & mri-1.9.3 gem 'statsd-ruby', '>= 1.3.0', :require => 'statsd' +gem 'connection_pool', '>= 2.2.1' # Test deps group :test do diff --git a/README.md b/README.md index 40e29ef..8dcbfbb 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,11 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian- ``` --- -:vsphere: - server: 'vsphere.company.com' - username: 'vmpooler' - password: 'swimsw1msw!m' +:providers: + :vsphere: + server: 'vsphere.company.com' + username: 'vmpooler' + password: 'swimsw1msw!m' :redis: server: 'redis.company.com' @@ -47,12 +48,14 @@ The following YAML configuration sets up two pools, `debian-7-i386` and `debian- pool: 'Pooled VMs/debian-7-i386' datastore: 'vmstorage' size: 5 + provider: vsphere - name: 'debian-7-x86_64' template: 'Templates/debian-7-x86_64' folder: 'Pooled VMs/debian-7-x86_64' pool: 'Pooled VMs/debian-7-x86_64' datastore: 'vmstorage' size: 5 + provider: vsphere ``` See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.yaml.example), for additional configuration options and parameters. diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 953dac1..919ddf9 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -12,7 +12,7 @@ module Vmpooler require 'yaml' require 'set' - %w(api graphite logger pool_manager vsphere_helper statsd dummy_statsd providers).each do |lib| + %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib| begin require "vmpooler/#{lib}" rescue LoadError diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb index 0821d9d..25fa7f6 100644 --- a/lib/vmpooler/api.rb +++ b/lib/vmpooler/api.rb @@ -30,7 +30,7 @@ module Vmpooler use Vmpooler::Dashboard # Load API components - %w(helpers dashboard reroute v1).each do |lib| + %w[helpers dashboard reroute v1].each do |lib| begin require "api/#{lib}" rescue LoadError diff --git a/lib/vmpooler/generic_connection_pool.rb b/lib/vmpooler/generic_connection_pool.rb new file mode 100644 index 0000000..ca18576 --- /dev/null +++ b/lib/vmpooler/generic_connection_pool.rb @@ -0,0 +1,53 @@ +require 'connection_pool' + +module Vmpooler + class PoolManager + class GenericConnectionPool < ConnectionPool + # Extend the ConnectionPool class with instrumentation + # https://github.com/mperham/connection_pool/blob/master/lib/connection_pool.rb + + def initialize(options = {}, &block) + super(options, &block) + @metrics = options[:metrics] + @metric_prefix = options[:metric_prefix] + @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == '' + end + + if Thread.respond_to?(:handle_interrupt) + # MRI + def with_metrics(options = {}) + Thread.handle_interrupt(Exception => :never) do + start = Time.now + conn = checkout(options) + timespan_ms = ((Time.now - start) * 1000).to_i + @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil? + @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil? + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin + @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil? + end + end + end + else + # jruby 1.7.x + def with_metrics(options = {}) + start = Time.now + conn = checkout(options) + timespan_ms = ((Time.now - start) * 1000).to_i + @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil? + @metrics.timing(@metric_prefix + '.waited', timespan_ms) unless @metrics.nil? + begin + yield conn + ensure + checkin + @metrics.gauge(@metric_prefix + '.available', @available.length) unless @metrics.nil? + end + end + end + end + end +end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index c8c033b..35e7966 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -19,37 +19,34 @@ module Vmpooler $threads = {} end + def config + $config + end + # Check the state of a VM def check_pending_vm(vm, pool, timeout, provider) Thread.new do - _check_pending_vm(vm, pool, timeout, provider) - end - end - - def open_socket(host, domain=nil, timeout=5, port=22, &block) - Timeout.timeout(timeout) do - target_host = host - target_host = "#{host}.#{domain}" if domain - sock = TCPSocket.new target_host, port begin - yield sock if block_given? - ensure - sock.close + _check_pending_vm(vm, pool, timeout, provider) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}") + fail_pending_vm(vm, pool, timeout) + raise end end end def _check_pending_vm(vm, pool, timeout, provider) - host = provider.find_vm(vm) - + host = provider.get_vm(pool, vm) if ! host fail_pending_vm(vm, pool, timeout, false) return end - open_socket vm - move_pending_vm_to_ready(vm, pool, host) - rescue - fail_pending_vm(vm, pool, timeout) + if provider.vm_ready?(pool, vm) + move_pending_vm_to_ready(vm, pool, host) + else + fail_pending_vm(vm, pool, timeout) + end end def remove_nonexistent_vm(vm, pool) @@ -57,9 +54,9 @@ module Vmpooler $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") end - def fail_pending_vm(vm, pool, timeout, exists=true) + def fail_pending_vm(vm, pool, timeout, exists = true) clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone') - return if ! clone_stamp + return true if !clone_stamp time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60 if time_since_clone > timeout @@ -70,18 +67,16 @@ module Vmpooler remove_nonexistent_vm(vm, pool) end end + true rescue => err $logger.log('d', "Fail pending VM failed with an error: #{err}") + false end def move_pending_vm_to_ready(vm, pool, host) - if (host.summary) && - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName == vm) - + if host['hostname'] == vm begin - Socket.getaddrinfo(vm, nil) # WTF? + Socket.getaddrinfo(vm, nil) # WTF? I assume this is just priming the local DNS resolver cache?!?! rescue end @@ -91,81 +86,84 @@ module Vmpooler $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm) $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) - $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") + $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") end end def check_ready_vm(vm, pool, ttl, provider) Thread.new do - if ttl > 0 - if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - return - end + begin + _check_ready_vm(vm, pool, ttl, provider) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : #{err}") + raise end + end + end - check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + def _check_ready_vm(vm, pool, ttl, provider) + # Periodically check that the VM is available + check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime']) - if - (!check_stamp) || - (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime']) + host = provider.get_vm(pool, vm) + # Check if the host even exists + if !host + $redis.srem('vmpooler__ready__' + pool, vm) + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in inventory, removed from 'ready' queue") + return + end - $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the hosts TTL has expired + if ttl > 0 + if (((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - host = provider.find_vm(vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + return + end + end - if host - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState != 'poweredOn') + $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the VM is not powered on + unless (host['powerstate'].casecmp('poweredon') == 0) + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + return + end - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + # Check if the hostname has magically changed from underneath Pooler + if (host['hostname'] != vm) + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") + return + end - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") - return - end - - if - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName != vm) - - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") - return - end - else - $redis.srem('vmpooler__ready__' + pool, vm) - - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - open_socket vm - rescue - if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") - else - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") - end - return - end + # Check if the VM is still ready/available + begin + fail "VM #{vm} is not ready" unless provider.vm_ready?(pool, vm) + rescue + if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") + else + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") end end end def check_running_vm(vm, pool, ttl, provider) Thread.new do - _check_running_vm(vm, pool, ttl, provider) + begin + _check_running_vm(vm, pool, ttl, provider) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{err}") + raise + end end end def _check_running_vm(vm, pool, ttl, provider) - host = provider.find_vm(vm) + host = provider.get_vm(pool, vm) if host queue_from, queue_to = 'running', 'completed' @@ -201,237 +199,182 @@ module Vmpooler end def _clone_vm(pool, provider) - template = pool['template'] - folder = pool['folder'] - datastore = pool['datastore'] - target = pool['clone_target'] - vm = {} - - if template =~ /\// - templatefolders = template.split('/') - vm['template'] = templatefolders.pop - end - - if templatefolders - vm[vm['template']] = provider.find_folder(templatefolders.join('/')).find(vm['template']) - else - fail 'Please provide a full path to the template' - end - - if vm['template'].length == 0 - fail "Unable to find template '#{vm['template']}'!" - end + pool_name = pool['name'] # Generate a randomized hostname o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - - # Annotate with creation time, origin template, etc. - # Add extraconfig options that can be queried by vmtools - configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - annotation: JSON.pretty_generate( - name: vm['hostname'], - created_by: $config[:vsphere]['username'], - base_template: vm['template'], - creation_timestamp: Time.now.utc - ), - extraConfig: [ - { key: 'guestinfo.hostname', - value: vm['hostname'] - } - ] - ) - - # Choose a clone target - if target - $clone_target = provider.find_least_used_host(target) - elsif $config[:config]['clone_target'] - $clone_target = provider.find_least_used_host($config[:config]['clone_target']) - end - - # Put the VM in the specified folder and resource pool - relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - datastore: provider.find_datastore(datastore), - host: $clone_target, - diskMoveType: :moveChildMostDiskBacking - ) - - # Create a clone spec - spec = RbVmomi::VIM.VirtualMachineCloneSpec( - location: relocateSpec, - config: configSpec, - powerOn: true, - template: false - ) - - # Clone the VM - $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + $redis.sadd('vmpooler__pending__' + pool_name, new_vmname) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now) + $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) begin + $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'") start = Time.now - vm[vm['template']].CloneVM_Task( - folder: provider.find_folder(folder), - name: vm['hostname'], - spec: spec - ).wait_for_completion + provider.create_vm(pool_name, new_vmname) finish = '%.2f' % (Time.now - start) - $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish) + $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds") - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + $metrics.timing("clone.#{pool_name}", finish) rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") - $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed with an error: #{err}") + $redis.srem('vmpooler__pending__' + pool_name, new_vmname) raise + ensure + $redis.decr('vmpooler__tasks__clone') end - - $redis.decr('vmpooler__tasks__clone') - - $metrics.timing("clone.#{vm['template']}", finish) end # Destroy a VM def destroy_vm(vm, pool, provider) Thread.new do - $redis.srem('vmpooler__completed__' + pool, vm) - $redis.hdel('vmpooler__active__' + pool, vm) - $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) - - # Auto-expire metadata key - $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) - - host = provider.find_vm(vm) - - if host - start = Time.now - - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState == 'poweredOn') - - $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") - host.PowerOffVM_Task.wait_for_completion - end - - host.Destroy_Task.wait_for_completion - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") - $metrics.timing("destroy.#{pool}", finish) + begin + _destroy_vm(vm, pool, provider) + rescue => err + $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}") + raise end end end - def create_vm_disk(vm, disk_size, provider) + def _destroy_vm(vm, pool, provider) + $redis.srem('vmpooler__completed__' + pool, vm) + $redis.hdel('vmpooler__active__' + pool, vm) + $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + + # Auto-expire metadata key + $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + + start = Time.now + + provider.destroy_vm(pool, vm) + + finish = '%.2f' % (Time.now - start) + $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") + $metrics.timing("destroy.#{pool}", finish) + end + + def create_vm_disk(pool_name, vm, disk_size, provider) Thread.new do - _create_vm_disk(vm, disk_size, provider) - end - end - - def _create_vm_disk(vm, disk_size, provider) - host = provider.find_vm(vm) - - if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0)) - $logger.log('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") - - start = Time.now - - template = $redis.hget('vmpooler__vm__' + vm, 'template') - datastore = nil - - $config[:pools].each do |pool| - if pool['name'] == template - datastore = pool['datastore'] - end - end - - if ((! datastore.nil?) && (! datastore.empty?)) - provider.add_disk(host, disk_size, datastore) - - rdisks = $redis.hget('vmpooler__vm__' + vm, 'disk') - disks = rdisks ? rdisks.split(':') : [] - disks.push("+#{disk_size}gb") - $redis.hset('vmpooler__vm__' + vm, 'disk', disks.join(':')) - - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[+] [disk_manager] '#{vm}' attached #{disk_size}gb disk in #{finish} seconds") - else - $logger.log('s', "[+] [disk_manager] '#{vm}' failed to attach disk") + begin + _create_vm_disk(pool_name, vm, disk_size, provider) + rescue => err + $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{err}") + raise end end end - def create_vm_snapshot(vm, snapshot_name, provider) + def _create_vm_disk(pool_name, vm_name, disk_size, provider) + raise("Invalid disk size of '#{disk_size}' passed") if (disk_size.nil?) || (disk_size.empty?) || (disk_size.to_i <= 0) + + $logger.log('s', "[ ] [disk_manager] '#{vm_name}' is attaching a #{disk_size}gb disk") + + start = Time.now + + result = provider.create_disk(pool_name, vm_name, disk_size.to_i) + + finish = '%.2f' % (Time.now - start) + + if result + rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk') + disks = rdisks ? rdisks.split(':') : [] + disks.push("+#{disk_size}gb") + $redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':')) + + $logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds") + else + $logger.log('s', "[+] [disk_manager] '#{vm_name}' failed to attach disk") + end + + result + end + + def create_vm_snapshot(pool_name, vm, snapshot_name, provider) Thread.new do - _create_vm_snapshot(vm, snapshot_name, provider) - end - end - - def _create_vm_snapshot(vm, snapshot_name, provider) - host = provider.find_vm(vm) - - if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?)) - $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being snapshotted") - - start = Time.now - - host.CreateSnapshot_Task( - name: snapshot_name, - description: 'vmpooler', - memory: true, - quiesce: true - ).wait_for_completion - - finish = '%.2f' % (Time.now - start) - - $redis.hset('vmpooler__vm__' + vm, 'snapshot:' + snapshot_name, Time.now.to_s) - - $logger.log('s', "[+] [snapshot_manager] '#{vm}' snapshot created in #{finish} seconds") - end - end - - def revert_vm_snapshot(vm, snapshot_name, provider) - Thread.new do - _revert_vm_snapshot(vm, snapshot_name, provider) - end - end - - def _revert_vm_snapshot(vm, snapshot_name, provider) - host = provider.find_vm(vm) - - if host - snapshot = provider.find_snapshot(host, snapshot_name) - - if snapshot - $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being reverted to snapshot '#{snapshot_name}'") - - start = Time.now - - snapshot.RevertToSnapshot_Task.wait_for_completion - - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[<] [snapshot_manager] '#{vm}' reverted to snapshot in #{finish} seconds") + begin + _create_vm_snapshot(pool_name, vm, snapshot_name, provider) + rescue => err + $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{err}") + raise end end end + def _create_vm_snapshot(pool_name, vm_name, snapshot_name, provider) + $logger.log('s', "[ ] [snapshot_manager] 'Attempting to snapshot #{vm_name} in pool #{pool_name}") + start = Time.now + + result = provider.create_snapshot(pool_name, vm_name, snapshot_name) + + finish = '%.2f' % (Time.now - start) + + if result + $redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s) + $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds") + else + $logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'") + end + + result + end + + def revert_vm_snapshot(pool_name, vm, snapshot_name, provider) + Thread.new do + begin + _revert_vm_snapshot(pool_name, vm, snapshot_name, provider) + rescue => err + $logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{err}") + raise + end + end + end + + def _revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider) + $logger.log('s', "[ ] [snapshot_manager] 'Attempting to revert #{vm_name}' in pool #{pool_name} to snapshot '#{snapshot_name}'") + start = Time.now + + result = provider.revert_snapshot(pool_name, vm_name, snapshot_name) + + finish = '%.2f' % (Time.now - start) + + if result + $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' reverted to snapshot '#{snapshot_name}' in #{finish} seconds") + else + $logger.log('s', "[+] [snapshot_manager] Failed to revert #{vm_name}' in pool #{pool_name} to snapshot '#{snapshot_name}'") + end + + result + end + + def get_pool_name_for_vm(vm_name) + # the 'template' is a bad name. Should really be 'poolname' + $redis.hget('vmpooler__vm__' + vm_name, 'template') + end + + def get_provider_for_pool(pool_name) + provider_name = nil + $config[:pools].each do |pool| + next unless pool['name'] == pool_name + provider_name = pool['provider'] + end + return nil if provider_name.nil? + + $providers[provider_name] + end + def check_disk_queue(maxloop = 0, loop_delay = 5) $logger.log('d', "[*] [disk_manager] starting worker thread") - $providers['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics $threads['disk_manager'] = Thread.new do loop_count = 1 loop do - _check_disk_queue $providers['disk_manager'] + _check_disk_queue sleep(loop_delay) unless maxloop.zero? @@ -442,15 +385,20 @@ module Vmpooler end end - def _check_disk_queue(provider) - vm = $redis.spop('vmpooler__tasks__disk') - - unless vm.nil? + def _check_disk_queue + task_detail = $redis.spop('vmpooler__tasks__disk') + unless task_detail.nil? begin - vm_name, disk_size = vm.split(':') - create_vm_disk(vm_name, disk_size, provider) - rescue - $logger.log('s', "[!] [disk_manager] disk creation appears to have failed") + vm_name, disk_size = task_detail.split(':') + pool_name = get_pool_name_for_vm(vm_name) + raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil? + + provider = get_provider_for_pool(pool_name) + raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil? + + create_vm_disk(pool_name, vm_name, disk_size, provider) + rescue => err + $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{err}") end end end @@ -458,12 +406,10 @@ module Vmpooler def check_snapshot_queue(maxloop = 0, loop_delay = 5) $logger.log('d', "[*] [snapshot_manager] starting worker thread") - $providers['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics - $threads['snapshot_manager'] = Thread.new do loop_count = 1 loop do - _check_snapshot_queue $providers['snapshot_manager'] + _check_snapshot_queue sleep(loop_delay) unless maxloop.zero? @@ -474,26 +420,38 @@ module Vmpooler end end - def _check_snapshot_queue(provider) - vm = $redis.spop('vmpooler__tasks__snapshot') + def _check_snapshot_queue + task_detail = $redis.spop('vmpooler__tasks__snapshot') - unless vm.nil? + unless task_detail.nil? begin - vm_name, snapshot_name = vm.split(':') - create_vm_snapshot(vm_name, snapshot_name, provider) - rescue - $logger.log('s', "[!] [snapshot_manager] snapshot appears to have failed") + vm_name, snapshot_name = task_detail.split(':') + pool_name = get_pool_name_for_vm(vm_name) + raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil? + + provider = get_provider_for_pool(pool_name) + raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil? + + create_vm_snapshot(pool_name, vm_name, snapshot_name, provider) + rescue => err + $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{err}") end end - vm = $redis.spop('vmpooler__tasks__snapshot-revert') + task_detail = $redis.spop('vmpooler__tasks__snapshot-revert') - unless vm.nil? + unless task_detail.nil? begin - vm_name, snapshot_name = vm.split(':') - revert_vm_snapshot(vm_name, snapshot_name, provider) - rescue - $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed") + vm_name, snapshot_name = task_detail.split(':') + pool_name = get_pool_name_for_vm(vm_name) + raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil? + + provider = get_provider_for_pool(pool_name) + raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil? + + revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider) + rescue => err + $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{err}") end end end @@ -504,50 +462,46 @@ module Vmpooler migration_limit if migration_limit >= 1 end - def migrate_vm(vm, pool, provider) + def migrate_vm(vm_name, pool_name, provider) Thread.new do - _migrate_vm(vm, pool, provider) + begin + _migrate_vm(vm_name, pool_name, provider) + rescue => err + $logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{err}") + remove_vmpooler_migration_vm(pool_name, vm_name) + end end end - def _migrate_vm(vm, pool, provider) - begin - $redis.srem('vmpooler__migrating__' + pool, vm) - vm_object = provider.find_vm(vm) - parent_host, parent_host_name = get_vm_host_info(vm_object) - migration_limit = migration_limit $config[:config]['migration_limit'] - migration_count = $redis.scard('vmpooler__migration') + def _migrate_vm(vm_name, pool_name, provider) + $redis.srem('vmpooler__migrating__' + pool_name, vm_name) - if ! migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + parent_host_name = provider.get_vm_host(pool_name, vm_name) + raise('Unable to determine which host the VM is running on') if parent_host_name.nil? + migration_limit = migration_limit $config[:config]['migration_limit'] + migration_count = $redis.scard('vmpooler__migration') + + if ! migration_limit + $logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{parent_host_name}") + return + else + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") return else - if migration_count >= migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") - return + $redis.sadd('vmpooler__migration', vm_name) + host_name = provider.find_least_used_compatible_host(vm_name) + if host_name == parent_host_name + $logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{parent_host_name}") else - $redis.sadd('vmpooler__migration', vm) - host, host_name = provider.find_least_used_compatible_host(vm_object) - if host == parent_host - $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") - else - finish = migrate_vm_and_record_timing(vm_object, vm, pool, host, parent_host_name, host_name, provider) - $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") - end - remove_vmpooler_migration_vm(pool, vm) + finish = migrate_vm_and_record_timing(vm_name, pool_name, parent_host_name, host_name, provider) + $logger.log('s', "[>] [#{pool_name}] '#{vm_name}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") end + remove_vmpooler_migration_vm(pool_name, vm_name) end - rescue => err - $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") - remove_vmpooler_migration_vm(pool, vm) end end - def get_vm_host_info(vm_object) - parent_host = vm_object.summary.runtime.host - [parent_host, parent_host.name] - end - def remove_vmpooler_migration_vm(pool, vm) begin $redis.srem('vmpooler__migration', vm) @@ -556,11 +510,11 @@ module Vmpooler end end - def migrate_vm_and_record_timing(vm_object, vm_name, pool, host, source_host_name, dest_host_name, provider) + def migrate_vm_and_record_timing(vm_name, pool_name, source_host_name, dest_host_name, provider) start = Time.now - provider.migrate_vm_host(vm_object, host) + provider.migrate_vm_to_host(pool_name, vm_name, dest_host_name) finish = '%.2f' % (Time.now - start) - $metrics.timing("migrate.#{pool}", finish) + $metrics.timing("migrate.#{pool_name}", finish) $metrics.increment("migrate_from.#{source_host_name}") $metrics.increment("migrate_to.#{dest_host_name}") checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget("vmpooler__vm__#{vm_name}", 'checkout'))) @@ -572,18 +526,24 @@ module Vmpooler def check_pool(pool, maxloop = 0, loop_delay = 5) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") - $providers[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics - $threads[pool['name']] = Thread.new do - loop_count = 1 - loop do - _check_pool(pool, $providers[pool['name']]) - sleep(loop_delay) + begin + loop_count = 1 + provider = get_provider_for_pool(pool['name']) + raise("Could not find provider '#{pool['provider']}") if provider.nil? + loop do + _check_pool(pool, provider) - unless maxloop.zero? - break if loop_count >= maxloop - loop_count += 1 + sleep(loop_delay) + + unless maxloop.zero? + break if loop_count >= maxloop + loop_count += 1 + end end + rescue => err + $logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{err}") + raise end end end @@ -592,9 +552,7 @@ module Vmpooler # INVENTORY inventory = {} begin - base = provider.find_folder(pool['folder']) - - base.childEntity.each do |vm| + provider.vms_in_pool(pool['name']).each do |vm| if (! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) && @@ -673,7 +631,7 @@ module Vmpooler # DISCOVERED begin $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm| - %w(pending ready running completed).each do |queue| + %w[pending ready running completed].each do |queue| if $redis.sismember("vmpooler__#{queue}__#{pool['name']}", vm) $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue") $redis.srem("vmpooler__discovered__#{pool['name']}", vm) @@ -736,6 +694,17 @@ module Vmpooler raise end + def create_provider_object(config, logger, metrics, provider_name, options) + case provider_name + when 'vsphere' + Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, provider_name, options) + when 'dummy' + Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, provider_name, options) + else + raise("Provider '#{provider_name}' is unknown") + end + end + def execute!(maxloop = 0, loop_delay = 1) $logger.log('d', 'starting vmpooler') @@ -744,6 +713,32 @@ module Vmpooler # Clear out vmpooler__migrations since stale entries may be left after a restart $redis.del('vmpooler__migration') + # Copy vSphere settings to correct location. This happens with older configuration files + if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?) + $logger.log('d', "[!] Detected an older configuration file. Copying the settings from ':vsphere:' to ':providers:/:vsphere:'") + $config[:providers] = {} if $config[:providers].nil? + $config[:providers][:vsphere] = $config[:vsphere] + end + + # Set default provider for all pools that do not have one defined + $config[:pools].each do |pool| + if pool['provider'].nil? + $logger.log('d', "[!] Setting provider for pool '#{pool['name']}' to 'vsphere' as default") + pool['provider'] = 'vsphere' + end + end + + # Create the providers + $config[:pools].each do |pool| + provider_name = pool['provider'] + begin + $providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_name, {}) if $providers[provider_name].nil? + rescue => err + $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{err}") + raise + end + end + loop_count = 1 loop do if ! $threads['disk_manager'] diff --git a/lib/vmpooler/providers.rb b/lib/vmpooler/providers.rb index f640163..9d2f0ae 100644 --- a/lib/vmpooler/providers.rb +++ b/lib/vmpooler/providers.rb @@ -1,4 +1,4 @@ -%w(base vsphere).each do |lib| +%w[base dummy vsphere].each do |lib| begin require "vmpooler/providers/#{lib}" rescue LoadError diff --git a/lib/vmpooler/providers/base.rb b/lib/vmpooler/providers/base.rb index 579124b..23b0bfb 100644 --- a/lib/vmpooler/providers/base.rb +++ b/lib/vmpooler/providers/base.rb @@ -18,7 +18,15 @@ module Vmpooler @metrics = metrics @provider_name = name + # Ensure that there is not a nil provider configuration + @config[:providers] = {} if @config[:providers].nil? + @config[:providers][@provider_name] = {} if provider_config.nil? + + # Ensure that there is not a nil pool configuration + @config[:pools] = {} if @config[:pools].nil? + @provider_options = options + logger.log('s', "[!] Creating provider '#{name}'") end # Helper Methods @@ -41,7 +49,7 @@ module Vmpooler def provider_config @config[:providers].each do |provider| # Convert the symbol from the config into a string for comparison - return provider[1] if provider[0].to_s == @provider_name + return (provider[1].nil? ? {} : provider[1]) if provider[0].to_s == @provider_name end nil @@ -60,6 +68,16 @@ module Vmpooler @provider_name end + # returns + # Array[String] : Array of pool names this provider services + def provided_pools + list = [] + @config[:pools].each do |pool| + list << pool['name'] if pool['provider'] == name + end + list + end + # Pool Manager Methods # inputs @@ -146,8 +164,8 @@ module Vmpooler # [String] new_snapshot_name : Name of the new snapshot to create # returns # [Boolean] : true if success, false if snapshot could not be created + # Raises RuntimeError if the Pool does not exist # Raises RuntimeError if the VM does not exist - # Raises RuntimeError if the snapshot already exists def create_snapshot(_pool_name, _vm_name, _new_snapshot_name) raise("#{self.class.name} does not implement create_snapshot") end @@ -158,8 +176,9 @@ module Vmpooler # [String] snapshot_name : Name of the snapshot to restore to # returns # [Boolean] : true if success, false if snapshot could not be revertted + # Raises RuntimeError if the Pool does not exist # Raises RuntimeError if the VM does not exist - # Raises RuntimeError if the snapshot already exists + # Raises RuntimeError if the snapshot does not exist def revert_snapshot(_pool_name, _vm_name, _snapshot_name) raise("#{self.class.name} does not implement revert_snapshot") end diff --git a/lib/vmpooler/providers/dummy.rb b/lib/vmpooler/providers/dummy.rb new file mode 100644 index 0000000..1173511 --- /dev/null +++ b/lib/vmpooler/providers/dummy.rb @@ -0,0 +1,402 @@ +require 'yaml' + +module Vmpooler + class PoolManager + class Provider + class Dummy < Vmpooler::PoolManager::Provider::Base + # Fake VM Provider for testing + + def initialize(config, logger, metrics, name, options) + super(config, logger, metrics, name, options) + dummyfilename = provider_config['filename'] + + # This initial_state option is only intended to be used by spec tests + @dummylist = provider_options['initial_state'].nil? ? {} : provider_options['initial_state'] + + @dummylist = YAML.load_file(dummyfilename) if !dummyfilename.nil? && File.exist?(dummyfilename) + + # Even though this code is using Mutexes, it's still no 100% atomic i.e. it's still possible for + # duplicate actions to put the @dummylist hashtable into a bad state, for example; + # Deleting a VM while it's in the middle of adding a disk. + @write_lock = Mutex.new + + # Create a dummy connection pool + connpool_size = provider_config['connection_pool_size'].nil? ? 1 : provider_config['connection_pool_size'].to_i + connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 10 : provider_config['connection_pool_timeout'].to_i + logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}") + @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + metric_prefix: "#{name}_provider_connection_pool", + size: connpool_size, + timeout: connpool_timeout + ) do + # Create a mock connection object + new_conn = { create_timestamp: Time.now, conn_id: rand(2048).to_s } + logger.log('d', "[#{name}] ConnPool - Creating a connection object ID #{new_conn[:conn_id]}") + new_conn + end + end + + def name + 'dummy' + end + + def vms_in_pool(pool_name) + vmlist = [] + + @connection_pool.with_metrics do |_conn| + get_dummy_pool_object(pool_name).each do |vm| + vmlist << { 'name' => vm['name'] } + end + end + + vmlist + end + + def get_vm_host(pool_name, vm_name) + current_vm = nil + @connection_pool.with_metrics do |_conn| + current_vm = get_dummy_vm(pool_name, vm_name) + end + + current_vm.nil? ? raise("VM #{vm_name} does not exist") : current_vm['vm_host'] + end + + def find_least_used_compatible_host(pool_name, vm_name) + current_vm = nil + @connection_pool.with_metrics do |_conn| + current_vm = get_dummy_vm(pool_name, vm_name) + end + + # Unless migratevm_couldmove_percent is specified, don't migrate + return current_vm['vm_host'] if provider_config['migratevm_couldmove_percent'].nil? + + # Only migrate if migratevm_couldmove_percent is met + return current_vm['vm_host'] if 1 + rand(100) > provider_config['migratevm_couldmove_percent'] + + # Simulate a 10 node cluster and randomly pick a different one + new_host = 'HOST' + (1 + rand(10)).to_s while new_host == current_vm['vm_host'] + + new_host + end + + def migrate_vm_to_host(pool_name, vm_name, dest_host_name) + @connection_pool.with_metrics do |_conn| + current_vm = get_dummy_vm(pool_name, vm_name) + + # Inject migration delay + unless provider_config['migratevm_max_time'].nil? + migrate_time = 1 + rand(provider_config['migratevm_max_time']) + sleep(migrate_time) + end + + # Inject clone failure + unless provider_config['migratevm_fail_percent'].nil? + raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['migratevm_fail_percent'] + end + + @write_lock.synchronize do + current_vm = get_dummy_vm(pool_name, vm_name) + current_vm['vm_host'] = dest_host_name + write_backing_file + end + end + + true + end + + def get_vm(pool_name, vm_name) + obj = {} + @connection_pool.with_metrics do |_conn| + dummy = get_dummy_vm(pool_name, vm_name) + return nil if dummy.nil? + + # Randomly power off the VM + unless dummy['powerstate'] != 'PoweredOn' || provider_config['getvm_poweroff_percent'].nil? + if 1 + rand(100) <= provider_config['getvm_poweroff_percent'] + @write_lock.synchronize do + dummy = get_dummy_vm(pool_name, vm_name) + dummy['powerstate'] = 'PoweredOff' + write_backing_file + end + logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off") + end + end + + # Randomly rename the host + unless dummy['hostname'] != dummy['name'] || provider_config['getvm_rename_percent'].nil? + if 1 + rand(100) <= provider_config['getvm_rename_percent'] + @write_lock.synchronize do + dummy = get_dummy_vm(pool_name, vm_name) + dummy['hostname'] = 'DUMMY' + dummy['name'] + write_backing_file + end + logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed") + end + end + + obj['name'] = dummy['name'] + obj['hostname'] = dummy['hostname'] + obj['boottime'] = dummy['boottime'] + obj['template'] = dummy['template'] + obj['poolname'] = dummy['poolname'] + obj['powerstate'] = dummy['powerstate'] + obj['snapshots'] = dummy['snapshots'] + end + + obj + end + + def create_vm(pool_name, dummy_hostname) + pool = pool_config(pool_name) + raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? + + template_name = pool['template'] + + vm = {} + vm['name'] = dummy_hostname + vm['hostname'] = dummy_hostname + vm['domain'] = 'dummy.local' + # 'vm_template' is the name of the template to use to clone the VM from <----- Do we need this?!?!? + vm['vm_template'] = template_name + # 'template' is the Template name in VM Pooler API, in our case that's the poolname. + vm['template'] = pool_name + vm['poolname'] = pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'UNKNOWN' + vm['snapshots'] = [] + vm['disks'] = [] + + # Make sure the pool exists in the dummy list + @write_lock.synchronize do + get_dummy_pool_object(pool_name) + @dummylist['pool'][pool_name] << vm + write_backing_file + end + + logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'") + + @connection_pool.with_metrics do |_conn| + # Inject clone time delay + unless provider_config['createvm_max_time'].nil? + @write_lock.synchronize do + vm['dummy_state'] = 'CLONING' + write_backing_file + end + clone_time = 1 + rand(provider_config['createvm_max_time']) + sleep(clone_time) + end + + begin + # Inject clone failure + unless provider_config['createvm_fail_percent'].nil? + raise('Dummy Failure for createvm_fail_percent') if 1 + rand(100) <= provider_config['createvm_fail_percent'] + end + + # Assert the VM is ready for use + @write_lock.synchronize do + vm['dummy_state'] = 'RUNNING' + write_backing_file + end + rescue => _err + @write_lock.synchronize do + remove_dummy_vm(pool_name, dummy_hostname) + write_backing_file + end + raise + end + end + + get_vm(pool_name, dummy_hostname) + end + + def create_disk(pool_name, vm_name, disk_size) + @connection_pool.with_metrics do |_conn| + vm_object = get_dummy_vm(pool_name, vm_name) + raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil? + + # Inject create time delay + unless provider_config['createdisk_max_time'].nil? + delay = 1 + rand(provider_config['createdisk_max_time']) + sleep(delay) + end + + # Inject create failure + unless provider_config['createdisk_fail_percent'].nil? + raise('Dummy Failure for createdisk_fail_percent') if 1 + rand(100) <= provider_config['createdisk_fail_percent'] + end + + @write_lock.synchronize do + vm_object = get_dummy_vm(pool_name, vm_name) + vm_object['disks'] << disk_size + write_backing_file + end + end + + true + end + + def create_snapshot(pool_name, vm_name, snapshot_name) + @connection_pool.with_metrics do |_conn| + vm_object = get_dummy_vm(pool_name, vm_name) + raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil? + + # Inject create time delay + unless provider_config['createsnapshot_max_time'].nil? + delay = 1 + rand(provider_config['createsnapshot_max_time']) + sleep(delay) + end + + # Inject create failure + unless provider_config['createsnapshot_fail_percent'].nil? + raise('Dummy Failure for createsnapshot_fail_percent') if 1 + rand(100) <= provider_config['createsnapshot_fail_percent'] + end + + @write_lock.synchronize do + vm_object = get_dummy_vm(pool_name, vm_name) + vm_object['snapshots'] << snapshot_name + write_backing_file + end + end + + true + end + + def revert_snapshot(pool_name, vm_name, snapshot_name) + vm_object = nil + @connection_pool.with_metrics do |_conn| + vm_object = get_dummy_vm(pool_name, vm_name) + raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil? + + # Inject create time delay + unless provider_config['revertsnapshot_max_time'].nil? + delay = 1 + rand(provider_config['revertsnapshot_max_time']) + sleep(delay) + end + + # Inject create failure + unless provider_config['revertsnapshot_fail_percent'].nil? + raise('Dummy Failure for revertsnapshot_fail_percent') if 1 + rand(100) <= provider_config['revertsnapshot_fail_percent'] + end + end + + vm_object['snapshots'].include?(snapshot_name) + end + + def destroy_vm(pool_name, vm_name) + @connection_pool.with_metrics do |_conn| + vm = get_dummy_vm(pool_name, vm_name) + return false if vm.nil? + return false if vm['poolname'] != pool_name + + # Shutdown down the VM if it's poweredOn + if vm['powerstate'] == 'PoweredOn' + logger.log('d', "[ ] [#{pool_name}] '#{vm_name}' is being shut down") + + # Inject shutdown delay time + unless provider_config['destroyvm_max_shutdown_time'].nil? + shutdown_time = 1 + rand(provider_config['destroyvm_max_shutdown_time']) + sleep(shutdown_time) + end + + @write_lock.synchronize do + vm = get_dummy_vm(pool_name, vm_name) + vm['powerstate'] = 'PoweredOff' + write_backing_file + end + end + + # Inject destroy VM delay + unless provider_config['destroyvm_max_time'].nil? + destroy_time = 1 + rand(provider_config['destroyvm_max_time']) + sleep(destroy_time) + end + + # Inject destroy VM failure + unless provider_config['destroyvm_fail_percent'].nil? + raise('Dummy Failure for migratevm_fail_percent') if 1 + rand(100) <= provider_config['destroyvm_fail_percent'] + end + + # 'Destroy' the VM + @write_lock.synchronize do + remove_dummy_vm(pool_name, vm_name) + write_backing_file + end + end + + true + end + + def vm_ready?(pool_name, vm_name) + @connection_pool.with_metrics do |_conn| + vm_object = get_dummy_vm(pool_name, vm_name) + return false if vm_object.nil? + return false if vm_object['poolname'] != pool_name + return true if vm_object['ready'] + + timeout = provider_config['is_ready_timeout'] || 5 + + Timeout.timeout(timeout) do + while vm_object['dummy_state'] != 'RUNNING' + sleep(2) + vm_object = get_dummy_vm(pool_name, vm_name) + end + end + + # Simulate how long it takes from a VM being powered on until + # it's ready to receive a connection + sleep(2) + + unless provider_config['vmready_fail_percent'].nil? + raise('Dummy Failure for vmready_fail_percent') if 1 + rand(100) <= provider_config['vmready_fail_percent'] + end + + @write_lock.synchronize do + vm_object['ready'] = true + write_backing_file + end + end + + true + end + + private + + # Note - NEVER EVER use the @write_lock object in the private methods!!!! Deadlocks will ensue + + def remove_dummy_vm(pool_name, vm_name) + return if @dummylist['pool'][pool_name].nil? + new_poollist = @dummylist['pool'][pool_name].delete_if { |vm| vm['name'] == vm_name } + @dummylist['pool'][pool_name] = new_poollist + end + + # Get's the pool config safely from the in-memory hashtable + def get_dummy_pool_object(pool_name) + @dummylist['pool'] = {} if @dummylist['pool'].nil? + @dummylist['pool'][pool_name] = [] if @dummylist['pool'][pool_name].nil? + + @dummylist['pool'][pool_name] + end + + def get_dummy_vm(pool_name, vm_name) + return nil if @dummylist['pool'][pool_name].nil? + + @dummylist['pool'][pool_name].each do |poolvm| + return poolvm if poolvm['name'] == vm_name + end + + nil + end + + def write_backing_file + dummyfilename = provider_config['filename'] + return if dummyfilename.nil? + File.open(dummyfilename, 'w') { |file| file.write(YAML.dump(@dummylist)) } + end + end + end + end +end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb index 38c6d51..ee6b3bc 100644 --- a/lib/vmpooler/providers/vsphere.rb +++ b/lib/vmpooler/providers/vsphere.rb @@ -2,10 +2,37 @@ module Vmpooler class PoolManager class Provider class VSphere < Vmpooler::PoolManager::Provider::Base + # The connection_pool method is normally used only for testing + attr_reader :connection_pool + def initialize(config, logger, metrics, name, options) super(config, logger, metrics, name, options) - @credentials = provider_config - @conf = global_config[:config] + + task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i + # The default connection pool size is: + # Whatever is biggest from: + # - How many pools this provider services + # - Maximum number of cloning tasks allowed + # - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc. + default_connpool_size = [provided_pools.count, task_limit, 2].max + connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i + # The default connection pool timeout should be quite large - 60 seconds + connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i + logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}") + @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + metric_prefix: "#{name}_provider_connection_pool", + size: connpool_size, + timeout: connpool_timeout + ) do + logger.log('d', "[#{name}] Connection Pool - Creating a connection object") + # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve + # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection + # object. Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the + # Hash can change, and is preserved across invocations. + new_conn = connect_to_vsphere + { connection: new_conn } + end end def name @@ -13,154 +40,162 @@ module Vmpooler end def vms_in_pool(pool_name) - connection = get_connection - - foldername = pool_config(pool_name)['folder'] - folder_object = find_folder(foldername, connection) - vms = [] + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + foldername = pool_config(pool_name)['folder'] + folder_object = find_folder(foldername, connection) - return vms if folder_object.nil? + return vms if folder_object.nil? - folder_object.childEntity.each do |vm| - vms << { 'name' => vm.name } + folder_object.childEntity.each do |vm| + vms << { 'name' => vm.name } + end end - vms end def get_vm_host(_pool_name, vm_name) - connection = get_connection - - vm_object = find_vm(vm_name, connection) - return nil if vm_object.nil? - host_name = nil - host_name = vm_object.summary.runtime.host.name if vm_object.summary && vm_object.summary.runtime && vm_object.summary.runtime.host + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + return host_name if vm_object.nil? + + host_name = vm_object.summary.runtime.host.name if vm_object.summary && vm_object.summary.runtime && vm_object.summary.runtime.host + end host_name end def find_least_used_compatible_host(_pool_name, vm_name) - connection = get_connection + hostname = nil + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) - vm_object = find_vm(vm_name, connection) + return hostname if vm_object.nil? + host_object = find_least_used_vpshere_compatible_host(vm_object) - return nil if vm_object.nil? - host_object = find_least_used_vpshere_compatible_host(vm_object) - - return nil if host_object.nil? - host_object[0].name + return hostname if host_object.nil? + hostname = host_object[0].name + end + hostname end def migrate_vm_to_host(pool_name, vm_name, dest_host_name) pool = pool_config(pool_name) raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? - connection = get_connection + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil? - vm_object = find_vm(vm_name, connection) - raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil? + target_cluster_name = get_target_cluster_from_config(pool_name) + cluster = find_cluster(target_cluster_name, connection) + raise("Pool #{pool_name} specifies cluster #{target_cluster_name} which does not exist for the provider #{name}") if cluster.nil? - target_cluster_name = get_target_cluster_from_config(pool_name) - cluster = find_cluster(target_cluster_name, connection) - raise("Pool #{pool_name} specifies cluster #{target_cluster_name} which does not exist for the provider #{name}") if cluster.nil? - - # Go through each host and initiate a migration when the correct host name is found - cluster.host.each do |host| - if host.name == dest_host_name - migrate_vm_host(vm_object, host) - return true + # Go through each host and initiate a migration when the correct host name is found + cluster.host.each do |host| + if host.name == dest_host_name + migrate_vm_host(vm_object, host) + return true + end end end - false end def get_vm(_pool_name, vm_name) - connection = get_connection + vm_hash = nil + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + return vm_hash if vm_object.nil? - vm_object = find_vm(vm_name, connection) - return nil if vm_object.nil? - - vm_folder_path = get_vm_folder_path(vm_object) - # Find the pool name based on the folder path - pool_name = nil - template_name = nil - global_config[:pools].each do |pool| - if pool['folder'] == vm_folder_path - pool_name = pool['name'] - template_name = pool['template'] + vm_folder_path = get_vm_folder_path(vm_object) + # Find the pool name based on the folder path + pool_name = nil + template_name = nil + global_config[:pools].each do |pool| + if pool['folder'] == vm_folder_path + pool_name = pool['name'] + template_name = pool['template'] + end end - end - generate_vm_hash(vm_object, template_name, pool_name) + vm_hash = generate_vm_hash(vm_object, template_name, pool_name) + end + vm_hash end def create_vm(pool_name, new_vmname) pool = pool_config(pool_name) raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? + vm_hash = nil + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + # Assume all pool config is valid i.e. not missing + template_path = pool['template'] + target_folder_path = pool['folder'] + target_datastore = pool['datastore'] + target_cluster_name = get_target_cluster_from_config(pool_name) - connection = get_connection + # Extract the template VM name from the full path + raise("Pool #{pool_name} did specify a full path for the template for the provider #{name}") unless template_path =~ /\// + templatefolders = template_path.split('/') + template_name = templatefolders.pop - # Assume all pool config is valid i.e. not missing - template_path = pool['template'] - target_folder_path = pool['folder'] - target_datastore = pool['datastore'] - target_cluster_name = get_target_cluster_from_config(pool_name) + # Get the actual objects from vSphere + template_folder_object = find_folder(templatefolders.join('/'), connection) + raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil? - # Extract the template VM name from the full path - raise("Pool #{pool_name} did specify a full path for the template for the provider #{name}") unless template_path =~ /\// - templatefolders = template_path.split('/') - template_name = templatefolders.pop + template_vm_object = template_folder_object.find(template_name) + raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil? - # Get the actual objects from vSphere - template_folder_object = find_folder(templatefolders.join('/'), connection) - raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil? + # Annotate with creation time, origin template, etc. + # Add extraconfig options that can be queried by vmtools + config_spec = RbVmomi::VIM.VirtualMachineConfigSpec( + annotation: JSON.pretty_generate( + name: new_vmname, + created_by: provider_config['username'], + base_template: template_path, + creation_timestamp: Time.now.utc + ), + extraConfig: [ + { key: 'guestinfo.hostname', value: new_vmname } + ] + ) - template_vm_object = template_folder_object.find(template_name) - raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil? + # Choose a cluster/host to place the new VM on + target_host_object = find_least_used_host(target_cluster_name, connection) - # Annotate with creation time, origin template, etc. - # Add extraconfig options that can be queried by vmtools - config_spec = RbVmomi::VIM.VirtualMachineConfigSpec( - annotation: JSON.pretty_generate( + # Put the VM in the specified folder and resource pool + relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec( + datastore: find_datastore(target_datastore, connection), + host: target_host_object, + diskMoveType: :moveChildMostDiskBacking + ) + + # Create a clone spec + clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec( + location: relocate_spec, + config: config_spec, + powerOn: true, + template: false + ) + + # Create the new VM + new_vm_object = template_vm_object.CloneVM_Task( + folder: find_folder(target_folder_path, connection), name: new_vmname, - created_by: provider_config['username'], - base_template: template_path, - creation_timestamp: Time.now.utc - ), - extraConfig: [ - { key: 'guestinfo.hostname', value: new_vmname } - ] - ) + spec: clone_spec + ).wait_for_completion - # Choose a cluster/host to place the new VM on - target_host_object = find_least_used_host(target_cluster_name, connection) - - # Put the VM in the specified folder and resource pool - relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec( - datastore: find_datastore(target_datastore, connection), - host: target_host_object, - diskMoveType: :moveChildMostDiskBacking - ) - - # Create a clone spec - clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec( - location: relocate_spec, - config: config_spec, - powerOn: true, - template: false - ) - - # Create the new VM - new_vm_object = template_vm_object.CloneVM_Task( - folder: find_folder(target_folder_path, connection), - name: new_vmname, - spec: clone_spec - ).wait_for_completion - - generate_vm_hash(new_vm_object, template_path, pool_name) + vm_hash = generate_vm_hash(new_vm_object, template_path, pool_name) + end + vm_hash end def create_disk(pool_name, vm_name, disk_size) @@ -170,62 +205,62 @@ module Vmpooler datastore_name = pool['datastore'] raise("Pool #{pool_name} does not have a datastore defined for the provider #{name}") if datastore_name.nil? - connection = get_connection - - vm_object = find_vm(vm_name, connection) - raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? - - add_disk(vm_object, disk_size, datastore_name, connection) + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? + add_disk(vm_object, disk_size, datastore_name, connection) + end true end def create_snapshot(pool_name, vm_name, new_snapshot_name) - connection = get_connection + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? - vm_object = find_vm(vm_name, connection) - raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? - - old_snap = find_snapshot(vm_object, new_snapshot_name) - raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil? - - vm_object.CreateSnapshot_Task( - name: new_snapshot_name, - description: 'vmpooler', - memory: true, - quiesce: true - ).wait_for_completion + old_snap = find_snapshot(vm_object, new_snapshot_name) + raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil? + vm_object.CreateSnapshot_Task( + name: new_snapshot_name, + description: 'vmpooler', + memory: true, + quiesce: true + ).wait_for_completion + end true end def revert_snapshot(pool_name, vm_name, snapshot_name) - connection = get_connection + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? - vm_object = find_vm(vm_name, connection) - raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? - - snapshot_object = find_snapshot(vm_object, snapshot_name) - raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil? - - snapshot_object.RevertToSnapshot_Task.wait_for_completion + snapshot_object = find_snapshot(vm_object, snapshot_name) + raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil? + snapshot_object.RevertToSnapshot_Task.wait_for_completion + end true end def destroy_vm(_pool_name, vm_name) - connection = get_connection + @connection_pool.with_metrics do |pool_object| + connection = ensured_vsphere_connection(pool_object) + vm_object = find_vm(vm_name, connection) + # If a VM doesn't exist then it is effectively deleted + return true if vm_object.nil? - vm_object = find_vm(vm_name, connection) - # If a VM doesn't exist then it is effectively deleted - return true if vm_object.nil? - - # Poweroff the VM if it's running - vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn' - - # Kill it with fire - vm_object.Destroy_Task.wait_for_completion + # Poweroff the VM if it's running + vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn' + # Kill it with fire + vm_object.Destroy_Task.wait_for_completion + end true end @@ -239,12 +274,6 @@ module Vmpooler true end - def provider_config - # The vSphere configuration is currently in it's own root. This will - # eventually shift into the same location base expects it - global_config[:vsphere] - end - # VSphere Helper methods def get_target_cluster_from_config(pool_name) @@ -275,25 +304,27 @@ module Vmpooler DISK_TYPE = 'thin'.freeze DISK_MODE = 'persistent'.freeze - def get_connection - begin - @connection.serviceInstance.CurrentTime - rescue - @connection = connect_to_vsphere @credentials - end - - @connection + def ensured_vsphere_connection(connection_pool_object) + connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection]) + connection_pool_object[:connection] end - def connect_to_vsphere(credentials) - max_tries = @conf['max_tries'] || 3 - retry_factor = @conf['retry_factor'] || 10 + def vsphere_connection_ok?(connection) + _result = connection.serviceInstance.CurrentTime + return true + rescue + return false + end + + def connect_to_vsphere + max_tries = global_config[:config]['max_tries'] || 3 + retry_factor = global_config[:config]['retry_factor'] || 10 try = 1 begin - connection = RbVmomi::VIM.connect host: credentials['server'], - user: credentials['username'], - password: credentials['password'], - insecure: credentials['insecure'] || true + connection = RbVmomi::VIM.connect host: provider_config['server'], + user: provider_config['username'], + password: provider_config['password'], + insecure: provider_config['insecure'] || true metrics.increment('connect.open') return connection rescue => err @@ -679,10 +710,6 @@ module Vmpooler relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) vm.RelocateVM_Task(spec: relospec).wait_for_completion end - - def close - @connection.close - end end end end diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb deleted file mode 100644 index 3ad8843..0000000 --- a/lib/vmpooler/vsphere_helper.rb +++ /dev/null @@ -1,415 +0,0 @@ -require 'rubygems' unless defined?(Gem) - -module Vmpooler - class VsphereHelper - ADAPTER_TYPE = 'lsiLogic' - DISK_TYPE = 'thin' - DISK_MODE = 'persistent' - - def initialize(config, metrics) - @credentials = config[:vsphere] - @conf = config[:config] - @metrics = metrics - end - - def ensure_connected(connection, credentials) - connection.serviceInstance.CurrentTime - rescue - connect_to_vsphere @credentials - end - - def connect_to_vsphere(credentials) - max_tries = @conf['max_tries'] || 3 - retry_factor = @conf['retry_factor'] || 10 - try = 1 - begin - @connection = RbVmomi::VIM.connect host: credentials['server'], - user: credentials['username'], - password: credentials['password'], - insecure: credentials['insecure'] || true - @metrics.increment("connect.open") - rescue => err - try += 1 - @metrics.increment("connect.fail") - raise err if try == max_tries - sleep(try * retry_factor) - retry - end - end - - def add_disk(vm, size, datastore) - ensure_connected @connection, @credentials - - return false unless size.to_i > 0 - - vmdk_datastore = find_datastore(datastore) - vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" - - controller = find_disk_controller(vm) - - vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( - capacityKb: size.to_i * 1024 * 1024, - adapterType: ADAPTER_TYPE, - diskType: DISK_TYPE - ) - - vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( - datastore: vmdk_datastore, - diskMode: DISK_MODE, - fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" - ) - - device = RbVmomi::VIM::VirtualDisk( - backing: vmdk_backing, - capacityInKB: size.to_i * 1024 * 1024, - controllerKey: controller.key, - key: -1, - unitNumber: find_disk_unit_number(vm, controller) - ) - - device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( - device: device, - operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') - ) - - vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( - deviceChange: [device_config_spec] - ) - - @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( - datacenter: @connection.serviceInstance.find_datacenter, - name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", - spec: vmdk_spec - ).wait_for_completion - - vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion - - true - end - - def find_datastore(datastorename) - ensure_connected @connection, @credentials - - datacenter = @connection.serviceInstance.find_datacenter - datacenter.find_datastore(datastorename) - end - - def find_device(vm, deviceName) - ensure_connected @connection, @credentials - - vm.config.hardware.device.each do |device| - return device if device.deviceInfo.label == deviceName - end - - nil - end - - def find_disk_controller(vm) - ensure_connected @connection, @credentials - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |device| - if devices[device]['children'].length < 15 - return find_device(vm, devices[device]['device'].deviceInfo.label) - end - end - - nil - end - - def find_disk_devices(vm) - ensure_connected @connection, @credentials - - devices = {} - - vm.config.hardware.device.each do |device| - if device.is_a? RbVmomi::VIM::VirtualSCSIController - if devices[device.controllerKey].nil? - devices[device.key] = {} - devices[device.key]['children'] = [] - end - - devices[device.key]['device'] = device - end - - if device.is_a? RbVmomi::VIM::VirtualDisk - if devices[device.controllerKey].nil? - devices[device.controllerKey] = {} - devices[device.controllerKey]['children'] = [] - end - - devices[device.controllerKey]['children'].push(device) - end - end - - devices - end - - def find_disk_unit_number(vm, controller) - ensure_connected @connection, @credentials - - used_unit_numbers = [] - available_unit_numbers = [] - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |c| - next unless controller.key == devices[c]['device'].key - used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) - devices[c]['children'].each do |disk| - used_unit_numbers.push(disk.unitNumber) - end - end - - (0..15).each do |scsi_id| - if used_unit_numbers.grep(scsi_id).length <= 0 - available_unit_numbers.push(scsi_id) - end - end - - available_unit_numbers.sort[0] - end - - def find_folder(foldername) - ensure_connected @connection, @credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.vmFolder - folders = foldername.split('/') - folders.each do |folder| - if base.is_a? RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == folder } - else - raise(RuntimeError, "Unexpected object type encountered (#{base.class}) while finding folder") - end - end - - base - end - - # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference - # Params: - # +model+:: CPU arch version to match on - # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments - def get_host_utilization(host, model=nil, limit=90) - if model - return nil unless host_has_cpu_model? host, model - end - return nil if host.runtime.inMaintenanceMode - return nil unless host.overallStatus == 'green' - - cpu_utilization = cpu_utilization_for host - memory_utilization = memory_utilization_for host - - return nil if cpu_utilization > limit - return nil if memory_utilization > limit - - [ cpu_utilization + memory_utilization, host ] - end - - def host_has_cpu_model?(host, model) - get_host_cpu_arch_version(host) == model - end - - def get_host_cpu_arch_version(host) - cpu_model = host.hardware.cpuPkg[0].description - cpu_model_parts = cpu_model.split() - arch_version = cpu_model_parts[4] - arch_version - end - - def cpu_utilization_for(host) - cpu_usage = host.summary.quickStats.overallCpuUsage - cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores - (cpu_usage.to_f / cpu_size.to_f) * 100 - end - - def memory_utilization_for(host) - memory_usage = host.summary.quickStats.overallMemoryUsage - memory_size = host.summary.hardware.memorySize / 1024 / 1024 - (memory_usage.to_f / memory_size.to_f) * 100 - end - - def find_least_used_host(cluster) - ensure_connected @connection, @credentials - - cluster_object = find_cluster(cluster) - target_hosts = get_cluster_host_utilization(cluster_object) - least_used_host = target_hosts.sort[0][1] - least_used_host - end - - def find_cluster(cluster) - datacenter = @connection.serviceInstance.find_datacenter - datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } - end - - def get_cluster_host_utilization(cluster) - cluster_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host) - cluster_hosts << host_usage if host_usage - end - cluster_hosts - end - - def find_least_used_compatible_host(vm) - ensure_connected @connection, @credentials - - source_host = vm.summary.runtime.host - model = get_host_cpu_arch_version(source_host) - cluster = source_host.parent - target_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host, model) - target_hosts << host_usage if host_usage - end - target_host = target_hosts.sort[0][1] - [target_host, target_host.name] - end - - def find_pool(poolname) - ensure_connected @connection, @credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.hostFolder - pools = poolname.split('/') - pools.each do |pool| - case - when base.is_a?(RbVmomi::VIM::Folder) - base = base.childEntity.find { |f| f.name == pool } - when base.is_a?(RbVmomi::VIM::ClusterComputeResource) - base = base.resourcePool.resourcePool.find { |f| f.name == pool } - when base.is_a?(RbVmomi::VIM::ResourcePool) - base = base.resourcePool.find { |f| f.name == pool } - else - raise(RuntimeError, "Unexpected object type encountered (#{base.class}) while finding resource pool") - end - end - - base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) - base - end - - def find_snapshot(vm, snapshotname) - if vm.snapshot - get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) - end - end - - def find_vm(vmname) - ensure_connected @connection, @credentials - find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] - end - - def find_vm_light(vmname) - ensure_connected @connection, @credentials - - @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) - end - - def find_vm_heavy(vmname) - ensure_connected @connection, @credentials - - vmname = vmname.is_a?(Array) ? vmname : [vmname] - containerView = get_base_vm_container_from @connection - propertyCollector = @connection.propertyCollector - - objectSet = [{ - obj: containerView, - skip: true, - selectSet: [RbVmomi::VIM::TraversalSpec.new( - name: 'gettingTheVMs', - path: 'view', - skip: false, - type: 'ContainerView' - )] - }] - - propSet = [{ - pathSet: ['name'], - type: 'VirtualMachine' - }] - - results = propertyCollector.RetrievePropertiesEx( - specSet: [{ - objectSet: objectSet, - propSet: propSet - }], - options: { maxObjects: nil } - ) - - vms = {} - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - - while results.token - results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - end - - vms - end - - def find_vmdks(vmname, datastore) - ensure_connected @connection, @credentials - - disks = [] - - vmdk_datastore = find_datastore(datastore) - - vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' - vm_files.keys.each do |f| - vm_files[f]['layoutEx.file'].each do |l| - if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) - disks.push(l) - end - end - end - - disks - end - - def get_base_vm_container_from(connection) - ensure_connected @connection, @credentials - - viewManager = connection.serviceContent.viewManager - viewManager.CreateContainerView( - container: connection.serviceContent.rootFolder, - recursive: true, - type: ['VirtualMachine'] - ) - end - - def get_snapshot_list(tree, snapshotname) - snapshot = nil - - tree.each do |child| - if child.name == snapshotname - snapshot ||= child.snapshot - else - snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) - end - end - - snapshot - end - - def migrate_vm_host(vm, host) - relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) - vm.RelocateVM_Task(spec: relospec).wait_for_completion - end - - def close - @connection.close - end - end -end diff --git a/spec/helpers.rb b/spec/helpers.rb index 7fda0d9..d9b6914 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -6,21 +6,6 @@ def redis @redis end -# Mock an object which is result from Vmpooler::VsphereHelper.find_folder(foldername) -class MockFindFolder - attr_reader :childEntity - - def initialize(vmlist = []) - # Generate an array of hashes - @childEntity = vmlist.map do |vm| - vm_object = {} - vm_object['name'] = vm - - vm_object - end - end -end - # Mock an object which represents a Logger. This stops the proliferation # of allow(logger).to .... expectations in tests. class MockLogger diff --git a/spec/unit/generic_connection_pool_spec.rb b/spec/unit/generic_connection_pool_spec.rb new file mode 100644 index 0000000..ff57472 --- /dev/null +++ b/spec/unit/generic_connection_pool_spec.rb @@ -0,0 +1,132 @@ +require 'spec_helper' + +describe 'GenericConnectionPool' do + let(:metrics) { Vmpooler::DummyStatsd.new } + let(:metric_prefix) { 'prefix' } + let(:default_metric_prefix) { 'connectionpool' } + let(:connection_object) { double('connection') } + let(:pool_size) { 1 } + let(:pool_timeout) { 1 } + + subject { Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + metric_prefix: metric_prefix, + size: pool_size, + timeout: pool_timeout + ) { connection_object } + } + + describe "When consuming a pool object" do + let(:pool_size) { 1 } + let(:pool_timeout) { 1 } + let(:connection_object) {{ + connection: 'connection' + }} + + it 'should return a connection object when grabbing one from the pool' do + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + end + end + + it 'should return the same connection object when calling the pool multiple times' do + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + end + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + end + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + end + end + + it 'should preserve connection state across mulitple pool calls' do + new_connection = 'new_connection' + # Ensure the connection is not modified + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + expect(conn_pool_object[:connection]).to_not eq(new_connection) + # Change the connection + conn_pool_object[:connection] = new_connection + end + # Ensure the connection is modified + subject.with_metrics do |conn_pool_object| + expect(conn_pool_object).to be(connection_object) + expect(conn_pool_object[:connection]).to eq(new_connection) + end + end + end + + describe "#with_metrics" do + before(:each) do + expect(subject).not_to be_nil + end + + context 'When metrics are configured' do + it 'should emit a gauge metric when the connection is grabbed and released' do + expect(metrics).to receive(:gauge).with(/\.available/,Integer).exactly(2).times + + subject.with_metrics do |conn1| + # do nothing + end + end + + it 'should emit a timing metric when the connection is grabbed' do + expect(metrics).to receive(:timing).with(/\.waited/,Integer).exactly(1).times + + subject.with_metrics do |conn1| + # do nothing + end + end + + it 'should emit metrics with the specified prefix' do + expect(metrics).to receive(:gauge).with(/#{metric_prefix}\./,Integer).at_least(1).times + expect(metrics).to receive(:timing).with(/#{metric_prefix}\./,Integer).at_least(1).times + + subject.with_metrics do |conn1| + # do nothing + end + end + + context 'Metrix prefix is missing' do + let(:metric_prefix) { nil } + + it 'should emit metrics with default prefix' do + expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times + expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times + + subject.with_metrics do |conn1| + # do nothing + end + end + end + + context 'Metrix prefix is empty' do + let(:metric_prefix) { '' } + + it 'should emit metrics with default prefix' do + expect(metrics).to receive(:gauge).with(/#{default_metric_prefix}\./,Integer).at_least(1).times + expect(metrics).to receive(:timing).with(/#{default_metric_prefix}\./,Integer).at_least(1).times + + subject.with_metrics do |conn1| + # do nothing + end + end + end + end + + context 'When metrics are not configured' do + let(:metrics) { nil } + + it 'should not emit any metrics' do + # if any metrics are called it would result in a method error on Nil. + + subject.with_metrics do |conn1| + # do nothing + end + end + end + + end +end diff --git a/spec/unit/pool_manager_spec.rb b/spec/unit/pool_manager_spec.rb index 125f357..de7c11e 100644 --- a/spec/unit/pool_manager_spec.rb +++ b/spec/unit/pool_manager_spec.rb @@ -13,18 +13,40 @@ describe 'Pool Manager' do let(:logger) { MockLogger.new } let(:redis) { MockRedis.new } let(:metrics) { Vmpooler::DummyStatsd.new } - let(:config) { {} } let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } let(:host) { double('host') } let(:token) { 'token1234'} + let(:provider_options) { {} } + let(:provider) { Vmpooler::PoolManager::Provider::Base.new(config, logger, metrics, 'mock_provider', provider_options) } + + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 +EOT + ) + } + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } - describe '#check_pending_vm' do - let(:provider) { double('provider') } + describe '#config' do + before do + expect(subject).not_to be_nil + end + it 'should return the current configuration' do + expect(subject.config).to eq(config) + end + end + + describe '#check_pending_vm' do before do expect(subject).not_to be_nil end @@ -37,72 +59,14 @@ describe 'Pool Manager' do end end - describe '#open_socket' do - let(:TCPSocket) { double('tcpsocket') } - let(:socket) { double('tcpsocket') } - let(:hostname) { 'host' } - let(:domain) { 'domain.local'} - let(:default_socket) { 22 } - - before do - expect(subject).not_to be_nil - allow(socket).to receive(:close) - end - - it 'opens socket with defaults' do - expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket) - - expect(subject.open_socket(hostname)).to eq(nil) - end - - it 'yields the socket if a block is given' do - expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket) - - expect{ |socket| subject.open_socket(hostname,nil,nil,default_socket,&socket) }.to yield_control.exactly(1).times - end - - it 'closes the opened socket' do - expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_return(socket) - expect(socket).to receive(:close) - - expect(subject.open_socket(hostname)).to eq(nil) - end - - it 'opens a specific socket' do - expect(TCPSocket).to receive(:new).with(hostname,80).and_return(socket) - - expect(subject.open_socket(hostname,nil,nil,80)).to eq(nil) - end - - it 'uses a specific domain with the hostname' do - expect(TCPSocket).to receive(:new).with("#{hostname}.#{domain}",default_socket).and_return(socket) - - expect(subject.open_socket(hostname,domain)).to eq(nil) - end - - it 'raises error if host is not resolvable' do - expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'getaddrinfo: No such host is known') - - expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError) - end - - it 'raises error if socket is not listening' do - expect(TCPSocket).to receive(:new).with(hostname,default_socket).and_raise(SocketError,'No connection could be made because the target machine actively refused it') - - expect { subject.open_socket(hostname,nil,1) }.to raise_error(SocketError) - end - end - describe '#_check_pending_vm' do - let(:provider) { double('provider') } - before do expect(subject).not_to be_nil end context 'host does not exist or not in pool' do it 'calls fail_pending_vm' do - expect(provider).to receive(:find_vm).and_return(nil) + expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil) expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout, false) subject._check_pending_vm(vm, pool, timeout, provider) @@ -110,17 +74,19 @@ describe 'Pool Manager' do end context 'host is in pool' do + before do + expect(provider).to receive(:get_vm).with(pool,vm).and_return(host) + end + it 'calls move_pending_vm_to_ready if host is ready' do - expect(provider).to receive(:find_vm).and_return(host) - expect(subject).to receive(:open_socket).and_return(nil) + expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(true) expect(subject).to receive(:move_pending_vm_to_ready).with(vm, pool, host) subject._check_pending_vm(vm, pool, timeout, provider) end - it 'calls fail_pending_vm if an error is raised' do - expect(provider).to receive(:find_vm).and_return(host) - expect(subject).to receive(:open_socket).and_raise(SocketError,'getaddrinfo: No such host is known') + it 'calls fail_pending_vm if host is not ready' do + expect(provider).to receive(:vm_ready?).with(pool,vm).and_return(false) expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout) subject._check_pending_vm(vm, pool, timeout, provider) @@ -158,19 +124,19 @@ describe 'Pool Manager' do end it 'takes no action if VM is not cloning' do - expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true) end it 'takes no action if VM is within timeout' do redis.hset("vmpooler__vm__#{vm}", 'clone',Time.now.to_s) - expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true) end it 'moves VM to completed queue if VM has exceeded timeout and exists' do redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s) - expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true) end @@ -178,18 +144,18 @@ describe 'Pool Manager' do it 'logs message if VM has exceeded timeout and exists' do redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s) expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") - expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) end it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do redis.hset("vmpooler__vm__#{vm}", 'clone',Date.new(2001,1,1).to_s) expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool) - expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(true) end it 'swallows error if an error is raised' do redis.hset("vmpooler__vm__#{vm}", 'clone','iamnotparsable_asdate') - expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(nil) + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(false) end it 'logs message if an error is raised' do @@ -201,6 +167,8 @@ describe 'Pool Manager' do end describe '#move_pending_vm_to_ready' do + let(:host) { { 'hostname' => vm }} + before do expect(subject).not_to be_nil allow(Socket).to receive(:getaddrinfo) @@ -215,21 +183,13 @@ describe 'Pool Manager' do expect(logger).to receive(:log).exactly(0).times expect(Socket).to receive(:getaddrinfo).exactly(0).times - allow(host).to receive(:summary).and_return( double('summary') ) - allow(host).to receive_message_chain(:summary, :guest).and_return( double('guest') ) - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return ('different_name') + host['hostname'] = 'different_name' subject.move_pending_vm_to_ready(vm, pool, host) end end context 'when hostname matches VM name' do - before do - allow(host).to receive(:summary).and_return( double('summary') ) - allow(host).to receive_message_chain(:summary, :guest).and_return( double('guest') ) - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return (vm) - end - it 'should move the VM from pending to ready pool' do expect(redis.sismember("vmpooler__pending__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) @@ -239,7 +199,7 @@ describe 'Pool Manager' do end it 'should log a message' do - expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") subject.move_pending_vm_to_ready(vm, pool, host) end @@ -266,103 +226,115 @@ describe 'Pool Manager' do end describe '#check_ready_vm' do - let(:provider) { double('provider') } let(:ttl) { 0 } - let(:config) { - YAML.load(<<-EOT ---- -:config: - vm_checktime: 15 - -EOT - ) - } - - before(:each) do - expect(Thread).to receive(:new).and_yield - create_ready_vm(pool,vm) + before do + expect(subject).not_to be_nil end - it 'should raise an error if a TTL above zero is specified' do - expect { subject.check_ready_vm(vm,pool,5,provider) }.to raise_error(NameError) # This is an implementation bug + it 'calls _check_ready_vm' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, provider) + + subject.check_ready_vm(vm, pool, ttl, provider) + end + end + + describe '#_check_ready_vm' do + let(:ttl) { 0 } + let(:host) { {} } + + before(:each) do + create_ready_vm(pool,vm) + config[:config] = {} + config[:config]['vm_checktime'] = 15 + + # Create a VM which is powered on + host['hostname'] = vm + host['powerstate'] = 'PoweredOn' + allow(provider).to receive(:get_vm).with(pool,vm).and_return(host) end context 'a VM that does not need to be checked' do it 'should do nothing' do - redis.hset("vmpooler__vm__#{vm}", 'check',Time.now.to_s) - subject.check_ready_vm(vm, pool, ttl, provider) + check_stamp = (Time.now - 60).to_s + redis.hset("vmpooler__vm__#{vm}", 'check', check_stamp) + expect(provider).to receive(:get_vm).exactly(0).times + subject._check_ready_vm(vm, pool, ttl, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(check_stamp) end end context 'a VM that does not exist' do before do - allow(provider).to receive(:find_vm).and_return(nil) + expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil) end - it 'should set the current check timestamp' do - allow(subject).to receive(:open_socket) + it 'should not set the current check timestamp' do + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil + subject._check_ready_vm(vm, pool, ttl, provider) expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil - subject.check_ready_vm(vm, pool, ttl, provider) - expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil end it 'should log a message' do - expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - allow(subject).to receive(:open_socket) - subject.check_ready_vm(vm, pool, ttl, provider) + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory, removed from 'ready' queue") + subject._check_ready_vm(vm, pool, ttl, provider) end it 'should remove the VM from the ready queue' do - allow(subject).to receive(:open_socket) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) end end - context 'a VM that needs to be checked' do - before(:each) do - redis.hset("vmpooler__vm__#{vm}", 'check',Date.new(2001,1,1).to_s) + context 'a VM that has never been checked' do + let(:last_check_date) { Date.new(2001,1,1).to_s } - allow(host).to receive(:summary).and_return( double('summary') ) - allow(host).to receive_message_chain(:summary, :guest).and_return( double('guest') ) - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return (vm) - - allow(provider).to receive(:find_vm).and_return(host) + it 'should set the current check timestamp' do + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to be_nil + subject._check_ready_vm(vm, pool, ttl, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not be_nil + end + end + + context 'a VM that needs to be checked' do + let(:last_check_date) { Date.new(2001,1,1).to_s } + before(:each) do + redis.hset("vmpooler__vm__#{vm}", 'check',last_check_date) + end + + it 'should set the current check timestamp' do + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to eq(last_check_date) + subject._check_ready_vm(vm, pool, ttl, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'check')).to_not eq(last_check_date) end context 'and is ready' do before(:each) do - allow(host).to receive(:runtime).and_return( double('runtime') ) - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOn') - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return (vm) - allow(subject).to receive(:open_socket).with(vm).and_return(nil) + expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(true) end it 'should only set the next check interval' do - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end end - context 'is turned off, a name mismatch and not available via TCP' do + context 'is turned off' do before(:each) do - allow(host).to receive(:runtime).and_return( double('runtime') ) - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOff') - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return ('') - allow(subject).to receive(:open_socket).with(vm).and_raise(SocketError,'getaddrinfo: No such host is known') + host['powerstate'] = 'PoweredOff' end it 'should move the VM to the completed queue' do expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end it 'should move the VM to the completed queue in Redis' do expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true) end @@ -370,28 +342,25 @@ EOT it 'should log messages about being powered off' do expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end end - context 'is turned on, a name mismatch and not available via TCP' do + context 'is turned on, a name mismatch' do before(:each) do - allow(host).to receive(:runtime).and_return( double('runtime') ) - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOn') - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return ('') - allow(subject).to receive(:open_socket).with(vm).and_raise(SocketError,'getaddrinfo: No such host is known') + host['hostname'] = 'different_name' end it 'should move the VM to the completed queue' do expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end it 'should move the VM to the completed queue in Redis' do expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true) end @@ -399,28 +368,25 @@ EOT it 'should log messages about being misnamed' do expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end end - context 'is turned on, with correct name and not available via TCP' do + context 'is turned on, with correct name and is not ready' do before(:each) do - allow(host).to receive(:runtime).and_return( double('runtime') ) - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOn') - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return (vm) - allow(subject).to receive(:open_socket).with(vm).and_raise(SocketError,'getaddrinfo: No such host is known') + expect(provider).to receive(:vm_ready?).with(pool, vm).and_return(false) end it 'should move the VM to the completed queue' do expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end it 'should move the VM to the completed queue in Redis' do expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false) - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) expect(redis.sismember("vmpooler__ready__#{pool}", vm)).to be(false) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true) end @@ -428,7 +394,7 @@ EOT it 'should log messages about being unreachable' do expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") - subject.check_ready_vm(vm, pool, ttl, provider) + subject._check_ready_vm(vm, pool, ttl, provider) end end end @@ -451,7 +417,7 @@ EOT end describe '#_check_running_vm' do - let(:provider) { double('provider') } + let(:host) { {} } before do expect(subject).not_to be_nil @@ -459,24 +425,26 @@ EOT before(:each) do create_running_vm(pool,vm) + + # Create a VM which is powered on + host['hostname'] = vm + host['powerstate'] = 'PoweredOn' + allow(provider).to receive(:get_vm).with(pool,vm).and_return(host) end it 'does nothing with a missing VM' do - allow(provider).to receive(:find_vm).and_return(nil) + expect(provider).to receive(:get_vm).with(pool,vm).and_return(nil) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) subject._check_running_vm(vm, pool, timeout, provider) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) end context 'valid host' do - let(:vm_host) { double('vmhost') } - it 'should not move VM when not poweredOn' do # I'm not sure this test is useful. There is no codepath # in _check_running_vm that looks at Power State - allow(provider).to receive(:find_vm).and_return vm_host - allow(vm_host).to receive(:runtime).and_return true - allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff' + host['powerstate'] = 'PoweredOff' + expect(logger).not_to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) subject._check_running_vm(vm, pool, timeout, provider) @@ -484,14 +452,12 @@ EOT end it 'should not move VM if it has no checkout time' do - allow(provider).to receive(:find_vm).and_return vm_host expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) subject._check_running_vm(vm, pool, 0, provider) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) end it 'should not move VM if TTL is zero' do - allow(provider).to receive(:find_vm).and_return vm_host redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) subject._check_running_vm(vm, pool, 0, provider) @@ -499,7 +465,6 @@ EOT end it 'should move VM when past TTL' do - allow(provider).to receive(:find_vm).and_return vm_host redis.hset("vmpooler__active__#{pool}", vm,(Time.now - timeout*60*60).to_s) expect(redis.sismember("vmpooler__running__#{pool}", vm)).to be(true) expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(false) @@ -549,437 +514,218 @@ EOT end describe '#clone_vm' do - let(:provider) { double('provider') } - - let(:config) { - YAML.load(<<-EOT ---- -:config: - prefix: "prefix" -:vsphere: - username: "vcenter_user" -:pools: - - name: #{pool} -EOT - ) - } - let (:pool_object) { config[:pools][0] } + let (:pool_object) { { 'name' => pool } } before do expect(subject).not_to be_nil + expect(Thread).to receive(:new).and_yield end it 'calls _clone_vm' do - expect(Thread).to receive(:new).and_yield expect(subject).to receive(:_clone_vm).with(pool_object,provider) subject.clone_vm(pool_object,provider) end it 'logs a message if an error is raised' do - expect(Thread).to receive(:new).and_yield - expect(logger).to receive(:log) - expect(subject).to receive(:_clone_vm).with(pool_object,provider).and_raise('an_error') + allow(logger).to receive(:log) + expect(logger).to receive(:log).with('s',"[!] [#{pool_object['name']}] failed while cloning VM with an error: MockError") + expect(subject).to receive(:_clone_vm).with(pool_object,provider).and_raise('MockError') - expect{subject.clone_vm(pool_object,provider)}.to raise_error(/an_error/) + expect{subject.clone_vm(pool_object,provider)}.to raise_error(/MockError/) end end describe '#_clone_vm' do + let (:pool_object) { { 'name' => pool } } + before do expect(subject).not_to be_nil end - let (:folder) { 'vmfolder' } - let (:folder_object) { double('folder_object') } - let (:template_name) { pool } - let (:template) { "template/#{template_name}" } - let (:datastore) { 'datastore' } - let (:target) { 'clone_target' } - let(:config) { YAML.load(<<-EOT --- :config: prefix: "prefix" -:vsphere: - username: "vcenter_user" -:pools: - - name: #{pool} - template: '#{template}' - folder: '#{folder}' - datastore: '#{datastore}' - clone_target: '#{target}' EOT ) } - let (:provider) { double('provider') } - let (:template_folder_object) { double('template_folder_object') } - let (:template_vm_object) { double('template_vm_object') } - let (:clone_task) { double('clone_task') } - let (:pool_object) { config[:pools][0] } - - context 'no template specified' do + context 'with no errors during cloning' do before(:each) do - pool_object['template'] = nil + expect(metrics).to receive(:timing).with(/clone\./,/0/) + expect(provider).to receive(:create_vm).with(pool, String) + allow(logger).to receive(:log) end - it 'should raise an error' do - expect{subject._clone_vm(pool_object,provider)}.to raise_error(/Please provide a full path to the template/) + it 'should create a cloning VM' do + expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) + + subject._clone_vm(pool_object,provider) + + expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1) + # Get the new VM Name from the pending pool queue as it should be the only entry + vm_name = redis.smembers("vmpooler__pending__#{pool}")[0] + expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil + expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(pool) + expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm_name}")).to_not be_nil + expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil + end + + it 'should decrement the clone tasks counter' do + redis.incr('vmpooler__tasks__clone') + redis.incr('vmpooler__tasks__clone') + expect(redis.get('vmpooler__tasks__clone')).to eq('2') + subject._clone_vm(pool_object,provider) + expect(redis.get('vmpooler__tasks__clone')).to eq('1') + end + + it 'should log a message that is being cloned from a template' do + expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] Starting to clone '(.+)'/) + + subject._clone_vm(pool_object,provider) + end + + it 'should log a message that it completed being cloned' do + expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned in [0-9.]+ seconds/) + + subject._clone_vm(pool_object,provider) end end - context 'a template with no forward slash in the string' do + context 'with an error during cloning' do before(:each) do - pool_object['template'] = template_name + expect(provider).to receive(:create_vm).with(pool, String).and_raise('MockError') + allow(logger).to receive(:log) end - it 'should raise an error' do - expect{subject._clone_vm(pool_object,provider)}.to raise_error(/Please provide a full path to the template/) - end - end + it 'should not create a cloning VM' do + expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - # Note - It is impossible to get into the following code branch - # ... - # if vm['template'].length == 0 - # fail "Unable to find template '#{vm['template']}'!" - # end - # ... + expect{subject._clone_vm(pool_object,provider)}.to raise_error(/MockError/) - context "Template name does not match pool name (Implementation Bug)" do - let (:template_name) { 'template_vm' } - - # The implementaion of _clone_vm incorrectly uses the VM Template name instead of the pool name. The VM Template represents the - # name of the VM to clone in vSphere whereas pool is the name of the pool in Pooler. The tests below document the behaviour of - # _clone_vm if the Template and Pool name differ. It is expected that these test will fail once this bug is removed. - - context 'a valid template' do - before(:each) do - expect(template_folder_object).to receive(:find).with(template_name).and_return(template_vm_object) - expect(provider).to receive(:find_folder).with('template').and_return(template_folder_object) - end - - context 'with no errors during cloning' do - before(:each) do - expect(provider).to receive(:find_least_used_host).with(target).and_return('least_used_host') - expect(provider).to receive(:find_datastore).with(datastore).and_return('datastore') - expect(provider).to receive(:find_folder).with('vmfolder').and_return(folder_object) - expect(template_vm_object).to receive(:CloneVM_Task).and_return(clone_task) - expect(clone_task).to receive(:wait_for_completion) - expect(metrics).to receive(:timing).with(/clone\./,/0/) - end - - it 'should create a cloning VM' do - expect(logger).to receive(:log).at_least(:once) - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - - subject._clone_vm(pool_object,provider) - - expect(redis.scard("vmpooler__pending__#{template_name}")).to eq(1) - # Get the new VM Name from the pending pool queue as it should be the only entry - vm_name = redis.smembers("vmpooler__pending__#{template_name}")[0] - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(template_name) - expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{template_name}:#{vm_name}")).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil - end - - it 'should log a message that is being cloned from a template' do - expect(logger).to receive(:log).with('d',/\[ \] \[#{template_name}\] '(.+)' is being cloned from '#{template_name}'/) - allow(logger).to receive(:log) - - subject._clone_vm(pool_object,provider) - end - - it 'should log a message that it completed being cloned' do - expect(logger).to receive(:log).with('s',/\[\+\] \[#{template_name}\] '(.+)' cloned from '#{template_name}' in [0-9.]+ seconds/) - allow(logger).to receive(:log) - - subject._clone_vm(pool_object,provider) - end - end - - # An error can be cause by the following configuration errors: - # - Missing or invalid datastore - # - Missing or invalid clone target - # also any runtime errors during the cloning process - # https://www.vmware.com/support/developer/converter-sdk/conv50_apireference/vim.VirtualMachine.html#clone - context 'with an error during cloning' do - before(:each) do - expect(provider).to receive(:find_least_used_host).with(target).and_return('least_used_host') - expect(provider).to receive(:find_datastore).with(datastore).and_return(nil) - expect(provider).to receive(:find_folder).with('vmfolder').and_return(folder_object) - expect(template_vm_object).to receive(:CloneVM_Task).and_return(clone_task) - expect(clone_task).to receive(:wait_for_completion).and_raise(RuntimeError,'SomeError') - expect(metrics).to receive(:timing).with(/clone\./,/0/).exactly(0).times - - end - - it 'should raise an error within the Thread' do - expect(logger).to receive(:log).at_least(:once) - expect{subject._clone_vm(pool_object,provider)}.to raise_error(/SomeError/) - end - - it 'should log a message that is being cloned from a template' do - expect(logger).to receive(:log).with('d',/\[ \] \[#{template_name}\] '(.+)' is being cloned from '#{template_name}'/) - allow(logger).to receive(:log) - - # Swallow the error - begin - subject._clone_vm(pool_object,provider) - rescue - end - end - - it 'should log messages that the clone failed' do - expect(logger).to receive(:log).with('s', /\[!\] \[#{template_name}\] '(.+)' clone failed with an error: SomeError/) - allow(logger).to receive(:log) - - # Swallow the error - begin - subject._clone_vm(pool_object,provider) - rescue - end - end - end - end - end - - context 'a valid template' do - before(:each) do - expect(template_folder_object).to receive(:find).with(template_name).and_return(template_vm_object) - expect(provider).to receive(:find_folder).with('template').and_return(template_folder_object) + expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) + # Get the new VM Name from the pending pool queue as it should be the only entry + vm_name = redis.smembers("vmpooler__pending__#{pool}")[0] + expect(vm_name).to be_nil end - context 'with no errors during cloning' do - before(:each) do - expect(provider).to receive(:find_least_used_host).with(target).and_return('least_used_host') - expect(provider).to receive(:find_datastore).with(datastore).and_return('datastore') - expect(provider).to receive(:find_folder).with('vmfolder').and_return(folder_object) - expect(template_vm_object).to receive(:CloneVM_Task).and_return(clone_task) - expect(clone_task).to receive(:wait_for_completion) - expect(metrics).to receive(:timing).with(/clone\./,/0/) - end - - it 'should create a cloning VM' do - expect(logger).to receive(:log).at_least(:once) - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(0) - - subject._clone_vm(pool_object,provider) - - expect(redis.scard("vmpooler__pending__#{pool}")).to eq(1) - # Get the new VM Name from the pending pool queue as it should be the only entry - vm_name = redis.smembers("vmpooler__pending__#{pool}")[0] - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone')).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'template')).to eq(template_name) - expect(redis.hget("vmpooler__clone__#{Date.today.to_s}", "#{pool}:#{vm_name}")).to_not be_nil - expect(redis.hget("vmpooler__vm__#{vm_name}", 'clone_time')).to_not be_nil - end - - it 'should decrement the clone tasks counter' do - redis.incr('vmpooler__tasks__clone') - redis.incr('vmpooler__tasks__clone') - expect(redis.get('vmpooler__tasks__clone')).to eq('2') - subject._clone_vm(pool_object,provider) - expect(redis.get('vmpooler__tasks__clone')).to eq('1') - end - - it 'should log a message that is being cloned from a template' do - expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] '(.+)' is being cloned from '#{template_name}'/) - allow(logger).to receive(:log) - - subject._clone_vm(pool_object,provider) - end - - it 'should log a message that it completed being cloned' do - expect(logger).to receive(:log).with('s',/\[\+\] \[#{pool}\] '(.+)' cloned from '#{template_name}' in [0-9.]+ seconds/) - allow(logger).to receive(:log) - - subject._clone_vm(pool_object,provider) - end + it 'should decrement the clone tasks counter' do + redis.incr('vmpooler__tasks__clone') + redis.incr('vmpooler__tasks__clone') + expect(redis.get('vmpooler__tasks__clone')).to eq('2') + expect{subject._clone_vm(pool_object,provider)}.to raise_error(/MockError/) + expect(redis.get('vmpooler__tasks__clone')).to eq('1') end - # An error can be cause by the following configuration errors: - # - Missing or invalid datastore - # - Missing or invalid clone target - # also any runtime errors during the cloning process - # https://www.vmware.com/support/developer/converter-sdk/conv50_apireference/vim.VirtualMachine.html#clone - context 'with an error during cloning' do - before(:each) do - expect(provider).to receive(:find_least_used_host).with(target).and_return('least_used_host') - expect(provider).to receive(:find_datastore).with(datastore).and_return(nil) - expect(provider).to receive(:find_folder).with('vmfolder').and_return(folder_object) - expect(template_vm_object).to receive(:CloneVM_Task).and_return(clone_task) - expect(clone_task).to receive(:wait_for_completion).and_raise(RuntimeError,'SomeError') - expect(metrics).to receive(:timing).with(/clone\./,/0/).exactly(0).times + it 'should log messages that the clone failed' do + expect(logger).to receive(:log).with('s', /\[!\] \[#{pool}\] '(.+)' clone failed with an error: MockError/) - end - - it 'should raise an error within the Thread' do - expect(logger).to receive(:log).at_least(:once) - expect{subject._clone_vm(pool_object,provider)}.to raise_error(/SomeError/) - end - - it 'should log a message that is being cloned from a template' do - expect(logger).to receive(:log).with('d',/\[ \] \[#{pool}\] '(.+)' is being cloned from '#{template_name}'/) - allow(logger).to receive(:log) - - # Swallow the error - begin - subject._clone_vm(pool_object,provider) - rescue - end - end - - it 'should log messages that the clone failed' do - expect(logger).to receive(:log).with('s', /\[!\] \[#{pool}\] '(.+)' clone failed with an error: SomeError/) - allow(logger).to receive(:log) - - # Swallow the error - begin - subject._clone_vm(pool_object,provider) - rescue - end - end + expect{subject._clone_vm(pool_object,provider)}.to raise_error(/MockError/) end end end - describe "#destroy_vm" do - let (:provider) { double('provider') } - - let(:config) { - YAML.load(<<-EOT ---- -:redis: - data_ttl: 168 -EOT - ) - } - + describe '#destroy_vm' do before do expect(subject).not_to be_nil + expect(Thread).to receive(:new).and_yield end + it 'calls _destroy_vm' do + expect(subject).to receive(:_destroy_vm).with(vm,pool,provider) + + subject.destroy_vm(vm,pool,provider) + end + + it 'logs a message if an error is raised' do + allow(logger).to receive(:log) + expect(logger).to receive(:log).with('d',"[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: MockError") + expect(subject).to receive(:_destroy_vm).with(vm,pool,provider).and_raise('MockError') + + expect{subject.destroy_vm(vm,pool,provider)}.to raise_error(/MockError/) + end + end + + describe "#_destroy_vm" do before(:each) do - expect(Thread).to receive(:new).and_yield + expect(subject).not_to be_nil create_completed_vm(vm,pool,true) + + allow(provider).to receive(:destroy_vm).with(pool,vm).and_return(true) + + # Set redis configuration + config[:redis] = {} + config[:redis]['data_ttl'] = 168 end context 'when redis data_ttl is not specified in the configuration' do - let(:config) { - YAML.load(<<-EOT ---- -:redis: - "key": "value" -EOT - ) - } - before(:each) do - expect(provider).to receive(:find_vm).and_return(nil) + config[:redis]['data_ttl'] = nil end it 'should call redis expire with 0' do expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to_not be_nil - subject.destroy_vm(vm,pool,provider) + subject._destroy_vm(vm,pool,provider) expect(redis.hget("vmpooler__vm__#{vm}", 'checkout')).to be_nil end end context 'when there is no redis section in the configuration' do - let(:config) {} + before(:each) do + config[:redis] = nil + end it 'should raise an error' do - expect{ subject.destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError) + expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError) end end context 'when a VM does not exist' do before(:each) do - expect(provider).to receive(:find_vm).and_return(nil) + # As per base_spec, destroy_vm will return true if the VM does not exist + expect(provider).to receive(:destroy_vm).with(pool,vm).and_return(true) end - it 'should not call any provider methods' do - subject.destroy_vm(vm,pool,provider) + it 'should not raise an error' do + subject._destroy_vm(vm,pool,provider) end end - context 'when a VM exists' do - let (:destroy_task) { double('destroy_task') } - let (:poweroff_task) { double('poweroff_task') } + context 'when the VM is destroyed without error' do + it 'should log a message the VM was destroyed' do + expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/) + allow(logger).to receive(:log) + subject._destroy_vm(vm,pool,provider) + end + + it 'should emit a timing metric' do + expect(metrics).to receive(:timing).with("destroy.#{pool}", String) + + subject._destroy_vm(vm,pool,provider) + end + end + + context 'when the VM destruction raises an eror' do before(:each) do - expect(provider).to receive(:find_vm).and_return(host) - allow(host).to receive(:runtime).and_return(true) + # As per base_spec, destroy_vm will return true if the VM does not exist + expect(provider).to receive(:destroy_vm).with(pool,vm).and_raise('MockError') end - context 'and an error occurs during destroy' do - before(:each) do - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOff') - expect(host).to receive(:Destroy_Task).and_return(destroy_task) - expect(destroy_task).to receive(:wait_for_completion).and_raise(RuntimeError,'DestroyFailure') - expect(metrics).to receive(:timing).exactly(0).times - end + it 'should not log a message the VM was destroyed' do + expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/).exactly(0).times + allow(logger).to receive(:log) - it 'should raise an error in the thread' do - expect { subject.destroy_vm(vm,pool,provider) }.to raise_error(/DestroyFailure/) - end + expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/) end - context 'and an error occurs during power off' do - before(:each) do - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOn') - expect(host).to receive(:PowerOffVM_Task).and_return(poweroff_task) - expect(poweroff_task).to receive(:wait_for_completion).and_raise(RuntimeError,'PowerOffFailure') - expect(logger).to receive(:log).with('d', "[ ] [#{pool}] '#{vm}' is being shut down") - expect(metrics).to receive(:timing).exactly(0).times - end + it 'should not emit a timing metric' do + expect(metrics).to receive(:timing).with("destroy.#{pool}", String).exactly(0).times - it 'should raise an error in the thread' do - expect { subject.destroy_vm(vm,pool,provider) }.to raise_error(/PowerOffFailure/) - end - end - - context 'and is powered off' do - before(:each) do - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOff') - expect(host).to receive(:Destroy_Task).and_return(destroy_task) - expect(destroy_task).to receive(:wait_for_completion) - expect(metrics).to receive(:timing).with("destroy.#{pool}", /0/) - end - - it 'should log a message the VM was destroyed' do - expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/) - subject.destroy_vm(vm,pool,provider) - end - end - - context 'and is powered on' do - before(:each) do - allow(host).to receive_message_chain(:runtime, :powerState).and_return('poweredOn') - expect(host).to receive(:Destroy_Task).and_return(destroy_task) - expect(host).to receive(:PowerOffVM_Task).and_return(poweroff_task) - expect(poweroff_task).to receive(:wait_for_completion) - expect(destroy_task).to receive(:wait_for_completion) - expect(metrics).to receive(:timing).with("destroy.#{pool}", /0/) - end - - it 'should log a message the VM is being shutdown' do - expect(logger).to receive(:log).with('d', "[ ] [#{pool}] '#{vm}' is being shut down") - allow(logger).to receive(:log) - - subject.destroy_vm(vm,pool,provider) - end - - it 'should log a message the VM was destroyed' do - expect(logger).to receive(:log).with('s', /\[-\] \[#{pool}\] '#{vm}' destroyed in [0-9.]+ seconds/) - allow(logger).to receive(:log) - - subject.destroy_vm(vm,pool,provider) - end + expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/) end end end @@ -994,109 +740,99 @@ EOT it 'calls _create_vm_disk' do expect(Thread).to receive(:new).and_yield - expect(subject).to receive(:_create_vm_disk).with(vm, disk_size, provider) + expect(subject).to receive(:_create_vm_disk).with(pool, vm, disk_size, provider) - subject.create_vm_disk(vm, disk_size, provider) + subject.create_vm_disk(pool, vm, disk_size, provider) end end describe "#_create_vm_disk" do - let(:provider) { double('provider') } let(:disk_size) { '15' } - let(:datastore) { 'datastore0'} - let(:config) { - YAML.load(<<-EOT ---- -:pools: - - name: #{pool} - datastore: '#{datastore}' -EOT - ) - } - - before do - expect(subject).not_to be_nil - end before(:each) do - allow(provider).to receive(:find_vm).with(vm).and_return(host) + expect(subject).not_to be_nil + allow(logger).to receive(:log) + create_running_vm(pool,vm,token) end - it 'should not do anything if the VM does not exist' do - expect(provider).to receive(:find_vm).with(vm).and_return(nil) - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_disk(vm, disk_size, provider) + context 'Given a VM that does not exist' do + before(:each) do + # As per base_spec, create_disk will raise if the VM does not exist + expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_raise("VM #{vm} does not exist") + end + + it 'should not update redis if the VM does not exist' do + expect(redis).to receive(:hset).exactly(0).times + expect{ subject._create_vm_disk(pool, vm, disk_size, provider) }.to raise_error(RuntimeError) + end end - it 'should not do anything if the disk size is nil' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_disk(vm, nil, provider) + context 'Given an invalid disk size' do + [{ :description => 'is nil', :value => nil }, + { :description => 'is an empty string', :value => '' }, + { :description => 'is less than 1', :value => '0' }, + { :description => 'cannot be converted to an integer', :value => 'abc123' }, + ].each do |testcase| + it "should not attempt the create the disk if the disk size #{testcase[:description]}" do + expect(provider).to receive(:create_disk).exactly(0).times + expect{ subject._create_vm_disk(pool, vm, testcase[:value], provider) }.to raise_error(/Invalid disk size/) + end + end + + it 'should raise an error if the disk size is a Fixnum' do + expect(redis).to receive(:hset).exactly(0).times + expect{ subject._create_vm_disk(pool, vm, 10, provider) }.to raise_error(NoMethodError,/empty?/) + end end - it 'should not do anything if the disk size is empty string' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_disk(vm, '', provider) + context 'Given a successful disk creation' do + before(:each) do + expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_return(true) + end + + it 'should log a message' do + expect(logger).to receive(:log).with('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") + expect(logger).to receive(:log).with('s', /\[\+\] \[disk_manager\] '#{vm}' attached #{disk_size}gb disk in 0.[\d]+ seconds/) + + subject._create_vm_disk(pool, vm, disk_size, provider) + end + + it 'should update redis information when attaching the first disk' do + subject._create_vm_disk(pool, vm, disk_size, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("+#{disk_size}gb") + end + + it 'should update redis information when attaching the additional disks' do + initial_disks = '+10gb:+20gb' + redis.hset("vmpooler__vm__#{vm}", 'disk', initial_disks) + + subject._create_vm_disk(pool, vm, disk_size, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("#{initial_disks}:+#{disk_size}gb") + end end - it 'should not do anything if the disk size is less than 1' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_disk(vm, '0', provider) - end + context 'Given a failed disk creation' do + before(:each) do + expect(provider).to receive(:create_disk).with(pool,vm,disk_size.to_i).and_return(false) + end - it 'should not do anything if the disk size cannot be converted to an integer' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_disk(vm, 'abc123', provider) - end + it 'should not update redis information' do + expect(redis).to receive(:hset).exactly(0).times - it 'should raise an error if the disk size is a Fixnum' do - expect(logger).to receive(:log).exactly(0).times - expect{ subject._create_vm_disk(vm, 10, provider) }.to raise_error(NoMethodError,/empty?/) - end + subject._create_vm_disk(pool, vm, disk_size, provider) + expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to be_nil + end - it 'should not do anything if the datastore for pool is nil' do - expect(logger).to receive(:log).with('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") - expect(logger).to receive(:log).with('s', "[+] [disk_manager] '#{vm}' failed to attach disk") - config[:pools][0]['datastore'] = nil + it 'should log a message' do + expect(logger).to receive(:log).with('s', "[+] [disk_manager] '#{vm}' failed to attach disk") - subject._create_vm_disk(vm, disk_size, provider) - end - - it 'should not do anything if the datastore for pool is empty' do - expect(logger).to receive(:log).with('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") - expect(logger).to receive(:log).with('s', "[+] [disk_manager] '#{vm}' failed to attach disk") - config[:pools][0]['datastore'] = '' - - subject._create_vm_disk(vm, disk_size, provider) - end - - it 'should attach the disk' do - expect(logger).to receive(:log).with('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") - expect(logger).to receive(:log).with('s', /\[\+\] \[disk_manager\] '#{vm}' attached #{disk_size}gb disk in 0.[\d]+ seconds/) - expect(provider).to receive(:add_disk).with(host,disk_size,datastore) - - subject._create_vm_disk(vm, disk_size, provider) - end - - it 'should update redis information when attaching the first disk' do - expect(provider).to receive(:add_disk).with(host,disk_size,datastore) - - subject._create_vm_disk(vm, disk_size, provider) - expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("+#{disk_size}gb") - end - - it 'should update redis information when attaching the additional disks' do - expect(provider).to receive(:add_disk).with(host,disk_size,datastore) - initial_disks = '+10gb:+20gb' - redis.hset("vmpooler__vm__#{vm}", 'disk', initial_disks) - - subject._create_vm_disk(vm, disk_size, provider) - expect(redis.hget("vmpooler__vm__#{vm}", 'disk')).to eq("#{initial_disks}:+#{disk_size}gb") + subject._create_vm_disk(pool, vm, disk_size, provider) + end end end describe '#create_vm_snapshot' do - let(:provider) { double('provider') } let(:snapshot_name) { 'snapshot' } before do @@ -1105,110 +841,277 @@ EOT it 'calls _create_vm_snapshot' do expect(Thread).to receive(:new).and_yield - expect(subject).to receive(:_create_vm_snapshot).with(vm, snapshot_name, provider) + expect(subject).to receive(:_create_vm_snapshot).with(pool, vm, snapshot_name, provider) - subject.create_vm_snapshot(vm, snapshot_name, provider) + subject.create_vm_snapshot(pool, vm, snapshot_name, provider) end end describe '#_create_vm_snapshot' do - let(:provider) { double('provider') } let(:snapshot_name) { 'snapshot1' } - let(:snapshot_task) { double('snapshot_task') } before do expect(subject).not_to be_nil end before(:each) do - allow(provider).to receive(:find_vm).with(vm).and_return(host) - allow(snapshot_task).to receive(:wait_for_completion).and_return(nil) - allow(host).to receive(:CreateSnapshot_Task).with({:name=>snapshot_name, :description=>"vmpooler", :memory=>true, :quiesce=>true}).and_return(snapshot_task) create_running_vm(pool,vm,token) end - it 'should not do anything if the VM does not exist' do - expect(provider).to receive(:find_vm).with(vm).and_return(nil) - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_snapshot(vm, snapshot_name, provider) + context 'Given a Pool that does not exist' do + let(:missing_pool) { 'missing_pool' } + + before(:each) do + expect(provider).to receive(:create_snapshot).with(missing_pool, vm, snapshot_name).and_raise("Pool #{missing_pool} not found") + end + + it 'should not update redis' do + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + expect{ subject._create_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found") + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + end end - it 'should not do anything if the snapshot name is nil' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_snapshot(vm, nil, provider) + context 'Given a VM that does not exist' do + let(:missing_vm) { 'missing_vm' } + before(:each) do + expect(provider).to receive(:create_snapshot).with(pool, missing_vm, snapshot_name).and_raise("VM #{missing_vm} not found") + end + + it 'should not update redis' do + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + expect{ subject._create_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found") + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + end end - it 'should not do anything if the snapshot name is empty string' do - expect(logger).to receive(:log).exactly(0).times - subject._create_vm_snapshot(vm, '', provider) + context 'Given a snapshot creation that succeeds' do + before(:each) do + expect(provider).to receive(:create_snapshot).with(pool, vm, snapshot_name).and_return(true) + end + + it 'should log messages' do + expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to snapshot #{vm} in pool #{pool}") + expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' snapshot created in 0.[\d]+ seconds/) + + subject._create_vm_snapshot(pool, vm, snapshot_name, provider) + end + + it 'should add snapshot redis information' do + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + subject._create_vm_snapshot(pool, vm, snapshot_name, provider) + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to_not be_nil + end end - it 'should invoke provider to snapshot the VM' do - expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] '#{vm}' is being snapshotted") - expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' snapshot created in 0.[\d]+ seconds/) - subject._create_vm_snapshot(vm, snapshot_name, provider) - end + context 'Given a snapshot creation that fails' do + before(:each) do + expect(provider).to receive(:create_snapshot).with(pool, vm, snapshot_name).and_return(false) + end - it 'should add snapshot redis information' do - expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil - subject._create_vm_snapshot(vm, snapshot_name, provider) - expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to_not be_nil + it 'should log messages' do + expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to snapshot #{vm} in pool #{pool}") + expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to snapshot '#{vm}'") + + subject._create_vm_snapshot(pool, vm, snapshot_name, provider) + end + + it 'should not update redis' do + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + subject._create_vm_snapshot(pool, vm, snapshot_name, provider) + expect(redis.hget("vmpooler__vm__#{vm}", "snapshot:#{snapshot_name}")).to be_nil + end end end describe '#revert_vm_snapshot' do - let(:provider) { double('provider') } let(:snapshot_name) { 'snapshot' } before do expect(subject).not_to be_nil end - it 'calls _create_vm_snapshot' do + it 'calls _revert_vm_snapshot' do expect(Thread).to receive(:new).and_yield - expect(subject).to receive(:_revert_vm_snapshot).with(vm, snapshot_name, provider) + expect(subject).to receive(:_revert_vm_snapshot).with(pool, vm, snapshot_name, provider) - subject.revert_vm_snapshot(vm, snapshot_name, provider) + subject.revert_vm_snapshot(pool, vm, snapshot_name, provider) end end describe '#_revert_vm_snapshot' do - let(:provider) { double('provider') } let(:snapshot_name) { 'snapshot1' } - let(:snapshot_object) { double('snapshot_object') } before do expect(subject).not_to be_nil end - before(:each) do - allow(provider).to receive(:find_vm).with(vm).and_return(host) - allow(snapshot_object).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion) - allow(provider).to receive(:find_snapshot).with(host,snapshot_name).and_return(snapshot_object) + context 'Given a Pool that does not exist' do + let(:missing_pool) { 'missing_pool' } + + before(:each) do + expect(provider).to receive(:revert_snapshot).with(missing_pool, vm, snapshot_name).and_raise("Pool #{missing_pool} not found") + end + + it 'should not log a result message' do + expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/).exactly(0).times + expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{vm}' in pool #{missing_pool} to snapshot '#{snapshot_name}'").exactly(0).times + + expect{ subject._revert_vm_snapshot(missing_pool, vm, snapshot_name, provider) }.to raise_error("Pool #{missing_pool} not found") + end end - it 'should not do anything if the VM does not exist' do - expect(provider).to receive(:find_vm).with(vm).and_return(nil) - expect(logger).to receive(:log).exactly(0).times - subject._revert_vm_snapshot(vm, snapshot_name, provider) + context 'Given a VM that does not exist' do + let(:missing_vm) { 'missing_vm' } + before(:each) do + expect(provider).to receive(:revert_snapshot).with(pool, missing_vm, snapshot_name).and_raise("VM #{missing_vm} not found") + end + + it 'should not log a result message' do + expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{missing_vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/).exactly(0).times + expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{missing_vm}' in pool #{pool} to snapshot '#{snapshot_name}'").exactly(0).times + + expect{ subject._revert_vm_snapshot(pool, missing_vm, snapshot_name, provider) }.to raise_error("VM #{missing_vm} not found") + end end - it 'should not do anything if the snapshot name is nil' do - expect(logger).to receive(:log).exactly(0).times - expect(provider).to receive(:find_snapshot).with(host,nil).and_return nil - subject._revert_vm_snapshot(vm, nil, provider) + context 'Given a snapshot revert that succeeds' do + before(:each) do + expect(provider).to receive(:revert_snapshot).with(pool, vm, snapshot_name).and_return(true) + end + + it 'should log success messages' do + expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'") + expect(logger).to receive(:log).with('s', /\[\+\] \[snapshot_manager\] '#{vm}' reverted to snapshot '#{snapshot_name}' in 0.[\d]+ seconds/) + + subject._revert_vm_snapshot(pool, vm, snapshot_name, provider) + end + + it 'should return true' do + expect(subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)).to be true + end end - it 'should not do anything if the snapshot name is empty string' do - expect(logger).to receive(:log).exactly(0).times - expect(provider).to receive(:find_snapshot).with(host,'').and_return nil - subject._revert_vm_snapshot(vm, '', provider) + context 'Given a snapshot creation that fails' do + before(:each) do + expect(provider).to receive(:revert_snapshot).with(pool, vm, snapshot_name).and_return(false) + end + + it 'should log failure messages' do + expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] 'Attempting to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'") + expect(logger).to receive(:log).with('s', "[+] [snapshot_manager] Failed to revert #{vm}' in pool #{pool} to snapshot '#{snapshot_name}'") + + subject._revert_vm_snapshot(pool, vm, snapshot_name, provider) + end + + it 'should return false' do + expect(subject._revert_vm_snapshot(pool, vm, snapshot_name, provider)).to be false + end + end + end + + describe '#get_pool_name_for_vm' do + context 'Given a valid VM' do + before(:each) do + create_running_vm(pool, vm, token) + end + + it 'should return the pool name' do + expect(subject.get_pool_name_for_vm(vm)).to eq(pool) + end end - it 'should invoke provider to revert the VM to the snapshot' do - expect(logger).to receive(:log).with('s', "[ ] [snapshot_manager] '#{vm}' is being reverted to snapshot '#{snapshot_name}'") - expect(logger).to receive(:log).with('s', /\[\<\] \[snapshot_manager\] '#{vm}' reverted to snapshot in 0\.[\d]+ seconds/) - subject._revert_vm_snapshot(vm, snapshot_name, provider) + context 'Given an invalid VM' do + it 'should return nil' do + expect(subject.get_pool_name_for_vm('does_not_exist')).to be_nil + end + end + end + + describe '#get_provider_for_pool' do + let(:provider_name) { 'mock_provider' } + + before do + expect(subject).not_to be_nil + # Inject mock provider into global variable - Note this is a code smell + $providers = { provider_name => provider } + end + + after(:each) do + # Reset the global variable - Note this is a code smell + $providers = nil + end + + context 'Given a pool name which does not exist' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 +EOT + )} + + it 'should return nil' do + expect(subject.get_provider_for_pool('pool_does_not_exist')).to be_nil + end + end + + context 'Given a pool which does not have a provider' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 +EOT + )} + + it 'should return nil' do + expect(subject.get_provider_for_pool(pool)).to be_nil + end + end + + context 'Given a pool which uses an invalid provider' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 + provider: 'does_not_exist' +EOT + )} + + it 'should return nil' do + expect(subject.get_provider_for_pool(pool)).to be_nil + end + end + + context 'Given a pool which uses a valid provider' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :mock: +:pools: + - name: '#{pool}' + size: 1 + provider: #{provider_name} +EOT + )} + + it 'should return a provider object' do + result = subject.get_provider_for_pool(pool) + expect(result).to_not be_nil + expect(result.name).to eq(provider_name) + end end end @@ -1236,7 +1139,7 @@ EOT end it 'should call _check_disk_queue' do - expect(subject).to receive(:_check_disk_queue).with(Vmpooler::VsphereHelper) + expect(subject).to receive(:_check_disk_queue).with(no_args) subject.check_disk_queue(1,0) end @@ -1252,12 +1155,9 @@ EOT end it 'when a non-default loop delay is specified' do - start_time = Time.now - subject.check_disk_queue(maxloop,loop_delay) - finish_time = Time.now + expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times - # Use a generous delta to take into account various CPU load etc. - expect(finish_time - start_time).to be_within(0.75).of(maxloop * loop_delay) + subject.check_disk_queue(maxloop,loop_delay) end end @@ -1271,7 +1171,7 @@ EOT end it 'should call _check_disk_queue 5 times' do - expect(subject).to receive(:_check_disk_queue).with(Vmpooler::VsphereHelper).exactly(maxloop).times + expect(subject).to receive(:_check_disk_queue).with(no_args).exactly(maxloop).times subject.check_disk_queue(maxloop,0) end @@ -1279,8 +1179,6 @@ EOT end describe '#_check_disk_queue' do - let(:provider) { double('provider') } - before do expect(subject).not_to be_nil end @@ -1288,31 +1186,72 @@ EOT context 'when no VMs in the queue' do it 'should not call create_vm_disk' do expect(subject).to receive(:create_vm_disk).exactly(0).times - subject._check_disk_queue(provider) + subject._check_disk_queue + end + end + + context 'when VM in the queue does not exist' do + before(:each) do + disk_task_vm(vm,"snapshot_#{vm}") + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/) + + subject._check_disk_queue + end + + it 'should not call create_vm_disk' do + expect(subject).to receive(:create_vm_disk).exactly(0).times + + subject._check_disk_queue + end + end + + context 'when specified provider does not exist' do + before(:each) do + disk_task_vm(vm,"snapshot_#{vm}") + create_running_vm(pool, vm, token) + expect(subject).to receive(:get_provider_for_pool).and_return(nil) + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Missing Provider for/) + + subject._check_disk_queue + end + + it 'should not call create_vm_disk' do + expect(subject).to receive(:create_vm_disk).exactly(0).times + + subject._check_disk_queue end end context 'when multiple VMs in the queue' do before(:each) do - disk_task_vm('vm1',1) - disk_task_vm('vm2',2) - disk_task_vm('vm3',3) + ['vm1', 'vm2', 'vm3'].each do |vm_name| + disk_task_vm(vm_name,"snapshot_#{vm_name}") + create_running_vm(pool, vm_name, token) + end + + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) end it 'should call create_vm_disk once' do expect(subject).to receive(:create_vm_disk).exactly(1).times - subject._check_disk_queue(provider) + subject._check_disk_queue end - it 'should snapshot the first VM in the queue' do - expect(subject).to receive(:create_vm_disk).with('vm1','1',provider) - subject._check_disk_queue(provider) + it 'should create the disk for the first VM in the queue' do + expect(subject).to receive(:create_vm_disk).with(pool,'vm1','snapshot_vm1',provider) + subject._check_disk_queue end it 'should log an error if one occurs' do expect(subject).to receive(:create_vm_disk).and_raise(RuntimeError,'MockError') - expect(logger).to receive(:log).with('s', "[!] [disk_manager] disk creation appears to have failed") - subject._check_disk_queue(provider) + expect(logger).to receive(:log).with('s', "[!] [disk_manager] disk creation appears to have failed: MockError") + subject._check_disk_queue end end end @@ -1322,10 +1261,10 @@ EOT before(:each) do expect(Thread).to receive(:new).and_yield - allow(subject).to receive(:_check_snapshot_queue) + allow(subject).to receive(:_check_snapshot_queue).with(no_args) end - it 'should log the disk manager is starting' do + it 'should log the snapshot manager is starting' do expect(logger).to receive(:log).with('d', "[*] [snapshot_manager] starting worker thread") expect($threads.count).to be(0) @@ -1341,7 +1280,7 @@ EOT end it 'should call _check_snapshot_queue' do - expect(subject).to receive(:_check_snapshot_queue).with(Vmpooler::VsphereHelper) + expect(subject).to receive(:_check_snapshot_queue).with(no_args) subject.check_snapshot_queue(1,0) end @@ -1357,12 +1296,9 @@ EOT end it 'when a non-default loop delay is specified' do - start_time = Time.now - subject.check_snapshot_queue(maxloop,loop_delay) - finish_time = Time.now + expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times - # Use a generous delta to take into account various CPU load etc. - expect(finish_time - start_time).to be_within(0.75).of(maxloop * loop_delay) + subject.check_snapshot_queue(maxloop,loop_delay) end end @@ -1376,7 +1312,7 @@ EOT end it 'should call _check_snapshot_queue 5 times' do - expect(subject).to receive(:_check_snapshot_queue).with(Vmpooler::VsphereHelper).exactly(maxloop).times + expect(subject).to receive(:_check_snapshot_queue).with(no_args).exactly(maxloop).times subject.check_snapshot_queue(maxloop,0) end @@ -1384,8 +1320,6 @@ EOT end describe '#_check_snapshot_queue' do - let(:provider) { double('provider') } - before do expect(subject).not_to be_nil end @@ -1394,64 +1328,146 @@ EOT context 'when no VMs in the queue' do it 'should not call create_vm_snapshot' do expect(subject).to receive(:create_vm_snapshot).exactly(0).times - subject._check_snapshot_queue(provider) + subject._check_snapshot_queue + end + end + + context 'when VM in the queue does not exist' do + before(:each) do + snapshot_vm(vm,"snapshot_#{vm}") + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/) + + subject._check_snapshot_queue + end + + it 'should not call create_vm_snapshot' do + expect(subject).to receive(:create_vm_snapshot).exactly(0).times + + subject._check_snapshot_queue + end + end + + context 'when specified provider does not exist' do + before(:each) do + snapshot_vm(vm,"snapshot_#{vm}") + create_running_vm(pool, vm, token) + expect(subject).to receive(:get_provider_for_pool).and_return(nil) + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Missing Provider for/) + + subject._check_snapshot_queue + end + + it 'should not call create_vm_snapshot' do + expect(subject).to receive(:create_vm_snapshot).exactly(0).times + + subject._check_snapshot_queue end end context 'when multiple VMs in the queue' do before(:each) do - snapshot_vm('vm1','snapshot1') - snapshot_vm('vm2','snapshot2') - snapshot_vm('vm3','snapshot3') + ['vm1', 'vm2', 'vm3'].each do |vm_name| + snapshot_vm(vm_name,"snapshot_#{vm_name}") + create_running_vm(pool, vm_name, token) + end + + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) end it 'should call create_vm_snapshot once' do expect(subject).to receive(:create_vm_snapshot).exactly(1).times - subject._check_snapshot_queue(provider) + subject._check_snapshot_queue end it 'should snapshot the first VM in the queue' do - expect(subject).to receive(:create_vm_snapshot).with('vm1','snapshot1',provider) - subject._check_snapshot_queue(provider) + expect(subject).to receive(:create_vm_snapshot).with(pool,'vm1','snapshot_vm1',provider) + subject._check_snapshot_queue end it 'should log an error if one occurs' do expect(subject).to receive(:create_vm_snapshot).and_raise(RuntimeError,'MockError') - expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot appears to have failed") - subject._check_snapshot_queue(provider) + expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot create appears to have failed: MockError") + subject._check_snapshot_queue end end end - context 'revert_vm_snapshot queue' do + context 'vmpooler__tasks__snapshot-revert queue' do context 'when no VMs in the queue' do it 'should not call revert_vm_snapshot' do expect(subject).to receive(:revert_vm_snapshot).exactly(0).times - subject._check_snapshot_queue(provider) + subject._check_snapshot_queue + end + end + + context 'when VM in the queue does not exist' do + before(:each) do + snapshot_revert_vm(vm,"snapshot_#{vm}") + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Unable to determine which pool #{vm} is a member of/) + + subject._check_snapshot_queue + end + + it 'should not call revert_vm_snapshot' do + expect(subject).to receive(:revert_vm_snapshot).exactly(0).times + + subject._check_snapshot_queue + end + end + + context 'when specified provider does not exist' do + before(:each) do + snapshot_revert_vm(vm,"snapshot_#{vm}") + create_running_vm(pool, vm, token) + expect(subject).to receive(:get_provider_for_pool).and_return(nil) + end + + it 'should log an error' do + expect(logger).to receive(:log).with('s', /Missing Provider for/) + + subject._check_snapshot_queue + end + + it 'should not call revert_vm_snapshot' do + expect(subject).to receive(:revert_vm_snapshot).exactly(0).times + + subject._check_snapshot_queue end end context 'when multiple VMs in the queue' do before(:each) do - snapshot_revert_vm('vm1','snapshot1') - snapshot_revert_vm('vm2','snapshot2') - snapshot_revert_vm('vm3','snapshot3') + ['vm1', 'vm2', 'vm3'].each do |vm_name| + snapshot_revert_vm(vm_name,"snapshot_#{vm_name}") + create_running_vm(pool, vm_name, token) + end + + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) end it 'should call revert_vm_snapshot once' do expect(subject).to receive(:revert_vm_snapshot).exactly(1).times - subject._check_snapshot_queue(provider) + subject._check_snapshot_queue end - it 'should revert snapshot the first VM in the queue' do - expect(subject).to receive(:revert_vm_snapshot).with('vm1','snapshot1',provider) - subject._check_snapshot_queue(provider) + it 'should snapshot the first VM in the queue' do + expect(subject).to receive(:revert_vm_snapshot).with(pool,'vm1','snapshot_vm1',provider) + subject._check_snapshot_queue end it 'should log an error if one occurs' do expect(subject).to receive(:revert_vm_snapshot).and_raise(RuntimeError,'MockError') - expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot revert appears to have failed") - subject._check_snapshot_queue(provider) + expect(logger).to receive(:log).with('s', "[!] [snapshot_manager] snapshot revert appears to have failed: MockError") + subject._check_snapshot_queue end end end @@ -1481,60 +1497,75 @@ EOT end describe '#migrate_vm' do - let(:provider) { double('provider') } - - before do + before(:each) do expect(subject).not_to be_nil + expect(Thread).to receive(:new).and_yield end it 'calls _migrate_vm' do - expect(Thread).to receive(:new).and_yield expect(subject).to receive(:_migrate_vm).with(vm, pool, provider) subject.migrate_vm(vm, pool, provider) end + + context 'When an error is raised' do + before(:each) do + expect(subject).to receive(:_migrate_vm).with(vm, pool, provider).and_raise('MockError') + end + + it 'logs a message' do + allow(logger).to receive(:log) + expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm}' migration failed with an error: MockError") + + subject.migrate_vm(vm, pool, provider) + end + + it 'should attempt to remove from vmpooler_migration queue' do + expect(subject).to receive(:remove_vmpooler_migration_vm).with(pool, vm) + + subject.migrate_vm(vm, pool, provider) + end + end end describe "#_migrate_vm" do - let(:provider) { double('provider') } let(:vm_parent_hostname) { 'parent1' } let(:config) { YAML.load(<<-EOT --- :config: migration_limit: 5 -:pools: - - name: #{pool} EOT ) } - before do + before(:each) do expect(subject).not_to be_nil + allow(provider).to receive(:get_vm_host).with(pool, vm).and_return(vm_parent_hostname) end - context 'when an error occurs' do - it 'should log an error message and attempt to remove from vmpooler_migration queue' do - expect(provider).to receive(:find_vm).with(vm).and_raise(RuntimeError,'MockError') - expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm}' migration failed with an error: MockError") - expect(subject).to receive(:remove_vmpooler_migration_vm) - subject._migrate_vm(vm, pool, provider) + context 'when an error occurs trying to retrieve the current host' do + before(:each) do + expect(provider).to receive(:get_vm_host).with(pool, vm).and_raise(RuntimeError,'MockError') + end + + it 'should raise an error' do + expect{ subject._migrate_vm(vm, pool, provider) }.to raise_error('MockError') end end - context 'when VM does not exist' do - it 'should log an error message when VM does not exist' do - expect(provider).to receive(:find_vm).with(vm).and_return(nil) - # This test is quite fragile. Should refactor the code to make this scenario easier to detect - expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm}' migration failed with an error: undefined method `summary' for nil:NilClass") - subject._migrate_vm(vm, pool, provider) + context 'when the current host can not be determined' do + before(:each) do + expect(provider).to receive(:get_vm_host).with(pool, vm).and_return(nil) + end + + it 'should raise an error' do + expect{ subject._migrate_vm(vm, pool, provider) }.to raise_error(/Unable to determine which host the VM is running on/) end end context 'when VM exists but migration is disabled' do before(:each) do - expect(provider).to receive(:find_vm).with(vm).and_return(host) - allow(subject).to receive(:get_vm_host_info).with(host).and_return([{'name' => vm_parent_hostname}, vm_parent_hostname]) create_migrating_vm(vm, pool) end @@ -1558,8 +1589,6 @@ EOT context 'when VM exists but migration limit is reached' do before(:each) do - expect(provider).to receive(:find_vm).with(vm).and_return(host) - allow(subject).to receive(:get_vm_host_info).with(host).and_return([{'name' => vm_parent_hostname}, vm_parent_hostname]) create_migrating_vm(vm, pool) redis.sadd('vmpooler__migration', 'fakevm1') @@ -1583,9 +1612,6 @@ EOT context 'when VM exists but migration limit is not yet reached' do before(:each) do - expect(provider).to receive(:find_vm).with(vm).and_return(host) - allow(subject).to receive(:get_vm_host_info).with(host).and_return([{'name' => vm_parent_hostname}, vm_parent_hostname]) - create_migrating_vm(vm, pool) redis.sadd('vmpooler__migration', 'fakevm1') redis.sadd('vmpooler__migration', 'fakevm2') @@ -1593,7 +1619,7 @@ EOT context 'and host to migrate to is the same as the current host' do before(:each) do - expect(provider).to receive(:find_least_used_compatible_host).with(host).and_return([{'name' => vm_parent_hostname}, vm_parent_hostname]) + expect(provider).to receive(:find_least_used_compatible_host).with(vm).and_return(vm_parent_hostname) end it "should not migrate the VM" do @@ -1622,8 +1648,8 @@ EOT context 'and host to migrate to different to the current host' do let(:vm_new_hostname) { 'new_hostname' } before(:each) do - expect(provider).to receive(:find_least_used_compatible_host).with(host).and_return([{'name' => vm_new_hostname}, vm_new_hostname]) - expect(subject).to receive(:migrate_vm_and_record_timing).with(host, vm, pool, Object, vm_parent_hostname, vm_new_hostname, provider).and_return('1.00') + expect(provider).to receive(:find_least_used_compatible_host).with(vm).and_return(vm_new_hostname) + expect(subject).to receive(:migrate_vm_and_record_timing).with(vm, pool, vm_parent_hostname, vm_new_hostname, provider).and_return('1.00') end it "should migrate the VM" do @@ -1651,25 +1677,7 @@ EOT end end - describe "#get_vm_host_info" do - before do - expect(subject).not_to be_nil - end - - let(:vm_object) { double('vm_object') } - let(:parent_host) { double('parent_host') } - - it 'should return an array with host information' do - expect(vm_object).to receive_message_chain(:summary, :runtime, :host).and_return(parent_host) - expect(parent_host).to receive(:name).and_return('vmhostname') - - expect(subject.get_vm_host_info(vm_object)).to eq([parent_host,'vmhostname']) - end - end - describe "#execute!" do - let(:threads) {{}} - let(:config) { YAML.load(<<-EOT --- @@ -1679,18 +1687,26 @@ EOT ) } - let(:thread) { double('thread') } - - before do + before(:each) do expect(subject).not_to be_nil + + allow(subject).to receive(:check_disk_queue) + allow(subject).to receive(:check_snapshot_queue) + allow(subject).to receive(:check_pool) + + allow(logger).to receive(:log) + end + + after(:each) do + # Reset the global variable - Note this is a code smell + $threads = nil end context 'on startup' do - before(:each) do - allow(subject).to receive(:check_disk_queue) - allow(subject).to receive(:check_snapshot_queue) - allow(subject).to receive(:check_pool) + it 'should log a message that VMPooler has started' do expect(logger).to receive(:log).with('d', 'starting vmpooler') + + subject.execute!(1,0) end it 'should set clone tasks to zero' do @@ -1722,69 +1738,146 @@ EOT subject.execute!(1,0) end + + context 'creating Providers' do + let(:vsphere_provider) { double('vsphere_provider') } + let(:config) { + YAML.load(<<-EOT +--- +:pools: + - name: #{pool} + - name: 'dummy' + provider: 'vsphere' +EOT + )} + + it 'should call create_provider_object idempotently' do + # Even though there are two pools using the vsphere provider (the default), it should only + # create the provider object once. + expect(subject).to receive(:create_provider_object).with(Object, Object, Object, 'vsphere', Object).and_return(vsphere_provider) + + subject.execute!(1,0) + end + + it 'should raise an error if the provider can not be created' do + expect(subject).to receive(:create_provider_object).and_raise(RuntimeError, "MockError") + + expect{ subject.execute!(1,0) }.to raise_error(/MockError/) + end + + it 'should log a message if the provider can not be created' do + expect(subject).to receive(:create_provider_object).and_raise(RuntimeError, "MockError") + expect(logger).to receive(:log).with('s',"Error while creating provider for pool #{pool}: MockError") + + expect{ subject.execute!(1,0) }.to raise_error(/MockError/) + end + end + end + + context 'modify configuration on startup' do + context 'move vSphere configuration to providers location' do + let(:config) { + YAML.load(<<-EOT +:vsphere: + server: 'vsphere.company.com' + username: 'vmpooler' + password: 'password' +:pools: + - name: #{pool} +EOT + )} + + it 'should set providers with no provider to vsphere' do + expect(subject.config[:providers]).to be nil + + subject.execute!(1,0) + expect(subject.config[:providers][:vsphere]['server']).to eq('vsphere.company.com') + expect(subject.config[:providers][:vsphere]['username']).to eq('vmpooler') + expect(subject.config[:providers][:vsphere]['password']).to eq('password') + end + + it 'should log a message' do + expect(logger).to receive(:log).with('d', "[!] Detected an older configuration file. Copying the settings from ':vsphere:' to ':providers:/:vsphere:'") + + subject.execute!(1,0) + end + end + + + context 'default to the vphere provider' do + let(:config) { + YAML.load(<<-EOT +--- +:pools: + - name: #{pool} + - name: 'dummy' + provider: 'dummy' +EOT + )} + + it 'should set providers with no provider to vsphere' do + expect(subject.config[:pools][0]['provider']).to be_nil + expect(subject.config[:pools][1]['provider']).to eq('dummy') + + subject.execute!(1,0) + + expect(subject.config[:pools][0]['provider']).to eq('vsphere') + expect(subject.config[:pools][1]['provider']).to eq('dummy') + end + + it 'should log a message' do + expect(logger).to receive(:log).with('d', "[!] Setting provider for pool '#{pool}' to 'vsphere' as default") + + subject.execute!(1,0) + end + end end context 'with dead disk_manager thread' do - before(:each) do - allow(subject).to receive(:check_snapshot_queue) - allow(subject).to receive(:check_pool) - expect(logger).to receive(:log).with('d', 'starting vmpooler') - end + let(:disk_manager_thread) { double('thread', :alive? => false) } - after(:each) do + before(:each) do # Reset the global variable - Note this is a code smell - $threads = nil + $threads = {} + $threads['disk_manager'] = disk_manager_thread end it 'should run the check_disk_queue method and log a message' do - expect(thread).to receive(:alive?).and_return(false) expect(subject).to receive(:check_disk_queue) expect(logger).to receive(:log).with('d', "[!] [disk_manager] worker thread died, restarting") - $threads['disk_manager'] = thread subject.execute!(1,0) end end context 'with dead snapshot_manager thread' do + let(:snapshot_manager_thread) { double('thread', :alive? => false) } before(:each) do - allow(subject).to receive(:check_disk_queue) - allow(subject).to receive(:check_pool) - expect(logger).to receive(:log).with('d', 'starting vmpooler') - end - - after(:each) do # Reset the global variable - Note this is a code smell - $threads = nil + $threads = {} + $threads['snapshot_manager'] = snapshot_manager_thread end it 'should run the check_snapshot_queue method and log a message' do - expect(thread).to receive(:alive?).and_return(false) expect(subject).to receive(:check_snapshot_queue) expect(logger).to receive(:log).with('d', "[!] [snapshot_manager] worker thread died, restarting") - $threads['snapshot_manager'] = thread + $threads['snapshot_manager'] = snapshot_manager_thread subject.execute!(1,0) end end context 'with dead pool thread' do + let(:pool_thread) { double('thread', :alive? => false) } before(:each) do - allow(subject).to receive(:check_disk_queue) - allow(subject).to receive(:check_snapshot_queue) - expect(logger).to receive(:log).with('d', 'starting vmpooler') - end - - after(:each) do # Reset the global variable - Note this is a code smell - $threads = nil + $threads = {} + $threads[pool] = pool_thread end it 'should run the check_pool method and log a message' do - expect(thread).to receive(:alive?).and_return(false) expect(subject).to receive(:check_pool).with(a_pool_with_name_of(pool)) expect(logger).to receive(:log).with('d', "[!] [#{pool}] worker thread died, restarting") - $threads[pool] = thread subject.execute!(1,0) end @@ -1802,30 +1895,22 @@ EOT end it 'when a non-default loop delay is specified' do - start_time = Time.now - subject.execute!(maxloop,loop_delay) - finish_time = Time.now + expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times - # Use a generous delta to take into account various CPU load etc. - expect(finish_time - start_time).to be_within(0.75).of(maxloop * loop_delay) + subject.execute!(maxloop,loop_delay) end end context 'loops specified number of times (5)' do + let(:alive_thread) { double('thread', :alive? => true) } let(:maxloop) { 5 } # Note a maxloop of zero can not be tested as it never terminates before(:each) do end - after(:each) do - # Reset the global variable - Note this is a code smell - $threads = nil - end - it 'should run startup tasks only once' do - allow(subject).to receive(:check_disk_queue) - allow(subject).to receive(:check_snapshot_queue) - allow(subject).to receive(:check_pool) + expect(redis).to receive(:set).with('vmpooler__tasks__clone', 0).once + expect(redis).to receive(:del).with('vmpooler__migration').once subject.execute!(maxloop,0) end @@ -1843,10 +1928,9 @@ EOT expect(subject).to receive(:check_snapshot_queue).exactly(0).times expect(subject).to receive(:check_pool).exactly(0).times - allow(thread).to receive(:alive?).and_return(true) - $threads[pool] = thread - $threads['disk_manager'] = thread - $threads['snapshot_manager'] = thread + $threads[pool] = alive_thread + $threads['disk_manager'] = alive_thread + $threads['snapshot_manager'] = alive_thread subject.execute!(maxloop,0) end @@ -1855,48 +1939,40 @@ EOT describe "#check_pool" do let(:threads) {{}} - let(:provider) {{}} - + let(:provider_name) { 'mock_provider' } let(:config) { YAML.load(<<-EOT --- :pools: - name: #{pool} + provider: #{provider_name} EOT ) } - let(:thread) { double('thread') } let(:pool_object) { config[:pools][0] } before do expect(subject).not_to be_nil expect(Thread).to receive(:new).and_yield + allow(subject).to receive(:get_provider_for_pool).with(pool).and_return(provider) end context 'on startup' do before(:each) do - # Note the Vmpooler::VsphereHelper is not mocked - allow(subject).to receive(:_check_pool) + allow(subject).to receive(:_check_pool) expect(logger).to receive(:log).with('d', "[*] [#{pool}] starting worker thread") end after(:each) do # Reset the global variable - Note this is a code smell $threads = nil - $providers = nil end it 'should log a message the worker thread is starting' do subject.check_pool(pool_object,1,0) end - it 'should populate the providers global variable' do - subject.check_pool(pool_object,1,0) - - expect($providers[pool]).to_not be_nil - end - it 'should populate the threads global variable' do subject.check_pool(pool_object,1,0) @@ -1913,22 +1989,18 @@ EOT before(:each) do allow(logger).to receive(:log) # Note the Vmpooler::VsphereHelper is not mocked - allow(subject).to receive(:_check_pool) + allow(subject).to receive(:_check_pool) end after(:each) do # Reset the global variable - Note this is a code smell $threads = nil - $provider = nil end it 'when a non-default loop delay is specified' do - start_time = Time.now - subject.check_pool(pool_object,maxloop,loop_delay) - finish_time = Time.now + expect(subject).to receive(:sleep).with(loop_delay).exactly(maxloop).times - # Use a generous delta to take into account various CPU load etc. - expect(finish_time - start_time).to be_within(0.75).of(maxloop * loop_delay) + subject.check_pool(pool_object,maxloop,loop_delay) end end @@ -1981,50 +2053,61 @@ EOT end describe '#migrate_vm_and_record_timing' do - let(:provider) { double('provider') } - let(:vm_object) { double('vm_object') } let(:source_host_name) { 'source_host' } let(:dest_host_name) { 'dest_host' } - before do - expect(subject).not_to be_nil - end - before(:each) do create_vm(vm,token) - expect(provider).to receive(:migrate_vm_host).with(vm_object, host) + expect(subject).not_to be_nil + + expect(provider).to receive(:migrate_vm_to_host).with(pool, vm, dest_host_name) end it 'should return the elapsed time for the migration' do - result = subject.migrate_vm_and_record_timing(vm_object, vm, pool, host, source_host_name, dest_host_name, provider) + result = subject.migrate_vm_and_record_timing(vm, pool, source_host_name, dest_host_name, provider) expect(result).to match(/0\.[\d]+/) end it 'should add timing metric' do expect(metrics).to receive(:timing).with("migrate.#{pool}",String) - subject.migrate_vm_and_record_timing(vm_object, vm, pool, host, source_host_name, dest_host_name, provider) + subject.migrate_vm_and_record_timing(vm, pool, source_host_name, dest_host_name, provider) end it 'should increment from_host and to_host metric' do expect(metrics).to receive(:increment).with("migrate_from.#{source_host_name}") expect(metrics).to receive(:increment).with("migrate_to.#{dest_host_name}") - subject.migrate_vm_and_record_timing(vm_object, vm, pool, host, source_host_name, dest_host_name, provider) + subject.migrate_vm_and_record_timing(vm, pool, source_host_name, dest_host_name, provider) end it 'should set migration_time metric in redis' do expect(redis.hget("vmpooler__vm__#{vm}", 'migration_time')).to be_nil - subject.migrate_vm_and_record_timing(vm_object, vm, pool, host, source_host_name, dest_host_name, provider) + subject.migrate_vm_and_record_timing(vm, pool, source_host_name, dest_host_name, provider) expect(redis.hget("vmpooler__vm__#{vm}", 'migration_time')).to match(/0\.[\d]+/) end it 'should set checkout_to_migration metric in redis' do expect(redis.hget("vmpooler__vm__#{vm}", 'checkout_to_migration')).to be_nil - subject.migrate_vm_and_record_timing(vm_object, vm, pool, host, source_host_name, dest_host_name, provider) + subject.migrate_vm_and_record_timing(vm, pool, source_host_name, dest_host_name, provider) expect(redis.hget("vmpooler__vm__#{vm}", 'checkout_to_migration')).to match(/0\.[\d]+/) end end describe '#_check_pool' do + let(:new_vm_response) { + # Mock response from Base Provider for vms_in_pool + [{ 'name' => new_vm}] + } + let(:vm_response) { + # Mock response from Base Provider for vms_in_pool + [{ 'name' => vm}] + } + let(:multi_vm_response) { + # Mock response from Base Provider for vms_in_pool + [{ 'name' => 'vm1'}, + { 'name' => 'vm2'}, + { 'name' => 'vm3'}] + } + # Default test fixtures will consist of; # - Empty Redis dataset # - A single pool with a pool size of zero i.e. no new VMs should be created @@ -2036,18 +2119,16 @@ EOT task_limit: 10 :pools: - name: #{pool} - folder: 'vm_folder' size: 0 EOT ) } let(:pool_object) { config[:pools][0] } - let(:provider) { double('provider') } let(:new_vm) { 'newvm'} before do expect(subject).not_to be_nil - allow(logger).to receive(:log).with("s", "[!] [#{pool}] is empty") + allow(logger).to receive(:log) end # INVENTORY @@ -2059,24 +2140,23 @@ EOT allow(subject).to receive(:check_pending_vm) allow(subject).to receive(:destroy_vm) allow(subject).to receive(:clone_vm) + allow(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) end it 'should log an error if one occurs' do - expect(provider).to receive(:find_folder).and_raise(RuntimeError,'Mock Error') + expect(provider).to receive(:vms_in_pool).and_raise(RuntimeError,'Mock Error') expect(logger).to receive(:log).with('s', "[!] [#{pool}] _check_pool failed with an error while inspecting inventory: Mock Error") subject._check_pool(pool_object,provider) end it 'should log the discovery of VMs' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") subject._check_pool(pool_object,provider) end it 'should add undiscovered VMs to the completed queue' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) allow(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false) @@ -2090,8 +2170,6 @@ EOT ['running','ready','pending','completed','discovered','migrating'].each do |queue_name| it "should not discover VMs in the #{queue_name} queue" do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) - expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue").exactly(0).times expect(redis.sismember("vmpooler__discovered__#{pool}", new_vm)).to be(false) redis.sadd("vmpooler__#{queue_name}__#{pool}", new_vm) @@ -2111,7 +2189,7 @@ EOT # RUNNING context 'Running VM not in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") create_running_vm(pool,vm,token) end @@ -2125,7 +2203,7 @@ EOT context 'Running VM in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) allow(subject).to receive(:check_running_vm) create_running_vm(pool,vm,token) end @@ -2164,7 +2242,7 @@ EOT # READY context 'Ready VM not in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") create_ready_vm(pool,vm,token) end @@ -2178,7 +2256,7 @@ EOT context 'Ready VM in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) allow(subject).to receive(:check_ready_vm) create_ready_vm(pool,vm,token) end @@ -2209,7 +2287,7 @@ EOT # PENDING context 'Pending VM not in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") create_pending_vm(pool,vm,token) end @@ -2224,7 +2302,7 @@ EOT context 'Pending VM in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) allow(subject).to receive(:check_pending_vm) create_pending_vm(pool,vm,token) end @@ -2264,7 +2342,7 @@ EOT # COMPLETED context 'Completed VM not in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory, removed from 'completed' queue") create_completed_vm(vm,pool,true) @@ -2295,7 +2373,7 @@ EOT context 'Completed VM in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) create_completed_vm(vm,pool,true) end @@ -2332,7 +2410,7 @@ EOT # DISCOVERED context 'Discovered VM' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) create_discovered_vm(vm,pool) end @@ -2391,7 +2469,7 @@ EOT # MIGRATIONS context 'Migrating VM not in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([new_vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(new_vm_response) expect(logger).to receive(:log).with('s', "[?] [#{pool}] '#{new_vm}' added to 'discovered' queue") create_migrating_vm(vm,pool) end @@ -2405,7 +2483,7 @@ EOT context 'Migrating VM in the inventory' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) allow(subject).to receive(:check_ready_vm) allow(logger).to receive(:log).with("s", "[!] [#{pool}] is empty") create_migrating_vm(vm,pool) @@ -2428,14 +2506,14 @@ EOT # REPOPULATE context 'Repopulate a pool' do it 'should not call clone_vm when number of VMs is equal to the pool size' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return([]) expect(subject).to receive(:clone_vm).exactly(0).times subject._check_pool(pool_object,provider) end it 'should not call clone_vm when number of VMs is greater than the pool size' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) create_ready_vm(pool,vm,token) expect(subject).to receive(:clone_vm).exactly(0).times @@ -2444,7 +2522,7 @@ EOT ['ready','pending'].each do |queue_name| it "should use VMs in #{queue_name} queue to caculate pool size" do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) expect(subject).to receive(:clone_vm).exactly(0).times # Modify the pool size to 1 and add a VM in the queue redis.sadd("vmpooler__#{queue_name}__#{pool}",vm) @@ -2456,7 +2534,7 @@ EOT ['running','completed','discovered','migrating'].each do |queue_name| it "should not use VMs in #{queue_name} queue to caculate pool size" do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([vm])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(vm_response) expect(subject).to receive(:clone_vm) # Modify the pool size to 1 and add a VM in the queue redis.sadd("vmpooler__#{queue_name}__#{pool}",vm) @@ -2467,7 +2545,7 @@ EOT end it 'should log a message the first time a pool is empty' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return([]) expect(logger).to receive(:log).with('s', "[!] [#{pool}] is empty") subject._check_pool(pool_object,provider) @@ -2475,7 +2553,7 @@ EOT context 'when pool is marked as empty' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return([]) redis.set("vmpooler__empty__#{pool}", 'true') end @@ -2496,7 +2574,7 @@ EOT context 'when number of VMs is less than the pool size' do before(:each) do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return([]) end it 'should call clone_vm to populate the pool' do @@ -2536,7 +2614,7 @@ EOT create_ready_vm(pool,'vm1') create_ready_vm(pool,'vm2') create_ready_vm(pool,'vm3') - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new(['vm1','vm2','vm3'])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(multi_vm_response) expect(metrics).to receive(:gauge).with("ready.#{pool}", 3) allow(metrics).to receive(:gauge) @@ -2548,7 +2626,7 @@ EOT create_running_vm(pool,'vm1',token) create_running_vm(pool,'vm2',token) create_running_vm(pool,'vm3',token) - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new(['vm1','vm2','vm3'])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return(multi_vm_response) expect(metrics).to receive(:gauge).with("running.#{pool}", 3) allow(metrics).to receive(:gauge) @@ -2557,7 +2635,7 @@ EOT end it 'increments metrics with 0 when pool empty' do - expect(provider).to receive(:find_folder).and_return(MockFindFolder.new([])) + expect(provider).to receive(:vms_in_pool).with(pool).and_return([]) expect(metrics).to receive(:gauge).with("ready.#{pool}", 0) expect(metrics).to receive(:gauge).with("running.#{pool}", 0) @@ -2567,21 +2645,4 @@ EOT end end end - - describe '#_check_snapshot_queue' do - let(:pool_helper) { double('pool') } - let(:provider) { {pool => pool_helper} } - - before do - expect(subject).not_to be_nil - $provider = provider - end - - it 'checks appropriate redis queues' do - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot') - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert') - - subject._check_snapshot_queue(provider) - end - end end diff --git a/spec/unit/providers/base_spec.rb b/spec/unit/providers/base_spec.rb index b9b5155..5e7feb6 100644 --- a/spec/unit/providers/base_spec.rb +++ b/spec/unit/providers/base_spec.rb @@ -84,7 +84,7 @@ EOT ) } - context 'Given a misconfigured provider name' do + context 'Given a provider with no configuration' do let(:config) { YAML.load(<<-EOT --- :providers: @@ -94,8 +94,8 @@ EOT EOT ) } - it 'should return nil' do - expect(subject.provider_config).to be_nil + it 'should return empty hash' do + expect(subject.provider_config).to eq({}) end end @@ -120,6 +120,26 @@ EOT end end + describe '#provided_pools' do + let(:config) { YAML.load(<<-EOT +--- +:pools: + - name: 'pool1' + provider: 'base' + - name: 'pool2' + provider: 'base' + - name: 'otherpool' + provider: 'other provider' + - name: 'no name' +EOT + ) + } + + it "should return pools serviced by this provider" do + expect(subject.provided_pools).to eq(['pool1','pool2']) + end + end + describe '#vms_in_pool' do it 'should raise error' do expect{subject.vms_in_pool('pool')}.to raise_error(/does not implement vms_in_pool/) diff --git a/spec/unit/providers/dummy_spec.rb b/spec/unit/providers/dummy_spec.rb new file mode 100644 index 0000000..f0156f9 --- /dev/null +++ b/spec/unit/providers/dummy_spec.rb @@ -0,0 +1,509 @@ +require 'spec_helper' + +describe 'Vmpooler::PoolManager::Provider::Dummy' do + let(:logger) { MockLogger.new } + let(:metrics) { Vmpooler::DummyStatsd.new } + let(:pool_name) { 'pool1' } + let(:other_pool_name) { 'pool2' } + let(:vm_name) { 'vm1' } + + let(:running_vm_name) { 'vm2' } + let(:notready_vm_name) { 'vm3' } + + let (:provider_options) { + # Construct an initial state for testing + dummylist = {} + dummylist['pool'] = {} + # pool1 is a pool of "normal" VMs + dummylist['pool'][pool_name] = [] + # A normal running VM + vm = {} + vm['name'] = vm_name + vm['hostname'] = vm_name + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = pool_name + vm['poolname'] = pool_name + vm['ready'] = true + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['snapshots'] = [] + vm['disks'] = [] + vm['dummy_state'] = 'RUNNING' + dummylist['pool'][pool_name] << vm + + # pool2 is a pool of "abnormal" VMs e.g. PoweredOff etc. + dummylist['pool'][other_pool_name] = [] + # A freshly provisioned VM that is not ready + vm = {} + vm['name'] = running_vm_name + vm['hostname'] = running_vm_name + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = other_pool_name + vm['poolname'] = other_pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['snapshots'] = [] + vm['disks'] = [] + vm['dummy_state'] = 'UNKNOWN' + dummylist['pool'][other_pool_name] << vm + # A freshly provisioned VM that is running but not ready + vm = {} + vm['name'] = notready_vm_name + vm['hostname'] = notready_vm_name + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = other_pool_name + vm['poolname'] = other_pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['snapshots'] = [] + vm['disks'] = [] + vm['dummy_state'] = 'RUNNING' + dummylist['pool'][other_pool_name] << vm + + { + 'initial_state' => dummylist + } + } + + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :dummy: + key1: 'value1' + # Drop the connection pool timeout way down for spec tests so they fail fast + connection_pool_timeout: 1 +:pools: + - name: '#{pool_name}' + size: 5 + - name: 'pool2' + size: 5 +EOT + ) + } + + subject { Vmpooler::PoolManager::Provider::Dummy.new(config, logger, metrics, 'dummy', provider_options) } + + describe '#name' do + it 'should be dummy' do + expect(subject.name).to eq('dummy') + end + end + + describe '#vms_in_pool' do + it 'should return [] when pool does not exist' do + vm_list = subject.vms_in_pool('missing_pool') + + expect(vm_list).to eq([]) + end + + it 'should return an array of VMs when pool exists' do + vm_list = subject.vms_in_pool(pool_name) + + expect(vm_list.count).to eq(1) + end + end + + describe '#get_vm_host' do + it 'should return the hostname when VM exists' do + expect(subject.get_vm_host(pool_name, vm_name)).to eq('HOST1') + end + + it 'should error when VM does not exist' do + expect{subject.get_vm_host(pool_name, 'doesnotexist')}.to raise_error(RuntimeError) + end + end + + describe '#find_least_used_compatible_host' do + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host(pool_name, vm_name) + expect(new_host).to eq('HOST1') + end + + context 'using migratevm_couldmove_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['migratevm_couldmove_percent'] = 0 + end + + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host(pool_name, vm_name) + expect(new_host).to eq('HOST1') + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['migratevm_couldmove_percent'] = 100 + end + + it 'should return a different host' do + new_host = subject.find_least_used_compatible_host(pool_name, vm_name) + expect(new_host).to_not eq('HOST1') + end + end + + end + end + + describe '#migrate_vm_to_host' do + it 'should move to the new host' do + expect(subject.migrate_vm_to_host(pool_name, 'vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host(pool_name, 'vm1')).to eq('NEWHOST') + end + + context 'using migratevm_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['migratevm_fail_percent'] = 0 + end + + it 'should move to the new host' do + expect(subject.migrate_vm_to_host(pool_name, 'vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host(pool_name, 'vm1')).to eq('NEWHOST') + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['migratevm_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{subject.migrate_vm_to_host(pool_name, 'vm1','NEWHOST')}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#get_vm' do + it 'should return the VM when VM exists' do + vm = subject.get_vm(pool_name, vm_name) + expect(vm['name']).to eq(vm_name) + expect(vm['powerstate']).to eq('PoweredOn') + expect(vm['hostname']).to eq(vm['name']) + end + + it 'should return nil when VM does not exist' do + expect(subject.get_vm(pool_name, 'doesnotexist')).to eq(nil) + end + + context 'using getvm_poweroff_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['getvm_poweroff_percent'] = 0 + end + + it 'will not power off a VM' do + vm = subject.get_vm(pool_name, vm_name) + expect(vm['name']).to eq(vm_name) + expect(vm['powerstate']).to eq('PoweredOn') + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['getvm_poweroff_percent'] = 100 + end + + it 'will power off a VM' do + vm = subject.get_vm(pool_name, vm_name) + expect(vm['name']).to eq(vm_name) + expect(vm['powerstate']).to eq('PoweredOff') + end + end + end + + context 'using getvm_rename_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['getvm_rename_percent'] = 0 + end + + it 'will not rename a VM' do + vm = subject.get_vm(pool_name, vm_name) + expect(vm['name']).to eq(vm_name) + expect(vm['hostname']).to eq(vm['name']) + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['getvm_rename_percent'] = 100 + end + + it 'will rename a VM' do + vm = subject.get_vm(pool_name, vm_name) + expect(vm['name']).to eq(vm_name) + expect(vm['hostname']).to_not eq(vm['name']) + end + end + end + end + + describe '#create_vm' do + let(:new_vm_name) { 'newvm' } + + it 'should return a new VM' do + expect(subject.create_vm(pool_name, new_vm_name)['name']).to eq(new_vm_name) + end + + it 'should increase the number of VMs in the pool' do + old_pool_count = subject.vms_in_pool(pool_name).count + + new_vm = subject.create_vm(pool_name, new_vm_name) + + expect(subject.vms_in_pool(pool_name).count).to eq(old_pool_count + 1) + end + + context 'using createvm_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['createvm_fail_percent'] = 0 + end + + it 'should return a new VM' do + expect(subject.create_vm(pool_name, new_vm_name)['name']).to eq(new_vm_name) + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['createvm_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{subject.create_vm(pool_name, new_vm_name)}.to raise_error(/createvm_fail_percent/) + end + + it 'new VM should not exist' do + begin + subject.create_vm(pool_name, new_vm_name) + rescue + end + expect(subject.get_vm(pool_name, new_vm_name)).to eq(nil) + end + end + end + end + + describe '#create_disk' do + let(:disk_size) { 10 } + + it 'should return true when the disk is created' do + expect(subject.create_disk(pool_name, vm_name,disk_size)).to be true + end + + it 'should raise an error when VM does not exist' do + expect{ subject.create_disk(pool_name, 'doesnotexist',disk_size) }.to raise_error(/VM doesnotexist does not exist/) + end + + context 'using createdisk_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['createdisk_fail_percent'] = 0 + end + + it 'should return true when the disk is created' do + expect(subject.create_disk(pool_name, vm_name,disk_size)).to be true + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['createdisk_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{subject.create_disk(pool_name, vm_name,disk_size)}.to raise_error(/createdisk_fail_percent/) + end + end + end + end + + describe '#create_snapshot' do + let(:snapshot_name) { 'newsnapshot' } + + it 'should return true when the snapshot is created' do + expect(subject.create_snapshot(pool_name, vm_name, snapshot_name)).to be true + end + + it 'should raise an error when VM does not exist' do + expect{ subject.create_snapshot(pool_name, 'doesnotexist', snapshot_name) }.to raise_error(/VM doesnotexist does not exist/) + end + + context 'using createsnapshot_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['createsnapshot_fail_percent'] = 0 + end + + it 'should return true when the disk is created' do + expect(subject.create_snapshot(pool_name, vm_name, snapshot_name)).to be true + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['createsnapshot_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{ subject.create_snapshot(pool_name, vm_name, snapshot_name) }.to raise_error(/createsnapshot_fail_percent/) + end + end + end + end + + describe '#revert_snapshot' do + let(:snapshot_name) { 'newsnapshot' } + + before(:each) do + # Create a snapshot + subject.create_snapshot(pool_name, vm_name, snapshot_name) + end + + it 'should return true when the snapshot is reverted' do + expect(subject.revert_snapshot(pool_name, vm_name, snapshot_name)).to be true + end + + it 'should raise an error when VM does not exist' do + expect{ subject.revert_snapshot(pool_name, 'doesnotexist', snapshot_name) }.to raise_error(/VM doesnotexist does not exist/) + end + + it 'should return false when the snapshot does not exist' do + expect(subject.revert_snapshot(pool_name, vm_name, 'doesnotexist')).to be false + end + + context 'using revertsnapshot_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['revertsnapshot_fail_percent'] = 0 + end + + it 'should return true when the snapshot is reverted' do + expect(subject.revert_snapshot(pool_name, vm_name, snapshot_name)).to be true + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['revertsnapshot_fail_percent'] = 100 + end + + it 'should raise an error when VM does not exist' do + expect{ subject.revert_snapshot(pool_name, vm_name, snapshot_name) }.to raise_error(/revertsnapshot_fail_percent/) + end + end + end + end + + describe '#destroy_vm' do + it 'should return true when destroyed' do + expect(subject.destroy_vm(pool_name, vm_name)).to eq(true) + end + + it 'should log if the VM is powered off' do + allow(logger).to receive(:log) + expect(logger).to receive(:log).with('d', "[ ] [pool1] 'vm1' is being shut down") + expect(subject.destroy_vm(pool_name, vm_name)).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.destroy_vm('doesnotexist',vm_name)).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.destroy_vm(other_pool_name, vm_name)).to eq(false) + end + + context 'using destroyvm_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['destroyvm_fail_percent'] = 0 + end + + it 'should return true when destroyed' do + expect(subject.destroy_vm(pool_name, vm_name)).to eq(true) + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['destroyvm_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{subject.destroy_vm(pool_name, vm_name)}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#vm_ready?' do + before(:each) do + # Speed up tests and ignore sleeping + allow(subject).to receive(:sleep) + end + + it 'should return true if ready' do + expect(subject.vm_ready?(pool_name, vm_name)).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.vm_ready?(pool_name, 'doesnotexist')).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.vm_ready?(other_pool_name, vm_name)).to eq(false) + end + + it 'should raise an error if timeout expires' do + expect{subject.vm_ready?(other_pool_name, running_vm_name)}.to raise_error(Timeout::Error) + end + + it 'should return true if VM becomes ready' do + expect(subject.vm_ready?(other_pool_name, notready_vm_name)).to eq(true) + end + + context 'using vmready_fail_percent' do + describe 'of zero' do + before(:each) do + config[:providers][:dummy]['vmready_fail_percent'] = 0 + end + + it 'should return true if VM becomes ready' do + expect(subject.vm_ready?(other_pool_name, notready_vm_name)).to eq(true) + end + end + + describe 'of 100' do + before(:each) do + config[:providers][:dummy]['vmready_fail_percent'] = 100 + end + + it 'should raise an error' do + expect{subject.vm_ready?(other_pool_name, notready_vm_name)}.to raise_error(/vmready_fail_percent/) + end + end + end + end + + describe '#vm_exists?' do + it 'should return true when VM exists' do + expect(subject.vm_exists?(pool_name, vm_name)).to eq(true) + end + + it 'should return true when VM does not exist' do + expect(subject.vm_exists?(pool_name, 'doesnotexist')).to eq(false) + end + end +end diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb index 906f1b3..74ccc66 100644 --- a/spec/unit/providers/vsphere_spec.rb +++ b/spec/unit/providers/vsphere_spec.rb @@ -47,11 +47,14 @@ describe 'Vmpooler::PoolManager::Provider::VSphere' do :config: max_tries: 3 retry_factor: 10 -:vsphere: - server: "vcenter.domain.local" - username: "vcenter_user" - password: "vcenter_password" - insecure: true +:providers: + :vsphere: + server: "vcenter.domain.local" + username: "vcenter_user" + password: "vcenter_password" + insecure: true + # Drop the connection pool timeout way down for spec tests so they fail fast + connection_pool_timeout: 1 :pools: - name: '#{poolname}' alias: [ 'mockpool' ] @@ -66,14 +69,16 @@ EOT ) } - let(:credentials) { config[:vsphere] } - let(:connection_options) {{}} let(:connection) { mock_RbVmomi_VIM_Connection(connection_options) } let(:vmname) { 'vm1' } subject { Vmpooler::PoolManager::Provider::VSphere.new(config, logger, metrics, 'vsphere', provider_options) } + before(:each) do + allow(subject).to receive(:vsphere_connection_ok?).and_return(true) + end + describe '#name' do it 'should be vsphere' do expect(subject.name).to eq('vsphere') @@ -85,7 +90,7 @@ EOT let(:pool_config) { config[:pools][0] } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) end context 'Given a pool folder that is missing' do @@ -94,7 +99,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.vms_in_pool(poolname) end @@ -112,7 +117,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.vms_in_pool(poolname) end @@ -141,7 +146,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.vms_in_pool(poolname) end @@ -156,7 +161,7 @@ EOT describe '#get_vm_host' do before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) expect(subject).to receive(:find_vm).with(vmname,connection).and_return(vm_object) end @@ -164,7 +169,7 @@ EOT let(:vm_object) { nil } it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.get_vm_host(poolname,vmname) end @@ -186,7 +191,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.get_vm_host(poolname,vmname) end @@ -209,7 +214,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.get_vm_host(poolname,vmname) end @@ -224,7 +229,7 @@ EOT let(:vm_object) { nil } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) expect(subject).to receive(:find_vm).with(vmname,connection).and_return(vm_object) end @@ -232,7 +237,7 @@ EOT let(:vm_object) { nil } it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.find_least_used_compatible_host(poolname,vmname) end @@ -251,7 +256,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.find_least_used_compatible_host(poolname,vmname) end @@ -273,7 +278,7 @@ EOT end it 'should get a connection' do - expect(subject).to receive(:get_connection).and_return(connection) + expect(subject).to receive(:connect_to_vsphere).and_return(connection) subject.find_least_used_compatible_host(poolname,vmname) end @@ -294,7 +299,7 @@ EOT before(:each) do config[:pools][0]['clone_target'] = cluster_name - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) allow(subject).to receive(:find_vm).and_return(vm_object) end @@ -390,7 +395,7 @@ EOT describe '#get_vm' do let(:vm_object) { nil } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) expect(subject).to receive(:find_vm).with(vmname,connection).and_return(vm_object) end @@ -511,7 +516,7 @@ EOT let(:new_vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object) end @@ -586,7 +591,7 @@ EOT let(:datastorename) { 'datastore0' } let(:disk_size) { 10 } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) allow(subject).to receive(:find_vm).with(vmname, connection).and_return(vm_object) end @@ -644,7 +649,7 @@ EOT let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) allow(subject).to receive(:find_vm).with(vmname,connection).and_return(vm_object) end @@ -699,7 +704,7 @@ EOT let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname, :snapshot_tree => snapshot_tree }) } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) allow(subject).to receive(:find_vm).with(vmname,connection).and_return(vm_object) end @@ -748,7 +753,7 @@ EOT let(:destroy_task) { mock_RbVmomi_VIM_Task() } before(:each) do - allow(subject).to receive(:get_connection).and_return(connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection) end context 'Given a missing VM name' do @@ -877,53 +882,65 @@ EOT end # vSphere helper methods - describe '#get_connection' do + describe '#ensured_vsphere_connection' do + let(:config) { YAML.load(<<-EOT +--- +:config: +:providers: + :vsphere: + # Drop the connection pool timeout way down for spec tests so they fail fast + connection_pool_timeout: 1 + connection_pool_size: 1 +:pools: +EOT + ) + } + let(:connection1) { mock_RbVmomi_VIM_Connection(connection_options) } + let(:connection2) { mock_RbVmomi_VIM_Connection(connection_options) } + before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) + allow(subject).to receive(:connect_to_vsphere).and_return(connection1) end - context 'when connection is ok' do - it 'should not attempt to reconnect' do - expect(subject).to receive(:connect_to_vsphere).exactly(0).times - - subject.get_connection() + # This is to ensure that the pool_size of 1 is in effect + it 'should return the same connection object when calling the pool multiple times' do + subject.connection_pool.with_metrics do |pool_object| + expect(pool_object[:connection]).to be(connection1) end - - it 'should return a connection' do - result = subject.get_connection() - - expect(result).to be(connection) + subject.connection_pool.with_metrics do |pool_object| + expect(pool_object[:connection]).to be(connection1) + end + subject.connection_pool.with_metrics do |pool_object| + expect(pool_object[:connection]).to be(connection1) end end - context 'when connection has broken' do + context 'when the connection breaks' do before(:each) do - expect(connection.serviceInstance).to receive(:CurrentTime).and_raise(RuntimeError,'MockConnectionError') + # Emulate the connection state being good, then bad, then good again + expect(subject).to receive(:vsphere_connection_ok?).and_return(true, false, true) + expect(subject).to receive(:connect_to_vsphere).and_return(connection1, connection2) end - it 'should not increment the connect.open metric' do - # https://github.com/puppetlabs/vmpooler/issues/195 - expect(metrics).to receive(:increment).with('connect.open').exactly(0).times - allow(subject).to receive(:connect_to_vsphere) + it 'should restore the connection' do + subject.connection_pool.with_metrics do |pool_object| + # This line needs to be added to all instances of the connection_pool allocation + connection = subject.ensured_vsphere_connection(pool_object) - subject.get_connection() - end + expect(connection).to be(connection1) + end - it 'should call connect_to_vsphere to reconnect' do - allow(metrics).to receive(:increment) - expect(subject).to receive(:connect_to_vsphere).with(credentials) + subject.connection_pool.with_metrics do |pool_object| + connection = subject.ensured_vsphere_connection(pool_object) + # The second connection would have failed. This test ensures that a + # new connection object was created. + expect(connection).to be(connection2) + end - subject.get_connection() - end - - it 'should return a new connection' do - new_connection = mock_RbVmomi_VIM_Connection(connection_options) - expect(subject).to receive(:connect_to_vsphere).with(credentials).and_return(new_connection) - - result = subject.get_connection() - - expect(result).to be(new_connection) + subject.connection_pool.with_metrics do |pool_object| + connection = subject.ensured_vsphere_connection(pool_object) + expect(connection).to be(connection2) + end end end end @@ -933,6 +950,8 @@ EOT allow(RbVmomi::VIM).to receive(:connect).and_return(connection) end + let (:credentials) { config[:providers][:vsphere] } + context 'succesful connection' do it 'should use the supplied credentials' do expect(RbVmomi::VIM).to receive(:connect).with({ @@ -941,7 +960,7 @@ EOT :password => credentials['password'], :insecure => credentials['insecure'] }).and_return(connection) - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere end it 'should honor the insecure setting' do @@ -954,11 +973,11 @@ EOT :password => credentials['password'], :insecure => false, }).and_return(connection) - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere end it 'should default to an insecure connection' do - config[:vsphere][:insecure] = nil + config[:providers][:vsphere][:insecure] = nil expect(RbVmomi::VIM).to receive(:connect).with({ :host => credentials['server'], @@ -967,18 +986,18 @@ EOT :insecure => true }).and_return(connection) - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere end it 'should return the connection object' do - result = subject.connect_to_vsphere(credentials) + result = subject.connect_to_vsphere expect(result).to be(connection) end it 'should increment the connect.open counter' do expect(metrics).to receive(:increment).with('connect.open') - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere end end @@ -992,7 +1011,7 @@ EOT end it 'should return the connection object' do - result = subject.connect_to_vsphere(credentials) + result = subject.connect_to_vsphere expect(result).to be(connection) end @@ -1000,7 +1019,7 @@ EOT it 'should increment the connect.fail and then connect.open counter' do expect(metrics).to receive(:increment).with('connect.fail').exactly(1).times expect(metrics).to receive(:increment).with('connect.open').exactly(1).times - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere end end @@ -1011,7 +1030,7 @@ EOT end it 'should raise an error' do - expect{subject.connect_to_vsphere(credentials)}.to raise_error(RuntimeError,'MockError') + expect{subject.connect_to_vsphere}.to raise_error(RuntimeError,'MockError') end it 'should retry the connection attempt config.max_tries times' do @@ -1020,7 +1039,7 @@ EOT begin # Swallow any errors - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere rescue end end @@ -1031,7 +1050,7 @@ EOT begin # Swallow any errors - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere rescue end end @@ -1051,7 +1070,7 @@ EOT begin # Swallow any errors - subject.connect_to_vsphere(credentials) + subject.connect_to_vsphere rescue end end @@ -2827,30 +2846,4 @@ EOT expect(subject.migrate_vm_host(vm_object,host_object)).to eq('RELOCATE_RESULT') end end - - describe '#close' do - context 'no connection has been made' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",nil) - end - - it 'should not error' do - pending('https://github.com/puppetlabs/vmpooler/issues/211') - subject.close - end - end - - context 'on an open connection' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should close the underlying connection object' do - expect(connection).to receive(:close) - subject.close - end - end - end end diff --git a/spec/unit/vsphere_helper_spec.rb b/spec/unit/vsphere_helper_spec.rb deleted file mode 100644 index 18a8915..0000000 --- a/spec/unit/vsphere_helper_spec.rb +++ /dev/null @@ -1,2104 +0,0 @@ -require 'spec_helper' - -RSpec::Matchers.define :relocation_spec_with_host do |value| - match { |actual| actual[:spec].host == value } -end - -RSpec::Matchers.define :create_virtual_disk_with_size do |value| - match { |actual| actual[:spec].capacityKb == value * 1024 * 1024 } -end - -describe 'Vmpooler::VsphereHelper' do - let(:metrics) { Vmpooler::DummyStatsd.new } - let(:config) { YAML.load(<<-EOT ---- -:config: - max_tries: 3 - retry_factor: 10 -:vsphere: - server: "vcenter.domain.local" - username: "vcenter_user" - password: "vcenter_password" - insecure: true -EOT - ) - } - subject { Vmpooler::VsphereHelper.new(config, metrics) } - - let(:credentials) { config[:vsphere] } - - let(:connection_options) {{}} - let(:connection) { mock_RbVmomi_VIM_Connection(connection_options) } - let(:vmname) { 'vm1' } - - describe '#ensure_connected' do - context 'when connection has ok' do - it 'should not attempt to reconnect' do - expect(subject).to receive(:connect_to_vsphere).exactly(0).times - - subject.ensure_connected(connection,credentials) - end - end - - context 'when connection has broken' do - before(:each) do - expect(connection.serviceInstance).to receive(:CurrentTime).and_raise(RuntimeError,'MockConnectionError') - end - - it 'should not increment the connect.open metric' do - # https://github.com/puppetlabs/vmpooler/issues/195 - expect(metrics).to receive(:increment).with('connect.open').exactly(0).times - allow(subject).to receive(:connect_to_vsphere) - - subject.ensure_connected(connection,credentials) - end - - it 'should call connect_to_vsphere to reconnect' do - allow(metrics).to receive(:increment) - allow(subject).to receive(:connect_to_vsphere).with(credentials) - - subject.ensure_connected(connection,credentials) - end - end - end - - describe '#connect_to_vsphere' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",nil) - - allow(RbVmomi::VIM).to receive(:connect).and_return(connection) - end - - context 'succesful connection' do - it 'should use the supplied credentials' do - expect(RbVmomi::VIM).to receive(:connect).with({ - :host => credentials['server'], - :user => credentials['username'], - :password => credentials['password'], - :insecure => credentials['insecure'] - }).and_return(connection) - subject.connect_to_vsphere(credentials) - end - - it 'should honor the insecure setting' do - pending('Resolution of issue https://github.com/puppetlabs/vmpooler/issues/207') - config[:vsphere][:insecure] = false - - expect(RbVmomi::VIM).to receive(:connect).with({ - :host => credentials['server'], - :user => credentials['username'], - :password => credentials['password'], - :insecure => false, - }).and_return(connection) - subject.connect_to_vsphere(credentials) - end - - it 'should default to an insecure connection' do - config[:vsphere][:insecure] = nil - - expect(RbVmomi::VIM).to receive(:connect).with({ - :host => credentials['server'], - :user => credentials['username'], - :password => credentials['password'], - :insecure => true - }).and_return(connection) - - subject.connect_to_vsphere(credentials) - end - - it 'should set the instance level connection object' do - # NOTE - Using instance_variable_get is a code smell of code that is not testable - expect(subject.instance_variable_get("@connection")).to be_nil - subject.connect_to_vsphere(credentials) - expect(subject.instance_variable_get("@connection")).to be(connection) - end - - it 'should increment the connect.open counter' do - expect(metrics).to receive(:increment).with('connect.open') - subject.connect_to_vsphere(credentials) - end - end - - context 'connection is initially unsuccessful' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",nil) - - # Simulate a failure and then success - expect(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError').ordered - expect(RbVmomi::VIM).to receive(:connect).and_return(connection).ordered - - allow(subject).to receive(:sleep) - end - - it 'should set the instance level connection object' do - # NOTE - Using instance_variable_get is a code smell of code that is not testable - expect(subject.instance_variable_get("@connection")).to be_nil - subject.connect_to_vsphere(credentials) - expect(subject.instance_variable_get("@connection")).to be(connection) - end - - it 'should increment the connect.fail and then connect.open counter' do - expect(metrics).to receive(:increment).with('connect.fail').exactly(1).times - expect(metrics).to receive(:increment).with('connect.open').exactly(1).times - subject.connect_to_vsphere(credentials) - end - end - - context 'connection is always unsuccessful' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",nil) - - allow(RbVmomi::VIM).to receive(:connect).and_raise(RuntimeError,'MockError') - allow(subject).to receive(:sleep) - end - - it 'should raise an error' do - expect{subject.connect_to_vsphere(credentials)}.to raise_error(RuntimeError,'MockError') - end - - it 'should retry the connection attempt config.max_tries times' do - pending('Resolution of issue https://github.com/puppetlabs/vmpooler/issues/199') - expect(RbVmomi::VIM).to receive(:connect).exactly(config[:config]['max_tries']).times.and_raise(RuntimeError,'MockError') - - begin - # Swallow any errors - subject.connect_to_vsphere(credentials) - rescue - end - end - - it 'should increment the connect.fail counter config.max_tries times' do - pending('Resolution of issue https://github.com/puppetlabs/vmpooler/issues/199') - expect(metrics).to receive(:increment).with('connect.fail').exactly(config[:config]['max_tries']).times - - begin - # Swallow any errors - subject.connect_to_vsphere(credentials) - rescue - end - end - - [{:max_tries => 5, :retry_factor => 1}, - {:max_tries => 8, :retry_factor => 5}, - ].each do |testcase| - context "Configuration set for max_tries of #{testcase[:max_tries]} and retry_facter of #{testcase[:retry_factor]}" do - it "should sleep #{testcase[:max_tries] - 1} times between attempts with increasing timeout" do - pending('Resolution of issue https://github.com/puppetlabs/vmpooler/issues/199') - config[:config]['max_tries'] = testcase[:max_tries] - config[:config]['retry_factor'] = testcase[:retry_factor] - - (1..testcase[:max_tries] - 1).each do |try| - expect(subject).to receive(:sleep).with(testcase[:retry_factor] * try).ordered - end - - begin - # Swallow any errors - subject.connect_to_vsphere(credentials) - rescue - end - end - end - end - end - end - - describe '#add_disk' do - let(:datastorename) { 'datastore' } - let(:disk_size) { 30 } - let(:collectMultiple_response) { {} } - - let(:vm_scsi_controller) { mock_RbVmomi_VIM_VirtualSCSIController() } - - # Require at least one SCSI Controller - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine({ - :name => vmname, - }) - mock_vm.config.hardware.device << vm_scsi_controller - - mock_vm - } - - # Require at least one DC with the requried datastore - let(:connection_options) {{ - :serviceContent => { - :datacenters => [ - { :name => 'MockDC', :datastores => [datastorename] } - ] - } - }} - - let(:create_virtual_disk_task) { mock_RbVmomi_VIM_Task() } - let(:reconfig_vm_task) { mock_RbVmomi_VIM_Task() } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - # NOTE - This method should not be using `_connection`, instead it should be using `@conection` - # This should not be required once https://github.com/puppetlabs/vmpooler/issues/213 is resolved - mock_ds = subject.find_datastore(datastorename) - allow(mock_ds).to receive(:_connection).and_return(connection) unless mock_ds.nil? - - # Mocking for find_vmdks - allow(connection.serviceContent.propertyCollector).to receive(:collectMultiple).and_return(collectMultiple_response) - - # Mocking for creating the disk - allow(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task).and_return(create_virtual_disk_task) - allow(create_virtual_disk_task).to receive(:wait_for_completion).and_return(true) - - # Mocking for adding disk to the VM - allow(vm_object).to receive(:ReconfigVM_Task).and_return(reconfig_vm_task) - allow(reconfig_vm_task).to receive(:wait_for_completion).and_return(true) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected).at_least(:once) - - subject.add_disk(vm_object,disk_size,datastorename) - end - - context 'Succesfully addding disk' do - it 'should return true' do - expect(subject.add_disk(vm_object,disk_size,datastorename)).to be true - end - - it 'should request a disk of appropriate size' do - expect(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task) - .with(create_virtual_disk_with_size(disk_size)) - .and_return(create_virtual_disk_task) - - - subject.add_disk(vm_object,disk_size,datastorename) - end - end - - context 'Requested disk size is 0' do - it 'should raise an error' do - expect(subject.add_disk(vm_object,0,datastorename)).to be false - end - end - - context 'No datastores or datastore missing' do - let(:connection_options) {{ - :serviceContent => { - :datacenters => [ - { :name => 'MockDC', :datastores => ['missing_datastore'] } - ] - } - }} - - it 'should return false' do - expect{ subject.add_disk(vm_object,disk_size,datastorename) }.to raise_error(NoMethodError) - end - end - - context 'VM does not have a SCSI Controller' do - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine({ - :name => vmname, - }) - - mock_vm - } - - it 'should raise an error' do - expect{ subject.add_disk(vm_object,disk_size,datastorename) }.to raise_error(NoMethodError) - end - end - end - - describe '#find_datastore' do - let(:datastorename) { 'datastore' } - let(:datastore_list) { [] } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - context 'No datastores in the datacenter' do - let(:connection_options) {{ - :serviceContent => { - :datacenters => [ - { :name => 'MockDC', :datastores => [] } - ] - } - }} - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_datastore(datastorename) - end - - it 'should return nil if the datastore is not found' do - result = subject.find_datastore(datastorename) - expect(result).to be_nil - end - end - - context 'Many datastores in the datacenter' do - let(:connection_options) {{ - :serviceContent => { - :datacenters => [ - { :name => 'MockDC', :datastores => ['ds1','ds2',datastorename,'ds3'] } - ] - } - }} - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_datastore(datastorename) - end - - it 'should return nil if the datastore is not found' do - result = subject.find_datastore('missing_datastore') - expect(result).to be_nil - end - - it 'should find the datastore in the datacenter' do - result = subject.find_datastore(datastorename) - - expect(result).to_not be_nil - expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true - expect(result.name).to eq(datastorename) - end - end - end - - describe '#find_device' do - let(:devicename) { 'device1' } - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine() - mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - mock_vm.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'}) - - mock_vm - } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_device(vm_object,devicename) - end - - it 'should return a device if the device name matches' do - result = subject.find_device(vm_object,devicename) - - expect(result.deviceInfo.label).to eq(devicename) - end - - it 'should return nil if the device name does not match' do - result = subject.find_device(vm_object,'missing_device') - - expect(result).to be_nil - end - end - - describe '#find_disk_controller' do - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine() - - mock_vm - } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should ensure the connection' do - # TODO There's no reason for this as the connection is not used in this method - expect(subject).to receive(:ensure_connected).at_least(:once) - - result = subject.find_disk_controller(vm_object) - end - - it 'should return nil when there are no devices' do - result = subject.find_disk_controller(vm_object) - - expect(result).to be_nil - end - - [0,1,14].each do |testcase| - it "should return a device for a single VirtualSCSIController with #{testcase} attached disks" do - mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController() - vm_object.config.hardware.device << mock_scsi - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'}) - - # Add the disks - (1..testcase).each do - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key }) - end - - result = subject.find_disk_controller(vm_object) - - expect(result).to eq(mock_scsi) - end - end - - [15].each do |testcase| - it "should return nil for a single VirtualSCSIController with #{testcase} attached disks" do - mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController() - vm_object.config.hardware.device << mock_scsi - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'}) - - # Add the disks - (1..testcase).each do - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualDisk({ :controllerKey => mock_scsi.key }) - end - - result = subject.find_disk_controller(vm_object) - - expect(result).to be_nil - end - end - - it 'should raise if a VirtualDisk is missing a controller' do - # Note - Typically this is not possible as a VirtualDisk requires a controller (SCSI, PVSCSI or IDE) - mock_scsi = mock_RbVmomi_VIM_VirtualDisk() - vm_object.config.hardware.device << mock_scsi - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'}) - - expect{subject.find_disk_controller(vm_object)}.to raise_error(NoMethodError) - end - end - - describe '#find_disk_devices' do - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine() - - mock_vm - } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should ensure the connection' do - # TODO There's no reason for this as the connection is not used in this method - expect(subject).to receive(:ensure_connected) - - result = subject.find_disk_devices(vm_object) - end - - it 'should return empty hash when there are no devices' do - result = subject.find_disk_devices(vm_object) - - expect(result).to eq({}) - end - - it 'should return empty hash when there are no VirtualSCSIController or VirtualDisk devices' do - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device2'}) - - result = subject.find_disk_devices(vm_object) - - expect(result).to eq({}) - end - - it 'should return a device for a VirtualSCSIController device with no children' do - mock_scsi = mock_RbVmomi_VIM_VirtualSCSIController() - vm_object.config.hardware.device << mock_scsi - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - - result = subject.find_disk_devices(vm_object) - - expect(result.count).to eq(1) - expect(result[mock_scsi.key]).to_not be_nil - expect(result[mock_scsi.key]['children']).to eq([]) - expect(result[mock_scsi.key]['device']).to eq(mock_scsi) - end - - it 'should return a device for a VirtualDisk device' do - mock_disk = mock_RbVmomi_VIM_VirtualDisk() - vm_object.config.hardware.device << mock_disk - vm_object.config.hardware.device << mock_RbVmomi_VIM_VirtualMachineDevice({:label => 'device1'}) - - result = subject.find_disk_devices(vm_object) - - expect(result.count).to eq(1) - expect(result[mock_disk.controllerKey]).to_not be_nil - expect(result[mock_disk.controllerKey]['children'][0]).to eq(mock_disk) - end - - it 'should return one device for many VirtualDisk devices on the same controller' do - controller1Key = rand(2000) - controller2Key = controller1Key + 1 - mock_disk1 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key}) - mock_disk2 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller1Key}) - mock_disk3 = mock_RbVmomi_VIM_VirtualDisk({:controllerKey => controller2Key}) - - vm_object.config.hardware.device << mock_disk2 - vm_object.config.hardware.device << mock_disk1 - vm_object.config.hardware.device << mock_disk3 - - result = subject.find_disk_devices(vm_object) - - expect(result.count).to eq(2) - - expect(result[controller1Key]).to_not be_nil - expect(result[controller2Key]).to_not be_nil - - expect(result[controller1Key]['children']).to contain_exactly(mock_disk1,mock_disk2) - expect(result[controller2Key]['children']).to contain_exactly(mock_disk3) - end - end - - describe '#find_disk_unit_number' do - let(:vm_object) { - mock_vm = mock_RbVmomi_VIM_VirtualMachine() - - mock_vm - } - let(:controller) { mock_RbVmomi_VIM_VirtualSCSIController() } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should ensure the connection' do - # TODO There's no reason for this as the connection is not used in this method - expect(subject).to receive(:ensure_connected).at_least(:once) - - result = subject.find_disk_unit_number(vm_object,controller) - end - - it 'should return 0 when there are no devices' do - result = subject.find_disk_unit_number(vm_object,controller) - - expect(result).to eq(0) - end - - context 'with a single SCSI Controller' do - before(:each) do - vm_object.config.hardware.device << controller - end - - it 'should return 1 when the host bus controller is at 0' do - controller.scsiCtlrUnitNumber = 0 - - result = subject.find_disk_unit_number(vm_object,controller) - - expect(result).to eq(1) - end - - it 'should return the next lowest id when disks are attached' do - expected_id = 9 - controller.scsiCtlrUnitNumber = 0 - - (1..expected_id-1).each do |disk_id| - mock_disk = mock_RbVmomi_VIM_VirtualDisk({ - :controllerKey => controller.key, - :unitNumber => disk_id, - }) - vm_object.config.hardware.device << mock_disk - end - result = subject.find_disk_unit_number(vm_object,controller) - - expect(result).to eq(expected_id) - end - - it 'should return nil when there are no spare units' do - controller.scsiCtlrUnitNumber = 0 - - (1..15).each do |disk_id| - mock_disk = mock_RbVmomi_VIM_VirtualDisk({ - :controllerKey => controller.key, - :unitNumber => disk_id, - }) - vm_object.config.hardware.device << mock_disk - end - result = subject.find_disk_unit_number(vm_object,controller) - - expect(result).to eq(nil) - end - end - end - - describe '#find_folder' do - let(:foldername) { 'folder'} - let(:missing_foldername) { 'missing_folder'} - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object) - end - - context 'with no folder hierarchy' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() } - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_folder(foldername) - end - - it 'should return nil if the folder is not found' do - expect(subject.find_folder(missing_foldername)).to be_nil - end - end - - context 'with a single layer folder hierarchy' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :vmfolder_tree => { - 'folder1' => nil, - 'folder2' => nil, - foldername => nil, - 'folder3' => nil, - } - }) } - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_folder(foldername) - end - - it 'should return the folder when found' do - result = subject.find_folder(foldername) - expect(result).to_not be_nil - expect(result.name).to eq(foldername) - end - - it 'should return nil if the folder is not found' do - expect(subject.find_folder(missing_foldername)).to be_nil - end - end - - context 'with a VM with the same name as a folder in a single layer folder hierarchy' do - # The folder hierarchy should include a VM with same name as folder, and appear BEFORE the - # folder in the child list. - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :vmfolder_tree => { - 'folder1' => nil, - 'vm1' => { :object_type => 'vm', :name => foldername }, - foldername => nil, - 'folder3' => nil, - } - }) } - - it 'should not return a VM' do - pending('https://github.com/puppetlabs/vmpooler/issues/204') - result = subject.find_folder(foldername) - expect(result).to_not be_nil - expect(result.name).to eq(foldername) - expect(result.is_a? RbVmomi::VIM::VirtualMachine).to be false - end - end - - context 'with a multi layer folder hierarchy' do - let(:end_folder_name) { 'folder'} - let(:foldername) { 'folder2/folder4/' + end_folder_name} - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :vmfolder_tree => { - 'folder1' => nil, - 'folder2' => { - :children => { - 'folder3' => nil, - 'folder4' => { - :children => { - end_folder_name => nil, - }, - } - }, - }, - 'folder5' => nil, - } - }) } - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_folder(foldername) - end - - it 'should return the folder when found' do - result = subject.find_folder(foldername) - expect(result).to_not be_nil - expect(result.name).to eq(end_folder_name) - end - - it 'should return nil if the folder is not found' do - expect(subject.find_folder(missing_foldername)).to be_nil - end - end - - context 'with a VM with the same name as a folder in a multi layer folder hierarchy' do - # The folder hierarchy should include a VM with same name as folder mid-hierarchy (i.e. not at the end level) - # and appear BEFORE the folder in the child list. - let(:end_folder_name) { 'folder'} - let(:foldername) { 'folder2/folder4/' + end_folder_name} - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :vmfolder_tree => { - 'folder1' => nil, - 'folder2' => { - :children => { - 'folder3' => nil, - 'vm1' => { :object_type => 'vm', :name => 'folder4' }, - 'folder4' => { - :children => { - end_folder_name => nil, - }, - } - }, - }, - 'folder5' => nil, - } - }) } - - it 'should not return a VM' do - pending('https://github.com/puppetlabs/vmpooler/issues/204') - result = subject.find_folder(foldername) - expect(result).to_not be_nil - expect(result.name).to eq(foldername) - expect(result.is_a? RbVmomi::VIM::VirtualMachine).to be false - end - end - end - - describe '#get_host_utilization' do - let(:cpu_model) { 'vendor line type sku v4 speed' } - let(:model) { 'v4' } - let(:different_model) { 'different_model' } - let(:limit) { 80 } - let(:default_limit) { 90 } - - context "host with a different model" do - let(:host) { mock_RbVmomi_VIM_HostSystem() } - it 'should return nil' do - expect(subject.get_host_utilization(host,different_model,limit)).to be_nil - end - end - - context "host in maintenance mode" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :maintenance_mode => true, - }) - } - it 'should return nil' do - host.runtime.inMaintenanceMode = true - - expect(subject.get_host_utilization(host,model,limit)).to be_nil - end - end - - context "host with status of not green" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_status => 'purple_alert', - }) - } - it 'should return nil' do - expect(subject.get_host_utilization(host,model,limit)).to be_nil - end - end - - # CPU utilization - context "host which exceeds limit in CPU utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => 100, - :overall_memory_usage => 1, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should return nil' do - expect(subject.get_host_utilization(host,model,limit)).to be_nil - end - end - - context "host which exceeds default limit in CPU utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => default_limit + 1.0, - :overall_memory_usage => 1, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should return nil' do - expect(subject.get_host_utilization(host,model)).to be_nil - end - end - - context "host which does not exceed default limit in CPU utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => default_limit, - :overall_memory_usage => 1, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should not return nil' do - expect(subject.get_host_utilization(host,model)).to_not be_nil - end - end - - # Memory utilization - context "host which exceeds limit in Memory utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => 1, - :overall_memory_usage => 100, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should return nil' do - # Set the Memory Usage to 100% - expect(subject.get_host_utilization(host,model,limit)).to be_nil - end - end - - context "host which exceeds default limit in Memory utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => 1, - :overall_memory_usage => default_limit + 1.0, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should return nil' do - expect(subject.get_host_utilization(host,model)).to be_nil - end - end - - context "host which does not exceed default limit in Memory utilization" do - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => 1, - :overall_memory_usage => default_limit, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should not return nil' do - expect(subject.get_host_utilization(host,model)).to_not be_nil - end - end - - context "host which does not exceed limits" do - # Set CPU to 10% - # Set Memory to 20% - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :overall_cpu_usage => 10, - :overall_memory_usage => 20, - :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }) - } - it 'should return the sum of CPU and Memory utilization' do - expect(subject.get_host_utilization(host,model,limit)[0]).to eq(10 + 20) - end - - it 'should return the host' do - expect(subject.get_host_utilization(host,model,limit)[1]).to eq(host) - end - end - end - - describe '#host_has_cpu_model?' do - let(:cpu_model) { 'vendor line type sku v4 speed' } - let(:model) { 'v4' } - let(:different_model) { 'different_model' } - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :cpu_model => cpu_model, - }) - } - - it 'should return true if the model matches' do - expect(subject.host_has_cpu_model?(host,model)).to eq(true) - end - - it 'should return false if the model is different' do - expect(subject.host_has_cpu_model?(host,different_model)).to eq(false) - end - end - - describe '#get_host_cpu_arch_version' do - let(:cpu_model) { 'vendor line type sku v4 speed' } - let(:model) { 'v4' } - let(:different_model) { 'different_model' } - let(:host) { mock_RbVmomi_VIM_HostSystem({ - :cpu_model => cpu_model, - :num_cpu => 2, - }) - } - - it 'should return the fifth element in the string delimited by spaces' do - expect(subject.get_host_cpu_arch_version(host)).to eq(model) - end - - it 'should use the description of the first CPU' do - host.hardware.cpuPkg[0].description = 'vendor line type sku v6 speed' - expect(subject.get_host_cpu_arch_version(host)).to eq('v6') - end - end - - describe '#cpu_utilization_for' do - [{ :cpu_usage => 10.0, - :core_speed => 10.0, - :num_cores => 2, - :expected_value => 50.0, - }, - { :cpu_usage => 10.0, - :core_speed => 10.0, - :num_cores => 4, - :expected_value => 25.0, - }, - { :cpu_usage => 14.0, - :core_speed => 12.0, - :num_cores => 5, - :expected_value => 23.0 + 1.0/3.0, - }, - ].each do |testcase| - context "CPU Usage of #{testcase[:cpu_usage]}MHz with #{testcase[:num_cores]} x #{testcase[:core_speed]}MHz cores" do - it "should be #{testcase[:expected_value]}%" do - host = mock_RbVmomi_VIM_HostSystem({ - :num_cores_per_cpu => testcase[:num_cores], - :cpu_speed => testcase[:core_speed], - :overall_cpu_usage => testcase[:cpu_usage], - }) - - expect(subject.cpu_utilization_for(host)).to eq(testcase[:expected_value]) - end - end - end - end - - describe '#memory_utilization_for' do - [{ :memory_usage_gigbytes => 10.0, - :memory_size_bytes => 10.0 * 1024 * 1024, - :expected_value => 100.0, - }, - { :memory_usage_gigbytes => 15.0, - :memory_size_bytes => 25.0 * 1024 * 1024, - :expected_value => 60.0, - }, - { :memory_usage_gigbytes => 9.0, - :memory_size_bytes => 31.0 * 1024 * 1024, - :expected_value => 29.03225806451613, - }, - ].each do |testcase| - context "Memory Usage of #{testcase[:memory_usage_gigbytes]}GBytes with #{testcase[:memory_size_bytes]}Bytes of total memory" do - it "should be #{testcase[:expected_value]}%" do - host = mock_RbVmomi_VIM_HostSystem({ - :memory_size => testcase[:memory_size_bytes], - :overall_memory_usage => testcase[:memory_usage_gigbytes], - }) - - expect(subject.memory_utilization_for(host)).to eq(testcase[:expected_value]) - end - end - end - end - - describe '#find_least_used_host' do - let(:cluster_name) { 'cluster' } - let(:missing_cluster_name) { 'missing_cluster' } - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - # This mocking is a little fragile but hard to do without a real vCenter instance - allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object) - datacenter_object.hostFolder.childEntity = [cluster_object] - end - - context 'missing cluster' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [{ - :name => cluster_name, - }]})} - let(:expected_host) { cluster_object.host[0] } - - it 'should raise an error' do - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError,/undefined method/) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError) - end - end - - context 'standalone host within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [{ - :name => cluster_name, - }]})} - let(:expected_host) { cluster_object.host[0] } - - it 'should return the standalone host' do - result = subject.find_least_used_host(cluster_name) - - expect(result).to be(expected_host) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_host(cluster_name) - end - end - - context 'standalone host outside the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [{ - :name => cluster_name, - :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024, - }]})} - let(:expected_host) { cluster_object.host[0] } - - it 'should raise an error' do - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError,/undefined method/) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError) - end - end - - context 'cluster of 3 hosts within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [ - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - it 'should return the standalone host' do - result = subject.find_least_used_host(cluster_name) - - expect(result).to be(expected_host) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_host(cluster_name) - end - end - - context 'cluster of 3 hosts all outside of the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [ - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - it 'should raise an error' do - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError,/undefined method/) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - expect{subject.find_least_used_host(missing_cluster_name)}.to raise_error(NoMethodError) - end - end - - context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [ - { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - it 'should return the standalone host' do - result = subject.find_least_used_host(cluster_name) - - expect(result).to be(expected_host) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_host(cluster_name) - end - end - - context 'cluster of 3 hosts all outside of the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({ - :name => cluster_name, - :hosts => [ - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - it 'should return a host' do - pending('https://github.com/puppetlabs/vmpooler/issues/206') - result = subject.find_least_used_host(missing_cluster_name) - expect(result).to_not be_nil - end - - it 'should ensure the connection' do - pending('https://github.com/puppetlabs/vmpooler/issues/206') - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_host(cluster_name) - end - end - end - - describe '#find_cluster' do - let(:cluster) {'cluster'} - let(:missing_cluster) {'missing_cluster'} - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object) - end - - context 'no clusters in the datacenter' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() } - - before(:each) do - end - - it 'should return nil if the cluster is not found' do - expect(subject.find_cluster(missing_cluster)).to be_nil - end - end - - context 'with a single layer folder hierarchy' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :hostfolder_tree => { - 'cluster1' => {:object_type => 'compute_resource'}, - 'cluster2' => {:object_type => 'compute_resource'}, - cluster => {:object_type => 'compute_resource'}, - 'cluster3' => {:object_type => 'compute_resource'}, - } - }) } - - it 'should return the cluster when found' do - result = subject.find_cluster(cluster) - - expect(result).to_not be_nil - expect(result.name).to eq(cluster) - end - - it 'should return nil if the cluster is not found' do - expect(subject.find_cluster(missing_cluster)).to be_nil - end - end - - context 'with a multi layer folder hierarchy' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ - :hostfolder_tree => { - 'cluster1' => {:object_type => 'compute_resource'}, - 'folder2' => { - :children => { - cluster => {:object_type => 'compute_resource'}, - } - }, - 'cluster3' => {:object_type => 'compute_resource'}, - } - }) } - - it 'should return the cluster when found' do - pending('https://github.com/puppetlabs/vmpooler/issues/205') - result = subject.find_cluster(cluster) - - expect(result).to_not be_nil - expect(result.name).to eq(cluster) - end - - it 'should return nil if the cluster is not found' do - expect(subject.find_cluster(missing_cluster)).to be_nil - end - end - end - - describe '#get_cluster_host_utilization' do - context 'standalone host within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) } - - it 'should return array with one element' do - result = subject.get_cluster_host_utilization(cluster_object) - expect(result).to_not be_nil - expect(result.count).to eq(1) - end - end - - context 'standalone host which is out the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - - it 'should return array with 0 elements' do - result = subject.get_cluster_host_utilization(cluster_object) - expect(result).to_not be_nil - expect(result.count).to eq(0) - end - end - - context 'cluster with 3 hosts within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - - it 'should return array with 3 elements' do - result = subject.get_cluster_host_utilization(cluster_object) - expect(result).to_not be_nil - expect(result.count).to eq(3) - end - end - - context 'cluster with 5 hosts of which 3 within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - - it 'should return array with 3 elements' do - result = subject.get_cluster_host_utilization(cluster_object) - expect(result).to_not be_nil - expect(result.count).to eq(3) - end - end - - context 'cluster with 3 hosts of which none are within the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - - it 'should return array with 0 elements' do - result = subject.get_cluster_host_utilization(cluster_object) - expect(result).to_not be_nil - expect(result.count).to eq(0) - end - end - end - - describe '#find_least_used_compatible_host' do - let(:vm) { mock_RbVmomi_VIM_VirtualMachine() } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - context 'standalone host within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [{}]}) } - let(:standalone_host) { cluster_object.host[0] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = standalone_host - end - - it 'should return the standalone host' do - result = subject.find_least_used_compatible_host(vm) - - expect(result).to_not be_nil - expect(result[0]).to be(standalone_host) - expect(result[1]).to eq(standalone_host.name) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_compatible_host(vm) - end - end - - context 'standalone host outside of limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:standalone_host) { cluster_object.host[0] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = standalone_host - end - - it 'should raise error' do - expect{subject.find_least_used_compatible_host(vm)}.to raise_error(NoMethodError,/undefined method/) - end - end - - context 'cluster of 3 hosts within limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = expected_host - end - - it 'should return the least used host' do - result = subject.find_least_used_compatible_host(vm) - - expect(result).to_not be_nil - expect(result[0]).to be(expected_host) - expect(result[1]).to eq(expected_host.name) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_compatible_host(vm) - end - end - - context 'cluster of 3 hosts all outside of the limits' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = expected_host - end - - it 'should raise error' do - expect{subject.find_least_used_compatible_host(vm)}.to raise_error(NoMethodError,/undefined method/) - end - end - - context 'cluster of 5 hosts of which one is out of limits and one has wrong CPU type' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 31, :overall_memory_usage => 31, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :cpu_model => 'different cpu model', :overall_cpu_usage => 1, :overall_memory_usage => 1, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 11, :overall_memory_usage => 11, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 100, :overall_memory_usage => 100, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 21, :overall_memory_usage => 21, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[2] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = expected_host - end - - it 'should return the least used host' do - result = subject.find_least_used_compatible_host(vm) - - expect(result).to_not be_nil - expect(result[0]).to be(expected_host) - expect(result[1]).to eq(expected_host.name) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_compatible_host(vm) - end - end - - context 'cluster of 3 hosts all with the same utilisation' do - let(:cluster_object) { mock_RbVmomi_VIM_ComputeResource({:hosts => [ - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - { :overall_cpu_usage => 10, :overall_memory_usage => 10, :cpu_speed => 100, :num_cores_per_cpu => 1, :num_cpu => 1, :memory_size => 100.0 * 1024 * 1024 }, - ]}) } - let(:expected_host) { cluster_object.host[1] } - - before(:each) do - # This mocking is a little fragile but hard to do without a real vCenter instance - vm.summary.runtime.host = expected_host - end - - it 'should return a host' do - pending('https://github.com/puppetlabs/vmpooler/issues/206 is fixed') - result = subject.find_least_used_compatible_host(vm) - - expect(result).to_not be_nil - end - - it 'should ensure the connection' do - pending('https://github.com/puppetlabs/vmpooler/issues/206 is fixed') - expect(subject).to receive(:ensure_connected) - - result = subject.find_least_used_compatible_host(vm) - end - end - end - - describe '#find_pool' do - let(:poolname) { 'pool'} - let(:missing_poolname) { 'missing_pool'} - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object) - end - - context 'with empty folder hierarchy' do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() } - - it 'should ensure the connection' do - pending('https://github.com/puppetlabs/vmpooler/issues/209') - expect(subject).to receive(:ensure_connected) - - subject.find_pool(poolname) - end - - it 'should return nil if the pool is not found' do - pending('https://github.com/puppetlabs/vmpooler/issues/209') - expect(subject.find_pool(missing_poolname)).to be_nil - end - end - - [ - # Single layer Host folder hierarchy - { - :context => 'single layer folder hierarchy with a resource pool', - :poolpath => 'pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => nil, - 'pool' => {:object_type => 'resource_pool'}, - 'folder3' => nil, - }, - }, - { - :context => 'single layer folder hierarchy with a child resource pool', - :poolpath => 'parentpool/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => nil, - 'parentpool' => {:object_type => 'resource_pool', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - 'folder3' => nil, - }, - }, - { - :context => 'single layer folder hierarchy with a resource pool within a cluster', - :poolpath => 'cluster/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => nil, - 'cluster' => {:object_type => 'cluster_compute_resource', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - 'folder3' => nil, - }, - }, - # Multi layer Host folder hierarchy - { - :context => 'multi layer folder hierarchy with a resource pool', - :poolpath => 'folder2/folder4/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => { :children => { - 'folder3' => nil, - 'folder4' => { :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - }}, - 'folder5' => nil, - }, - }, - { - :context => 'multi layer folder hierarchy with a child resource pool', - :poolpath => 'folder2/folder4/parentpool/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => { :children => { - 'folder3' => nil, - 'folder4' => { :children => { - 'parentpool' => {:object_type => 'resource_pool', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - }}, - }}, - 'folder5' => nil, - }, - }, - { - :context => 'multi layer folder hierarchy with a resource pool within a cluster', - :poolpath => 'folder2/folder4/cluster/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => { :children => { - 'folder3' => nil, - 'folder4' => { :children => { - 'cluster' => {:object_type => 'cluster_compute_resource', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - }}, - }}, - 'folder5' => nil, - }, - }, - ].each do |testcase| - context testcase[:context] do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ :hostfolder_tree => testcase[:hostfolder_tree]}) } - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_pool(testcase[:poolpath]) - end - - it 'should return the pool when found' do - result = subject.find_pool(testcase[:poolpath]) - - expect(result).to_not be_nil - expect(result.name).to eq(testcase[:poolname]) - expect(result.is_a?(RbVmomi::VIM::ResourcePool)).to be true - end - - it 'should return nil if the poolname is not found' do - pending('https://github.com/puppetlabs/vmpooler/issues/209') - expect(subject.find_pool(missing_poolname)).to be_nil - end - end - end - - # Tests for issue https://github.com/puppetlabs/vmpooler/issues/210 - [ - { - :context => 'multi layer folder hierarchy with a resource pool the same name as a folder', - :poolpath => 'folder2/folder4/cluster/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => { :children => { - 'folder3' => nil, - 'bad_pool' => {:object_type => 'resource_pool', :name => 'folder4'}, - 'folder4' => { :children => { - 'cluster' => {:object_type => 'cluster_compute_resource', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - }}, - }}, - 'folder5' => nil, - }, - }, - { - :context => 'multi layer folder hierarchy with a cluster the same name as a folder', - :poolpath => 'folder2/folder4/cluster/pool', - :poolname => 'pool', - :hostfolder_tree => { - 'folder1' => nil, - 'folder2' => { :children => { - 'folder3' => nil, - 'bad_cluster' => {:object_type => 'cluster_compute_resource', :name => 'folder4'}, - 'folder4' => { :children => { - 'cluster' => {:object_type => 'cluster_compute_resource', :children => { - 'pool' => {:object_type => 'resource_pool'}, - }}, - }}, - }}, - 'folder5' => nil, - }, - }, - ].each do |testcase| - context testcase[:context] do - let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ :hostfolder_tree => testcase[:hostfolder_tree]}) } - - it 'should ensure the connection' do - pending('https://github.com/puppetlabs/vmpooler/issues/210') - expect(subject).to receive(:ensure_connected) - - subject.find_pool(testcase[:poolpath]) - end - - it 'should return the pool when found' do - pending('https://github.com/puppetlabs/vmpooler/issues/210') - result = subject.find_pool(testcase[:poolpath]) - - expect(result).to_not be_nil - expect(result.name).to eq(testcase[:poolname]) - expect(result.is_a?(RbVmomi::VIM::ResourcePool)).to be true - end - end - end - end - - describe '#find_snapshot' do - let(:snapshot_name) {'snapshot'} - let(:missing_snapshot_name) {'missing_snapshot'} - let(:vm) { mock_RbVmomi_VIM_VirtualMachine(mock_options) } - let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachine() } - - context 'VM with no snapshots' do - let(:mock_options) {{ :snapshot_tree => nil }} - it 'should return nil' do - expect(subject.find_snapshot(vm,snapshot_name)).to be_nil - end - end - - context 'VM with a single layer of snapshots' do - let(:mock_options) {{ - :snapshot_tree => { - 'snapshot1' => nil, - 'snapshot2' => nil, - 'snapshot3' => nil, - 'snapshot4' => nil, - snapshot_name => { :ref => snapshot_object}, - } - }} - - it 'should return snapshot which matches the name' do - result = subject.find_snapshot(vm,snapshot_name) - expect(result).to be(snapshot_object) - end - - it 'should return nil which no matches are found' do - result = subject.find_snapshot(vm,missing_snapshot_name) - expect(result).to be_nil - end - end - - context 'VM with a nested layers of snapshots' do - let(:mock_options) {{ - :snapshot_tree => { - 'snapshot1' => nil, - 'snapshot2' => nil, - 'snapshot3' => { :children => { - 'snapshot4' => nil, - 'snapshot5' => { :children => { - snapshot_name => { :ref => snapshot_object}, - }}, - }}, - 'snapshot6' => nil, - } - }} - - it 'should return snapshot which matches the name' do - result = subject.find_snapshot(vm,snapshot_name) - expect(result).to be(snapshot_object) - end - - it 'should return nil which no matches are found' do - result = subject.find_snapshot(vm,missing_snapshot_name) - expect(result).to be_nil - end - end - end - - describe '#find_vm' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - allow(subject).to receive(:find_vm_light).and_return('vmlight') - allow(subject).to receive(:find_vm_heavy).and_return( { vmname => 'vmheavy' }) - end - - it 'should ensure the connection' do - # TODO This seems like overkill as we immediately call vm_light and heavy which - # does the same thing. Also the connection isn't actually used in this method - expect(subject).to receive(:ensure_connected) - - subject.find_vm(vmname) - end - - it 'should call find_vm_light' do - expect(subject).to receive(:find_vm_light).and_return('vmlight') - - expect(subject.find_vm(vmname)).to eq('vmlight') - end - - it 'should not call find_vm_heavy if find_vm_light finds the VM' do - expect(subject).to receive(:find_vm_light).and_return('vmlight') - expect(subject).to receive(:find_vm_heavy).exactly(0).times - - expect(subject.find_vm(vmname)).to eq('vmlight') - end - - it 'should call find_vm_heavy when find_vm_light returns nil' do - expect(subject).to receive(:find_vm_light).and_return(nil) - expect(subject).to receive(:find_vm_heavy).and_return( { vmname => 'vmheavy' }) - - expect(subject.find_vm(vmname)).to eq('vmheavy') - end - end - - describe '#find_vm_light' do - let(:missing_vm) { 'missing_vm' } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - allow(connection.searchIndex).to receive(:FindByDnsName).and_return(nil) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected) - - subject.find_vm_light(vmname) - end - - it 'should call FindByDnsName with the correct parameters' do - expect(connection.searchIndex).to receive(:FindByDnsName).with({ - :vmSearch => true, - dnsName: vmname, - }) - - subject.find_vm_light(vmname) - end - - it 'should return the VM object when found' do - vm_object = mock_RbVmomi_VIM_VirtualMachine() - expect(connection.searchIndex).to receive(:FindByDnsName).with({ - :vmSearch => true, - dnsName: vmname, - }).and_return(vm_object) - - expect(subject.find_vm_light(vmname)).to be(vm_object) - end - - it 'should return nil if the VM is not found' do - expect(connection.searchIndex).to receive(:FindByDnsName).with({ - :vmSearch => true, - dnsName: missing_vm, - }).and_return(nil) - - expect(subject.find_vm_light(missing_vm)).to be_nil - end - end - - describe '#find_vm_heavy' do - let(:missing_vm) { 'missing_vm' } - # Return an empty result by default - let(:retrieve_result) {{}} - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - allow(connection.propertyCollector).to receive(:RetrievePropertiesEx).and_return(mock_RbVmomi_VIM_RetrieveResult(retrieve_result)) - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected).at_least(:once) - - subject.find_vm_heavy(vmname) - end - - context 'Search result is empty' do - it 'should return empty hash' do - expect(subject.find_vm_heavy(vmname)).to eq({}) - end - end - - context 'Search result contains VMs but no matches' do - let(:retrieve_result) { - { :response => [ - { 'name' => 'no_match001'}, - { 'name' => 'no_match002'}, - { 'name' => 'no_match003'}, - { 'name' => 'no_match004'}, - ] - } - } - - it 'should return empty hash' do - expect(subject.find_vm_heavy(vmname)).to eq({}) - end - end - - context 'Search contains a single match' do - let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })} - let(:retrieve_result) { - { :response => [ - { 'name' => 'no_match001'}, - { 'name' => 'no_match002'}, - { 'name' => vmname, :object => vm_object }, - { 'name' => 'no_match003'}, - { 'name' => 'no_match004'}, - ] - } - } - - it 'should return single result' do - result = subject.find_vm_heavy(vmname) - expect(result.keys.count).to eq(1) - end - - it 'should return the matching VM Object' do - result = subject.find_vm_heavy(vmname) - expect(result[vmname]).to be(vm_object) - end - end - - context 'Search contains a two matches' do - let(:vm_object1) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })} - let(:vm_object2) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })} - let(:retrieve_result) { - { :response => [ - { 'name' => 'no_match001'}, - { 'name' => 'no_match002'}, - { 'name' => vmname, :object => vm_object1 }, - { 'name' => 'no_match003'}, - { 'name' => 'no_match004'}, - { 'name' => vmname, :object => vm_object2 }, - ] - } - } - - it 'should return one result' do - result = subject.find_vm_heavy(vmname) - expect(result.keys.count).to eq(1) - end - - it 'should return the last matching VM Object' do - result = subject.find_vm_heavy(vmname) - expect(result[vmname]).to be(vm_object2) - end - end - end - - describe '#find_vmdks' do - let(:datastorename) { 'datastore' } - let(:connection_options) {{ - :serviceContent => { - :datacenters => [ - { :name => 'MockDC', :datastores => [datastorename] } - ] - } - }} - - let(:collectMultiple_response) { {} } - - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - - # NOTE - This method should not be using `_connection`, instead it should be using `@conection` - mock_ds = subject.find_datastore(datastorename) - allow(mock_ds).to receive(:_connection).and_return(connection) - allow(connection.serviceContent.propertyCollector).to receive(:collectMultiple).and_return(collectMultiple_response) - end - - it 'should not use _connction to get the underlying connection object' do - pending('https://github.com/puppetlabs/vmpooler/issues/213') - - mock_ds = subject.find_datastore(datastorename) - expect(mock_ds).to receive(:_connection).exactly(0).times - - begin - # ignore all errors. What's important is that it doesn't call _connection - subject.find_vmdks(vmname,datastorename) - rescue - end - end - - it 'should ensure the connection' do - expect(subject).to receive(:ensure_connected).at_least(:once) - - subject.find_vmdks(vmname,datastorename) - end - - context 'Searching all files for all VMs on a Datastore' do - # This is fairly fragile mocking - let(:collectMultiple_response) { { - 'FakeVMObject1' => { 'layoutEx.file' => - [ - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 101, :name => "[#{datastorename}] mock1/mock1_0.vmdk"}) - ]}, - vmname => { 'layoutEx.file' => - [ - # VMDKs which should match - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 1, :name => "[#{datastorename}] #{vmname}/#{vmname}_0.vmdk"}), - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 2, :name => "[#{datastorename}] #{vmname}/#{vmname}_1.vmdk"}), - # VMDKs which should not match - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 102, :name => "[otherdatastore] #{vmname}/#{vmname}_0.vmdk"}), - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 103, :name => "[otherdatastore] #{vmname}/#{vmname}.vmdk"}), - mock_RbVmomi_VIM_VirtualMachineFileLayoutExFileInfo({ :key => 104, :name => "[otherdatastore] #{vmname}/#{vmname}_abc.vmdk"}), - ]}, - } } - - it 'should return empty array if no VMDKs match the VM name' do - expect(subject.find_vmdks('missing_vm_name',datastorename)).to eq([]) - end - - it 'should return matching VMDKs for the VM' do - result = subject.find_vmdks(vmname,datastorename) - expect(result).to_not be_nil - expect(result.count).to eq(2) - # The keys for each VMDK should be less that 100 as per the mocks - result.each do |fileinfo| - expect(fileinfo.key).to be < 100 - end - end - end - end - - describe '#get_base_vm_container_from' do - let(:local_connection) { mock_RbVmomi_VIM_Connection() } - - before(:each) do - allow(subject).to receive(:ensure_connected) - end - - it 'should ensure the connection' do - pending('https://github.com/puppetlabs/vmpooler/issues/212') - expect(subject).to receive(:ensure_connected).with(local_connection,credentials) - - subject.get_base_vm_container_from(local_connection) - end - - it 'should return a recursive view of type VirtualMachine' do - result = subject.get_base_vm_container_from(local_connection) - - expect(result.recursive).to be true - expect(result.type).to eq(['VirtualMachine']) - end - end - - describe '#get_snapshot_list' do - let(:snapshot_name) {'snapshot'} - let(:snapshot_tree) { mock_RbVmomi_VIM_VirtualMachine(mock_options).snapshot.rootSnapshotList } - let(:snapshot_object) { mock_RbVmomi_VIM_VirtualMachine() } - - it 'should raise if the snapshot tree is nil' do - expect{ subject.get_snapshot_list(nil,snapshot_name)}.to raise_error(NoMethodError) - end - - context 'VM with a single layer of snapshots' do - let(:mock_options) {{ - :snapshot_tree => { - 'snapshot1' => nil, - 'snapshot2' => nil, - 'snapshot3' => nil, - 'snapshot4' => nil, - snapshot_name => { :ref => snapshot_object}, - } - }} - - it 'should return snapshot which matches the name' do - result = subject.get_snapshot_list(snapshot_tree,snapshot_name) - expect(result).to be(snapshot_object) - end - end - - context 'VM with a nested layers of snapshots' do - let(:mock_options) {{ - :snapshot_tree => { - 'snapshot1' => nil, - 'snapshot2' => nil, - 'snapshot3' => { :children => { - 'snapshot4' => nil, - 'snapshot5' => { :children => { - snapshot_name => { :ref => snapshot_object}, - }}, - }}, - 'snapshot6' => nil, - } - }} - - it 'should return snapshot which matches the name' do - result = subject.get_snapshot_list(snapshot_tree,snapshot_name) - expect(result).to be(snapshot_object) - end - end - end - - describe '#migrate_vm_host' do - let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname })} - let(:host_object) { mock_RbVmomi_VIM_HostSystem({ :name => 'HOST' })} - let(:relocate_task) { mock_RbVmomi_VIM_Task() } - - before(:each) do - allow(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task) - allow(relocate_task).to receive(:wait_for_completion) - end - - it 'should call RelovateVM_Task' do - expect(vm_object).to receive(:RelocateVM_Task).and_return(relocate_task) - - subject.migrate_vm_host(vm_object,host_object) - end - - it 'should use a Relocation Spec object with correct host' do - expect(vm_object).to receive(:RelocateVM_Task).with(relocation_spec_with_host(host_object)) - - subject.migrate_vm_host(vm_object,host_object) - end - - it 'should wait for the relocation to complete' do - expect(relocate_task).to receive(:wait_for_completion) - - subject.migrate_vm_host(vm_object,host_object) - end - - it 'should return the result of the relocation' do - expect(relocate_task).to receive(:wait_for_completion).and_return('RELOCATE_RESULT') - - expect(subject.migrate_vm_host(vm_object,host_object)).to eq('RELOCATE_RESULT') - end - end - - describe '#close' do - context 'no connection has been made' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",nil) - end - - it 'should not error' do - pending('https://github.com/puppetlabs/vmpooler/issues/211') - subject.close - end - end - - context 'on an open connection' do - before(:each) do - # NOTE - Using instance_variable_set is a code smell of code that is not testable - subject.instance_variable_set("@connection",connection) - end - - it 'should close the underlying connection object' do - expect(connection).to receive(:close) - subject.close - end - end - end -end diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index c128b5d..86fd852 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -1,9 +1,20 @@ --- +:providers: +# :providers: +# +# This section contains the VM providers for VMs and Pools +# The currently supported backing services are: +# - vsphere +# - dummy + # :vsphere: # # This section contains the server hostname and authentication credentials # needed for vmpooler to connect to VMware vSphere. # +# NOTE - To support older configuration files, a :vsphere: configuration section +# will be copied into :providers:/:vsphere: if one does not already exist. +# # Available configuration parameters: # # - server @@ -17,13 +28,109 @@ # - password # The password used to authenticate VMware vSphere. # (required) +# +# - insecure +# Whether to ignore any HTTPS negotiation errors (e.g. untrusted self-signed certificates) +# (optional: default true) +# Example: + + :vsphere: + server: 'vsphere.company.com' + username: 'vmpooler' + password: 'swimsw1msw!m' + +# :dummy: +# +# The dummy backing service is a simple text file service that can be used +# to test vmpooler operations in a development or test environment +# +# Available configuration parameters: +# +# - filename (Optional) +# The filename used to store the backing text file. If this is not specified the VM state is only +# kept in memory, and is lost when the Provider is shutdown +# +# - connection_pool_size (Optional) +# The size of the dummy connection pool. This can be used to simulate constrained provider resources e.g. 200 pools sharing on connection +# (optional; default 1) +# +# - connection_pool_timeout (Optional) +# The number of seconds to wait for a connection object from the pool. If the timeout is exceeded an error is raised +# (optional; default 10 seconds) +# +# - migratevm_couldmove_percent +# Percent chance that a VM could be moved to another host +# (optional; default 0%) +# +# - migratevm_max_time +# Maximum amount of random time a VM migration action will take in seconds +# (optional; default 0 seconds) +# +# - migratevm_fail_percent +# Percent chance that a VM migration action will fail +# (optional; default 0%) +# +# - getvm_poweroff_percent +# Percent chance that when the VM information is gathered that the VM will be powered off +# (optional; default 0%) +# +# - getvm_rename_percent +# Percent chance that when the VM information is gathered that the VM will be renamed +# (optional; default 0%) +# +# - createvm_max_time +# Maximum amount of random time a VM creation action will take, in seconds +# (optional; default 0 seconds) +# +# - createvm_fail_percent +# Percent chance that a VM creation action will fail +# (optional; default 0%) +# +# - createdisk_max_time +# Maximum amount of random time a VM create disk action will take, in seconds +# (optional; default 0 seconds) +# +# - createdisk_fail_percent +# Percent chance that a VM create disk action will fail +# (optional; default 0%) +# +# - createsnapshot_max_time +# Maximum amount of random time a VM create snapshot action will take, in seconds +# (optional; default 0 seconds) +# +# - createsnapshot_fail_percent +# Percent chance that a VM create snapshot action will fail +# (optional; default 0%) +# +# - revertsnapshot_max_time +# Maximum amount of random time a VM revert snapshot action will take, in seconds +# (optional; default 0 seconds) +# +# - revertsnapshot_fail_percent +# Percent chance that a VM revert snapshot action will fail +# (optional; default 0%) +# +# - destroyvm_max_shutdown_time +# Maximum amount of random time a VM shutdown action will take during destroy, in seconds +# (optional; default 0 seconds) +# +# - destroyvm_max_time +# Maximum amount of random time a VM destroy action will take, in seconds +# (optional; default 0 seconds) +# +# - destroyvm_fail_percent +# Percent chance that a VM destroy action will fail +# (optional; default 0%) +# +# - vmready_fail_percent +# Percent chance that an error is raised when vm_ready? is called +# (optional; default 0%) # Example: -:vsphere: - server: 'vsphere.company.com' - username: 'vmpooler' - password: 'swimsw1msw!m' + :dummy: + filename: '/tmp/dummy-backing.yaml' + # :redis: # @@ -54,58 +161,58 @@ server: 'redis.company.com' - # :graphs: - # - # This section contains the server and prefix information for a graphite- - # compatible web front-end where graphs may be viewed. This is used by the - # vmpooler dashboard to retrieve statistics and graphs for a given instance. - # - # NOTE: This is not the endpoint for publishing metrics data. See `graphite:` - # and `statsd:` below. - # - # NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be - # consulted for `server`/`prefix` information to use in locating a - # graph server for our dashboard. `graphs:` is recommended over - # `graphite:` - # - # - # Available configuration parameters: - # - # - # - server - # The FQDN hostname of the statsd daemon. - # (required) - # - # - prefix - # The prefix to use while storing statsd data. - # (optional; default: 'vmpooler') +# :graphs: +# +# This section contains the server and prefix information for a graphite- +# compatible web front-end where graphs may be viewed. This is used by the +# vmpooler dashboard to retrieve statistics and graphs for a given instance. +# +# NOTE: This is not the endpoint for publishing metrics data. See `graphite:` +# and `statsd:` below. +# +# NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be +# consulted for `server`/`prefix` information to use in locating a +# graph server for our dashboard. `graphs:` is recommended over +# `graphite:` +# +# +# Available configuration parameters: +# +# +# - server +# The FQDN hostname of the statsd daemon. +# (required) +# +# - prefix +# The prefix to use while storing statsd data. +# (optional; default: 'vmpooler') - # :statsd: - # - # This section contains the connection information required to store - # historical data via statsd. This is mutually exclusive with graphite - # and takes precedence. - # - # Available configuration parameters: - # - # - server - # The FQDN hostname of the statsd daemon. - # (required) - # - # - prefix - # The prefix to use while storing statsd data. - # (optional; default: 'vmpooler') - # - # - port - # The UDP port to communicate with the statsd daemon. - # (optional; default: 8125) +# :statsd: +# +# This section contains the connection information required to store +# historical data via statsd. This is mutually exclusive with graphite +# and takes precedence. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the statsd daemon. +# (required) +# +# - prefix +# The prefix to use while storing statsd data. +# (optional; default: 'vmpooler') +# +# - port +# The UDP port to communicate with the statsd daemon. +# (optional; default: 8125) - # Example: +# Example: - :statsd: - server: 'statsd.company.com' - prefix: 'vmpooler' - port: 8125 +:statsd: + server: 'statsd.company.com' + prefix: 'vmpooler' + port: 8125 # :graphite: # @@ -217,7 +324,7 @@ # (optional; default: same cluster/host as origin template) # # - task_limit -# The number of concurrent VMware vSphere tasks to perform. +# The number of concurrent VM creation tasks to perform. # (optional; default: '10') # # - timeout @@ -249,21 +356,23 @@ # # - migration_limit # When set to any value greater than 0 enable VM migration at checkout. -# When enabled this capability will evaluate a VM for migration when it is requested +# When enabled this capability will evaluate a VM for migration to a different host when it is requested # in an effort to maintain a more even distribution of load across compute resources. -# The migration_limit ensures that no more than n migrations will be evaluated at any one time +# The migration_limit ensures that no more than the specified migrations will be evaluated at any one time # and greatly reduces the possibilty of VMs ending up bunched together on a particular host. # -# - max_tries -# Set the max number of times a connection should retry in vsphere helper. -# This optional setting allows a user to dial in retry limits to -# suit your environment. +# - max_tries +# Set the max number of times a connection should retry in VM providers. +# This optional setting allows a user to dial in retry limits to +# suit your environment. +# (optional; default: 3) # -# - retry_factor -# When retrying, each attempt sleeps for the try count * retry_factor. -# Increase this number to lengthen the delay between retry attempts. -# This is particularly useful for instances with a large number of pools -# to prevent a thundering herd when retrying connections. +# - retry_factor +# When retrying, each attempt sleeps for the try count * retry_factor. +# Increase this number to lengthen the delay between retry attempts. +# This is particularly useful for instances with a large number of pools +# to prevent a thundering herd when retrying connections. +# (optional; default: 10) # Example: @@ -300,18 +409,15 @@ # The template or virtual machine target to spawn clones from. # (required) # -# - folder -# The vSphere 'folder' destination for spawned clones. -# (required) -# -# - datastore -# The vSphere 'datastore' destination for spawned clones. -# (required) -# # - size # The number of waiting VMs to keep in a pool. # (required) # +# - provider +# The name of the VM provider which manage this pool. This should match +# a name in the :providers: section above e.g. vsphere +# (required; will default to vsphere for backwards compatibility) +# # - clone_target # Per-pool option to override the global 'clone_target' cluster. # (optional) @@ -323,8 +429,18 @@ # # - ready_ttl # How long (in minutes) to keep VMs in 'ready' queues before destroying. -# (optional) - +# (optional; default: no limit) +# +# Provider specific pool settings +# vSphere provider +# - folder +# The vSphere 'folder' destination for spawned clones. +# (required) +# +# - datastore +# The vSphere 'datastore' destination for spawned clones. +# (required) +# # Example: :pools: @@ -336,6 +452,7 @@ size: 5 timeout: 15 ready_ttl: 1440 + provider: vsphere - name: 'debian-7-x86_64' alias: [ 'debian-7-64', 'debian-7-amd64' ] template: 'Templates/debian-7-x86_64' @@ -344,3 +461,4 @@ size: 5 timeout: 15 ready_ttl: 1440 + provider: vsphere