diff --git a/.gitignore b/.gitignore index 835dcd0..99789c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .ruby-version Gemfile.lock +Gemfile.local vendor +vmpooler.yaml +.bundle +coverage diff --git a/Gemfile b/Gemfile index 4fef816..975f8fd 100644 --- a/Gemfile +++ b/Gemfile @@ -23,3 +23,13 @@ group :test do gem 'simplecov', '>= 0.11.2' gem 'yarjuf', '>= 2.0' end + +# Evaluate Gemfile.local if it exists +if File.exists? "#{__FILE__}.local" + eval(File.read("#{__FILE__}.local"), binding) +end + +# Evaluate ~/.gemfile if it exists +if File.exists?(File.join(Dir.home, '.gemfile')) + eval(File.read(File.join(Dir.home, '.gemfile')), binding) +end diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index 844aa6e..f3b565f 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -12,7 +12,7 @@ module Vmpooler require 'yaml' require 'set' - %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd ).each do |lib| + %w( api graphite logger pool_manager vsphere_helper statsd dummy_statsd backingservice ).each do |lib| begin require "vmpooler/#{lib}" rescue LoadError @@ -21,7 +21,7 @@ module Vmpooler end def self.config(filepath='vmpooler.yaml') - parsed_config = {} + parsed_config = {} if ENV['VMPOOLER_CONFIG'] # Load configuration from ENV diff --git a/lib/vmpooler/api/helpers.rb b/lib/vmpooler/api/helpers.rb index 47c9062..0bd6275 100644 --- a/lib/vmpooler/api/helpers.rb +++ b/lib/vmpooler/api/helpers.rb @@ -56,6 +56,8 @@ module Vmpooler def authenticate(auth, username_str, password_str) case auth['provider'] + when 'dummy' + return (username_str != password_str) when 'ldap' require 'rubygems' require 'net/ldap' diff --git a/lib/vmpooler/backingservice.rb b/lib/vmpooler/backingservice.rb new file mode 100644 index 0000000..75645c3 --- /dev/null +++ b/lib/vmpooler/backingservice.rb @@ -0,0 +1,8 @@ +# TODO remove dummy for commit history +%w( base vsphere dummy ).each do |lib| + begin + require "vmpooler/backingservice/#{lib}" + rescue LoadError + require File.expand_path(File.join(File.dirname(__FILE__), 'backingservice', lib)) + end +end diff --git a/lib/vmpooler/backingservice/base.rb b/lib/vmpooler/backingservice/base.rb new file mode 100644 index 0000000..015be93 --- /dev/null +++ b/lib/vmpooler/backingservice/base.rb @@ -0,0 +1,111 @@ +module Vmpooler + class PoolManager + class BackingService + class Base + # These defs must be overidden in child classes + + def initialize(options) + @options = options + end + + # returns + # [String] Name of the backing service + def name + 'base' + end + + #def validate_config(config) + # false + #end + + # inputs + # pool : hashtable from config file + # returns + # hashtable + # name : name of the device <---- TODO is this all? + def vms_in_pool(pool) + fail "#{self.class.name} does not implement vms_in_pool" + end + + # inputs + # vm_name: string + # returns + # [String] hostname = Name of the host computer running the vm. If this is not a Virtual Machine, it returns the vm_name + def get_vm_host(vm_name) + fail "#{self.class.name} does not implement get_vm_host" + end + + # inputs + # vm_name: string + # returns + # [String] hostname = Name of the most appropriate host computer to run this VM. Useful for load balancing VMs in a cluster + # If this is not a Virtual Machine, it returns the vm_name + def find_least_used_compatible_host(vm_name) + fail "#{self.class.name} does not implement find_least_used_compatible_host" + end + + # inputs + # vm_name: string + # dest_host_name: string (Name of the host to migrate `vm_name` to) + # returns + # [Boolean] Returns true on success or false on failure + def migrate_vm_to_host(vm_name, dest_host_name) + fail "#{self.class.name} does not implement migrate_vm_to_host" + end + + # inputs + # vm_name: string + # returns + # nil if it doesn't exist + # Hastable of the VM + # [String] name = Name of the VM + # [String] hostname = Name reported by Vmware tools (host.summary.guest.hostName) + # [String] template = This is the name of template exposed by the API. It must _match_ the poolname + # [String] poolname = Name of the pool the VM is located + # [Time] boottime = Time when the VM was created/booted + # [String] powerstate = Current power state of a VM. Valid values (as per vCenter API) + # - 'PoweredOn','PoweredOff' + def get_vm(vm_name) + fail "#{self.class.name} does not implement get_vm" + end + + # inputs + # pool : hashtable from config file + # new_vmname : string Name the new VM should use + # returns + # Hashtable of the VM as per get_vm + def create_vm(pool,new_vmname) + fail "#{self.class.name} does not implement create_vm" + end + + # inputs + # vm_name: string + # pool: string + # returns + # boolean : true if success, false on error + def destroy_vm(vm_name,pool) + fail "#{self.class.name} does not implement destroy_vm" + end + + # inputs + # vm : string + # pool: string + # timeout: int (Seconds) + # returns + # result: boolean + def is_vm_ready?(vm,pool,timeout) + fail "#{self.class.name} does not implement is_vm_ready?" + end + + # inputs + # vm : string + # returns + # result: boolean + def vm_exists?(vm) + !get_vm(vm).nil? + end + + end + end + end +end \ No newline at end of file diff --git a/lib/vmpooler/backingservice/dummy.rb b/lib/vmpooler/backingservice/dummy.rb new file mode 100644 index 0000000..486abfd --- /dev/null +++ b/lib/vmpooler/backingservice/dummy.rb @@ -0,0 +1,238 @@ +require 'yaml' + +module Vmpooler + class PoolManager + class BackingService + class Dummy < Vmpooler::PoolManager::BackingService::Base + + # Fake VM backing service for testing, with initial configuration set in a simple text YAML filename + # or via YAML in the config file + def initialize(options) + super(options) + + dummyfilename = options['filename'] + + # TODO Accessing @dummylist is not thread safe :-( Mutexes? + + # This initial_state option is only intended to be used by spec tests + @dummylist = options['initial_state'].nil? ? {} : options['initial_state'] + + if !dummyfilename.nil? && File.exists?(dummyfilename) + @dummylist ||= YAML.load_file(dummyfilename) + end + end + + def vms_in_pool(pool) + get_pool_object(pool['name']).each do |vm| + vm + end + end + + def name + 'dummy' + end + + def get_vm(vm) + dummy = get_dummy_vm(vm) + return nil if dummy.nil? + + # Randomly power off the VM + unless dummy['powerstate'] != 'PoweredOn' || @options['getvm_poweroff_percent'].nil? + if 1 + rand(100) <= @options['getvm_poweroff_percent'] + dummy['powerstate'] = 'PoweredOff' + $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy Powered Off") + end + end + + # Randomly rename the host + unless dummy['hostname'] != dummy['name'] || @options['getvm_rename_percent'].nil? + if 1 + rand(100) <= @options['getvm_rename_percent'] + dummy['hostname'] = 'DUMMY' + dummy['name'] + $logger.log('d', "[ ] [#{dummy['poolname']}] '#{dummy['name']}' is being Dummy renamed") + end + end + + obj = {} + obj['name'] = dummy['name'] + obj['hostname'] = dummy['hostname'] + obj['boottime'] = dummy['boottime'] + obj['template'] = dummy['template'] + obj['poolname'] = dummy['poolname'] + obj['powerstate'] = dummy['powerstate'] + + obj + end + + def find_least_used_compatible_host(vm_name) + current_vm = get_dummy_vm(vm_name) + + # Unless migratevm_couldmove_percent is specified, don't migrate + return current_vm['vm_host'] if @options['migratevm_couldmove_percent'].nil? + + # Only migrate if migratevm_couldmove_percent is met + return current_vm['vm_host'] if 1 + rand(100) > @options['migratevm_couldmove_percent'] + + # Simulate a 10 node cluster and randomly pick a different one + new_host = "HOST" + (1 + rand(10)).to_s while new_host == current_vm['vm_host'] + + new_host + end + + def get_vm_host(vm_name) + current_vm = get_dummy_vm(vm_name) + + current_vm.nil? ? fail("VM #{vm_name} does not exist") : current_vm['vm_host'] + end + + def migrate_vm_to_host(vm_name, dest_host_name) + current_vm = get_dummy_vm(vm_name) + + # Inject migration delay + unless @options['migratevm_max_time'].nil? + migrate_time = 1 + rand(@options['migratevm_max_time']) + sleep(migrate_time) + end + + # Inject clone failure + unless @options['migratevm_fail_percent'].nil? + fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['migratevm_fail_percent'] + end + + current_vm['vm_host'] = dest_host_name + + true + end + + def is_vm_ready?(vm,pool,timeout) + host = get_dummy_vm(vm) + if !host then return false end + if host['poolname'] != pool then return false end + if host['ready'] then return true end + + Timeout.timeout(timeout) do + while host['dummy_state'] != 'RUNNING' + sleep(2) + host = get_dummy_vm(vm) + end + end + + # Simulate how long it takes from a VM being powered on until + # it's ready to receive a connection + sleep(2) + + unless @options['vmready_fail_percent'].nil? + fail "Dummy Failure for vmready_fail_percent" if 1 + rand(100) <= @options['vmready_fail_percent'] + end + + host['ready'] = true + true + end + + def create_vm(pool,dummy_hostname) + template_name = pool['template'] + pool_name = pool['name'] + + vm = {} + vm['name'] = dummy_hostname + vm['hostname'] = dummy_hostname + vm['domain'] = 'dummy.local' + # 'vm_template' is the name of the template to use to clone the VM from <----- Do we need this?!?!? + vm['vm_template'] = template_name + # 'template' is the Template name in VM Pooler API, in our case that's the poolname. + vm['template'] = pool_name + vm['poolname'] = pool_name + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'UNKNOWN' + get_pool_object(pool_name) + @dummylist['pool'][pool_name] << vm + + $logger.log('d', "[ ] [#{pool_name}] '#{dummy_hostname}' is being cloned from '#{template_name}'") + begin + # Inject clone time delay + unless @options['createvm_max_time'].nil? + vm['dummy_state'] = 'CLONING' + clone_time = 1 + rand(@options['createvm_max_time']) + sleep(clone_time) + end + + # Inject clone failure + unless @options['createvm_fail_percent'].nil? + fail "Dummy Failure for createvm_fail_percent" if 1 + rand(100) <= @options['createvm_fail_percent'] + end + + # Assert the VM is ready for use + vm['dummy_state'] = 'RUNNING' + rescue => err + remove_dummy_vm(dummy_hostname,pool_name) + raise + end + + get_vm(dummy_hostname) + end + + def destroy_vm(vm_name,pool) + vm = get_dummy_vm(vm_name) + if !vm then return false end + if vm['poolname'] != pool then return false end + + # Shutdown down the VM if it's poweredOn + if vm['powerstate'] = 'PoweredOn' + $logger.log('d', "[ ] [#{pool}] '#{vm_name}' is being shut down") + + # Inject shutdown delay time + unless @options['destroyvm_max_shutdown_time'].nil? + shutdown_time = 1 + rand(@options['destroyvm_max_shutdown_time']) + sleep(shutdown_time) + end + + vm['powerstate'] = 'PoweredOff' + end + + # Inject destroy VM delay + unless @options['destroyvm_max_time'].nil? + destroy_time = 1 + rand(@options['destroyvm_max_time']) + sleep(destroy_time) + end + + # Inject destroy VM failure + unless @options['destroyvm_fail_percent'].nil? + fail "Dummy Failure for migratevm_fail_percent" if 1 + rand(100) <= @options['destroyvm_fail_percent'] + end + + # 'Destroy' the VM + remove_dummy_vm(vm_name,pool) + + true + end + + private + def remove_dummy_vm(vm_name,pool) + return if @dummylist['pool'][pool].nil? + new_poollist = @dummylist['pool'][pool].delete_if { |vm| vm['name'] == vm_name } + @dummylist['pool'][pool] = new_poollist + end + + # Get's the pool config safely from the in-memory hashtable + def get_pool_object(pool_name) + @dummylist['pool'] = {} if @dummylist['pool'].nil? + @dummylist['pool'][pool_name] = [] if @dummylist['pool'][pool_name].nil? + + return @dummylist['pool'][pool_name] + end + + def get_dummy_vm(vm) + @dummylist['pool'].keys.each do |poolname| + @dummylist['pool'][poolname].each do |poolvm| + return poolvm if poolvm['name'] == vm + end + end + + nil + end + end + end + end +end \ No newline at end of file diff --git a/lib/vmpooler/backingservice/vsphere.rb b/lib/vmpooler/backingservice/vsphere.rb new file mode 100644 index 0000000..ee69e13 --- /dev/null +++ b/lib/vmpooler/backingservice/vsphere.rb @@ -0,0 +1,622 @@ +require 'rubygems' unless defined?(Gem) + +module Vmpooler + class PoolManager + class BackingService + class Vsphere < Vmpooler::PoolManager::BackingService::Base + #--------------- Public methods + + def initialize(options) + $credentials = options['credentials'] + $metrics = options['metrics'] + end + + def devices_in_pool(pool) + base = find_folder(pool['folder']) + + base.childEntity.each do |vm| + vm + end + end + +# def destroy_vm() +# # Destroy a VM +# def _destroy_vm(vm, pool, vsphere) +# $redis.srem('vmpooler__completed__' + pool, vm) +# $redis.hdel('vmpooler__active__' + pool, vm) +# $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + +# # Auto-expire metadata key +# $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + +# # TODO This is all vSphere specific + +# host = vsphere.find_vm(vm) + +# if host +# start = Time.now + +# if +# (host.runtime) && +# (host.runtime.powerState) && +# (host.runtime.powerState == 'poweredOn') + +# $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") +# host.PowerOffVM_Task.wait_for_completion +# end + +# host.Destroy_Task.wait_for_completion +# finish = '%.2f' % (Time.now - start) + +# $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") +# $metrics.timing("destroy.#{pool}", finish) +# end +# end +# end + + def create_device(pool) +'12345' + # clone_vm( + # pool['template'], + # pool['folder'], + # pool['datastore'], + # pool['clone_target'], + # vsphere + # ) + + # Thread.new do + # begin + # vm = {} + + # if template =~ /\// + # templatefolders = template.split('/') + # vm['template'] = templatefolders.pop + # end + + # if templatefolders + # vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) + # else + # fail 'Please provide a full path to the template' + # end + + # if vm['template'].length == 0 + # fail "Unable to find template '#{vm['template']}'!" + # end + + # # Generate a randomized hostname + # o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten + # vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + + # # Add VM to Redis inventory ('pending' pool) + # $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) + + # # Annotate with creation time, origin template, etc. + # # Add extraconfig options that can be queried by vmtools + # configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( + # annotation: JSON.pretty_generate( + # name: vm['hostname'], + # created_by: $config[:vsphere]['username'], + # base_template: vm['template'], + # creation_timestamp: Time.now.utc + # ), + # extraConfig: [ + # { key: 'guestinfo.hostname', + # value: vm['hostname'] + # } + # ] + # ) + + # # Choose a clone target + # if target + # $clone_target = vsphere.find_least_used_host(target) + # elsif $config[:config]['clone_target'] + # $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) + # end + + # # Put the VM in the specified folder and resource pool + # relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( + # datastore: vsphere.find_datastore(datastore), + # host: $clone_target, + # diskMoveType: :moveChildMostDiskBacking + # ) + + # # Create a clone spec + # spec = RbVmomi::VIM.VirtualMachineCloneSpec( + # location: relocateSpec, + # config: configSpec, + # powerOn: true, + # template: false + # ) + + # # Clone the VM + # $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + + # begin + # start = Time.now + # vm[vm['template']].CloneVM_Task( + # folder: vsphere.find_folder(folder), + # name: vm['hostname'], + # spec: spec + # ).wait_for_completion + # finish = '%.2f' % (Time.now - start) + + # $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) + # $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + + # $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") + # $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + # raise + # end + + # $redis.decr('vmpooler__tasks__clone') + + # $metrics.timing("clone.#{vm['template']}", finish) + # rescue => err + # $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") + # raise + # end + # end + + + end + +#**** When getting the VM details + # if (host.summary) && + # (host.summary.guest) && + # (host.summary.guest.hostName) && + # (host.summary.guest.hostName == vm) +# + + def is_vm_ready?(vm,pool,timeout) + fail "!!!!" + + + # def open_socket(host, domain=nil, timeout=5, port=22, &block) + # Timeout.timeout(timeout) do + # target_host = host + # target_host = "#{host}.#{domain}" if domain + # sock = TCPSocket.new target_host, port + # begin + # yield sock if block_given? + # ensure + # sock.close + # end + # end + # end + + # def _check_pending_vm(vm, pool, timeout, vsphere) + # host = vsphere.find_vm(vm) + + # if ! host + # fail_pending_vm(vm, pool, timeout, false) + # return + # end + # open_socket vm + # move_pending_vm_to_ready(vm, pool, host) + # rescue + # fail_pending_vm(vm, pool, timeout) + # end + + + end + + + + #--------------- Private methods + private + ADAPTER_TYPE = 'lsiLogic' + DISK_TYPE = 'thin' + DISK_MODE = 'persistent' + + def ensure_connected(connection, credentials) + connection.serviceInstance.CurrentTime + rescue + $metrics.increment("connect.open") + connect_to_vsphere $credentials + end + + def connect_to_vsphere(credentials) + @connection = RbVmomi::VIM.connect host: credentials['server'], + user: credentials['username'], + password: credentials['password'], + insecure: credentials['insecure'] || true + end + + def add_disk(vm, size, datastore) + ensure_connected @connection, $credentials + + return false unless size.to_i > 0 + + vmdk_datastore = find_datastore(datastore) + vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" + + controller = find_disk_controller(vm) + + vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( + capacityKb: size.to_i * 1024 * 1024, + adapterType: ADAPTER_TYPE, + diskType: DISK_TYPE + ) + + vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( + datastore: vmdk_datastore, + diskMode: DISK_MODE, + fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" + ) + + device = RbVmomi::VIM::VirtualDisk( + backing: vmdk_backing, + capacityInKB: size.to_i * 1024 * 1024, + controllerKey: controller.key, + key: -1, + unitNumber: find_disk_unit_number(vm, controller) + ) + + device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( + device: device, + operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') + ) + + vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( + deviceChange: [device_config_spec] + ) + + @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( + datacenter: @connection.serviceInstance.find_datacenter, + name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", + spec: vmdk_spec + ).wait_for_completion + + vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion + + true + end + + def find_datastore(datastorename) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + datacenter.find_datastore(datastorename) + end + + def find_device(vm, deviceName) + ensure_connected @connection, $credentials + + vm.config.hardware.device.each do |device| + return device if device.deviceInfo.label == deviceName + end + + nil + end + + def find_disk_controller(vm) + ensure_connected @connection, $credentials + + devices = find_disk_devices(vm) + + devices.keys.sort.each do |device| + if devices[device]['children'].length < 15 + return find_device(vm, devices[device]['device'].deviceInfo.label) + end + end + + nil + end + + def find_disk_devices(vm) + ensure_connected @connection, $credentials + + devices = {} + + vm.config.hardware.device.each do |device| + if device.is_a? RbVmomi::VIM::VirtualSCSIController + if devices[device.controllerKey].nil? + devices[device.key] = {} + devices[device.key]['children'] = [] + end + + devices[device.key]['device'] = device + end + + if device.is_a? RbVmomi::VIM::VirtualDisk + if devices[device.controllerKey].nil? + devices[device.controllerKey] = {} + devices[device.controllerKey]['children'] = [] + end + + devices[device.controllerKey]['children'].push(device) + end + end + + devices + end + + def find_disk_unit_number(vm, controller) + ensure_connected @connection, $credentials + + used_unit_numbers = [] + available_unit_numbers = [] + + devices = find_disk_devices(vm) + + devices.keys.sort.each do |c| + next unless controller.key == devices[c]['device'].key + used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) + devices[c]['children'].each do |disk| + used_unit_numbers.push(disk.unitNumber) + end + end + + (0..15).each do |scsi_id| + if used_unit_numbers.grep(scsi_id).length <= 0 + available_unit_numbers.push(scsi_id) + end + end + + available_unit_numbers.sort[0] + end + + def find_folder(foldername) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.vmFolder + folders = foldername.split('/') + folders.each do |folder| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == folder } + else + abort "Unexpected object type encountered (#{base.class}) while finding folder" + end + end + + base + end + + # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference + # Params: + # +model+:: CPU arch version to match on + # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments + def get_host_utilization(host, model=nil, limit=90) + if model + return nil unless host_has_cpu_model? host, model + end + return nil if host.runtime.inMaintenanceMode + return nil unless host.overallStatus == 'green' + + cpu_utilization = cpu_utilization_for host + memory_utilization = memory_utilization_for host + + return nil if cpu_utilization > limit + return nil if memory_utilization > limit + + [ cpu_utilization + memory_utilization, host ] + end + + def host_has_cpu_model?(host, model) + get_host_cpu_arch_version(host) == model + end + + def get_host_cpu_arch_version(host) + cpu_model = host.hardware.cpuPkg[0].description + cpu_model_parts = cpu_model.split() + arch_version = cpu_model_parts[4] + arch_version + end + + def cpu_utilization_for(host) + cpu_usage = host.summary.quickStats.overallCpuUsage + cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores + (cpu_usage.to_f / cpu_size.to_f) * 100 + end + + def memory_utilization_for(host) + memory_usage = host.summary.quickStats.overallMemoryUsage + memory_size = host.summary.hardware.memorySize / 1024 / 1024 + (memory_usage.to_f / memory_size.to_f) * 100 + end + + def find_least_used_host(cluster) + ensure_connected @connection, $credentials + + cluster_object = find_cluster(cluster) + target_hosts = get_cluster_host_utilization(cluster_object) + least_used_host = target_hosts.sort[0][1] + least_used_host + end + + def find_cluster(cluster) + datacenter = @connection.serviceInstance.find_datacenter + datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } + end + + def get_cluster_host_utilization(cluster) + cluster_hosts = [] + cluster.host.each do |host| + host_usage = get_host_utilization(host) + cluster_hosts << host_usage if host_usage + end + cluster_hosts + end + + def find_least_used_compatible_host(vm) + ensure_connected @connection, $credentials + + source_host = vm.summary.runtime.host + model = get_host_cpu_arch_version(source_host) + cluster = source_host.parent + target_hosts = [] + cluster.host.each do |host| + host_usage = get_host_utilization(host, model) + target_hosts << host_usage if host_usage + end + target_host = target_hosts.sort[0][1] + [target_host, target_host.name] + end + + def find_pool(poolname) + ensure_connected @connection, $credentials + + datacenter = @connection.serviceInstance.find_datacenter + base = datacenter.hostFolder + pools = poolname.split('/') + pools.each do |pool| + case base + when RbVmomi::VIM::Folder + base = base.childEntity.find { |f| f.name == pool } + when RbVmomi::VIM::ClusterComputeResource + base = base.resourcePool.resourcePool.find { |f| f.name == pool } + when RbVmomi::VIM::ResourcePool + base = base.resourcePool.find { |f| f.name == pool } + else + abort "Unexpected object type encountered (#{base.class}) while finding resource pool" + end + end + + base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) + base + end + + def find_snapshot(vm, snapshotname) + if vm.snapshot + get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) + end + end + + def find_vm(vmname) + ensure_connected @connection, $credentials + find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] + end + + def find_vm_light(vmname) + ensure_connected @connection, $credentials + + @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) + end + + def find_vm_heavy(vmname) + ensure_connected @connection, $credentials + + vmname = vmname.is_a?(Array) ? vmname : [vmname] + containerView = get_base_vm_container_from @connection + propertyCollector = @connection.propertyCollector + + objectSet = [{ + obj: containerView, + skip: true, + selectSet: [RbVmomi::VIM::TraversalSpec.new( + name: 'gettingTheVMs', + path: 'view', + skip: false, + type: 'ContainerView' + )] + }] + + propSet = [{ + pathSet: ['name'], + type: 'VirtualMachine' + }] + + results = propertyCollector.RetrievePropertiesEx( + specSet: [{ + objectSet: objectSet, + propSet: propSet + }], + options: { maxObjects: nil } + ) + + vms = {} + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + + while results.token + results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) + results.objects.each do |result| + name = result.propSet.first.val + next unless vmname.include? name + vms[name] = result.obj + end + end + + vms + end + + def find_vmdks(vmname, datastore) + ensure_connected @connection, $credentials + + disks = [] + + vmdk_datastore = find_datastore(datastore) + + vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' + vm_files.keys.each do |f| + vm_files[f]['layoutEx.file'].each do |l| + if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) + disks.push(l) + end + end + end + + disks + end + + def get_base_vm_container_from(connection) + ensure_connected @connection, $credentials + + viewManager = connection.serviceContent.viewManager + viewManager.CreateContainerView( + container: connection.serviceContent.rootFolder, + recursive: true, + type: ['VirtualMachine'] + ) + end + + def get_snapshot_list(tree, snapshotname) + snapshot = nil + + tree.each do |child| + if child.name == snapshotname + snapshot ||= child.snapshot + else + snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) + end + end + + snapshot + end + + def migrate_vm_host(vm, host) + relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) + vm.RelocateVM_Task(spec: relospec).wait_for_completion + end + + def close + @connection.close + end + + end + end + end +end + + + + + + + + + +module Vmpooler + class VsphereHelper + + end +end diff --git a/lib/vmpooler/logger.rb b/lib/vmpooler/logger.rb index 2c70756..8d1d52e 100644 --- a/lib/vmpooler/logger.rb +++ b/lib/vmpooler/logger.rb @@ -11,9 +11,11 @@ module Vmpooler def log(_level, string) time = Time.new stamp = time.strftime('%Y-%m-%d %H:%M:%S') - open(@file, 'a') do |f| f.puts "[#{stamp}] #{string}" + if ENV['VMPOOLER_DEBUG'] + puts "[#{stamp}] #{string}" + end end end end diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb index 4f55711..809d547 100644 --- a/lib/vmpooler/pool_manager.rb +++ b/lib/vmpooler/pool_manager.rb @@ -12,54 +12,55 @@ module Vmpooler # Connect to Redis $redis = redis - # vSphere object - $vsphere = {} + # per pool VM Backing Services + $backing_services = {} # Our thread-tracker object $threads = {} + + # WARNING DEBUG + #$logger.log('d',"Flushing REDIS WARNING!!!") + #$redis.flushdb end # Check the state of a VM - def check_pending_vm(vm, pool, timeout, vsphere) + # DONE + def check_pending_vm(vm, pool, timeout, backingservice) Thread.new do - _check_pending_vm(vm, pool, timeout, vsphere) - end - end - - def open_socket(host, domain=nil, timeout=5, port=22, &block) - Timeout.timeout(timeout) do - target_host = host - target_host = "#{host}.#{domain}" if domain - sock = TCPSocket.new target_host, port begin - yield sock if block_given? - ensure - sock.close + _check_pending_vm(vm, pool, timeout, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' errored while checking a pending vm : #{err}") + fail_pending_vm(vm, pool, timeout) + raise end end end - def _check_pending_vm(vm, pool, timeout, vsphere) - host = vsphere.find_vm(vm) - + # DONE + def _check_pending_vm(vm, pool, timeout, backingservice) + host = backingservice.get_vm(vm) if ! host fail_pending_vm(vm, pool, timeout, false) return end - open_socket vm - move_pending_vm_to_ready(vm, pool, host) - rescue - fail_pending_vm(vm, pool, timeout) + if backingservice.is_vm_ready?(vm,pool,timeout) + move_pending_vm_to_ready(vm, pool, host) + else + fail "VM is not ready" + end end + # DONE def remove_nonexistent_vm(vm, pool) $redis.srem("vmpooler__pending__#{pool}", vm) $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") end + # DONE def fail_pending_vm(vm, pool, timeout, exists=true) clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone') - return if ! clone_stamp + return true if ! clone_stamp time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60 if time_since_clone > timeout @@ -70,18 +71,19 @@ module Vmpooler remove_nonexistent_vm(vm, pool) end end + + true rescue => err $logger.log('d', "Fail pending VM failed with an error: #{err}") + + false end + # DONE def move_pending_vm_to_ready(vm, pool, host) - if (host.summary) && - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName == vm) - + if host['hostname'] == vm begin - Socket.getaddrinfo(vm, nil) # WTF? + Socket.getaddrinfo(vm, nil) # WTF? I assume this is just priming the local DNS resolver cache?!?! rescue end @@ -91,77 +93,91 @@ module Vmpooler $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm) $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) - $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue") + $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") end end - def check_ready_vm(vm, pool, ttl, vsphere) + # DONE + def check_ready_vm(vm, pool, ttl, backingservice) Thread.new do - if ttl > 0 - if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") - end + begin + _check_ready_vm(vm, pool, ttl, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : #{err}") + raise end + end + end - check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + # DONE + def _check_ready_vm(vm, pool, ttl, backingservice) + host = backingservice.get_vm(vm) + # Check if the host even exists + if !host + $redis.srem('vmpooler__ready__' + pool, vm) + $logger.log('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue") + return + end + # Check if the hosts TTL has expired + if ttl > 0 + if (((Time.now - host['boottime']) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + + $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + return + end + end + + # Periodically check that the VM is available + check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check') + if + (!check_stamp) || + (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime']) + + $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + + # Check if the VM is not powered on if - (!check_stamp) || - (((Time.now - Time.parse(check_stamp)) / 60) > $config[:config]['vm_checktime']) + (host['powerstate'] != 'PoweredOn') + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") + end - $redis.hset('vmpooler__vm__' + vm, 'check', Time.now) + # Check if the hostname has magically changed from underneath Pooler + if (host['hostname'] != vm) + $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") + end - host = vsphere.find_vm(vm) - - if host - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState != 'poweredOn') - - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue") - end - - if - (host.summary.guest) && - (host.summary.guest.hostName) && - (host.summary.guest.hostName != vm) - - $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - - $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue") - end + # Check if the VM is still ready/available + begin + fail "VM #{vm} is not ready" unless backingservice.is_vm_ready?(vm,pool,5) + rescue + if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") else - $redis.srem('vmpooler__ready__' + pool, vm) - - $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue") - end - - begin - open_socket vm - rescue - if $redis.smove('vmpooler__ready__' + pool, 'vmpooler__completed__' + pool, vm) - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue") - else - $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") - end + $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, and failed to remove from 'ready' queue") end end end end - def check_running_vm(vm, pool, ttl, vsphere) + # DONE + def check_running_vm(vm, pool, ttl, backingservice) Thread.new do - _check_running_vm(vm, pool, ttl, vsphere) + begin + _check_running_vm(vm, pool, ttl, backingservice) + rescue => err + $logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{err}") + raise + end end end - def _check_running_vm(vm, pool, ttl, vsphere) - host = vsphere.find_vm(vm) + # DONE + def _check_running_vm(vm, pool, ttl, backingservice) + host = backingservice.get_vm(vm) if host queue_from, queue_to = 'running', 'completed' @@ -179,151 +195,94 @@ module Vmpooler end end + # DONE def move_vm_queue(pool, vm, queue_from, queue_to, msg) $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm) $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") end - # Clone a VM - def clone_vm(template, folder, datastore, target, vsphere) + # DONE + def clone_vm(pool, backingservice) Thread.new do begin - vm = {} - - if template =~ /\// - templatefolders = template.split('/') - vm['template'] = templatefolders.pop - end - - if templatefolders - vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template']) - else - fail 'Please provide a full path to the template' - end - - if vm['template'].length == 0 - fail "Unable to find template '#{vm['template']}'!" - end + pool_name = pool['name'] # Generate a randomized hostname o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten - vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join + new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join # Add VM to Redis inventory ('pending' pool) - $redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname']) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template']) - - # Annotate with creation time, origin template, etc. - # Add extraconfig options that can be queried by vmtools - configSpec = RbVmomi::VIM.VirtualMachineConfigSpec( - annotation: JSON.pretty_generate( - name: vm['hostname'], - created_by: $config[:vsphere]['username'], - base_template: vm['template'], - creation_timestamp: Time.now.utc - ), - extraConfig: [ - { key: 'guestinfo.hostname', - value: vm['hostname'] - } - ] - ) - - # Choose a clone target - if target - $clone_target = vsphere.find_least_used_host(target) - elsif $config[:config]['clone_target'] - $clone_target = vsphere.find_least_used_host($config[:config]['clone_target']) - end - - # Put the VM in the specified folder and resource pool - relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec( - datastore: vsphere.find_datastore(datastore), - host: $clone_target, - diskMoveType: :moveChildMostDiskBacking - ) - - # Create a clone spec - spec = RbVmomi::VIM.VirtualMachineCloneSpec( - location: relocateSpec, - config: configSpec, - powerOn: true, - template: false - ) - - # Clone the VM - $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'") + $redis.sadd('vmpooler__pending__' + pool_name, new_vmname) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now) + $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) begin start = Time.now - vm[vm['template']].CloneVM_Task( - folder: vsphere.find_folder(folder), - name: vm['hostname'], - spec: spec - ).wait_for_completion + backingservice.create_vm(pool,new_vmname) finish = '%.2f' % (Time.now - start) - $redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) - $redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) + $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish) + $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish) + $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned from '#{pool_name}' in #{finish} seconds") - $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") + $metrics.timing("clone.#{pool_name}", finish) rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}") - $redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) + $logger.log('s', "[!] [#{pool_name}] '#{new_vmname}' clone failed with an error: #{err}") + $redis.srem('vmpooler__pending__' + pool_name, new_vmname) raise + ensure + $redis.decr('vmpooler__tasks__clone') end - - $redis.decr('vmpooler__tasks__clone') - - $metrics.timing("clone.#{vm['template']}", finish) rescue => err - $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}") + $logger.log('s', "[!] [#{pool['name']}] failed while preparing to clone with an error: #{err}") raise end end end # Destroy a VM - def destroy_vm(vm, pool, vsphere) + # DONE + def destroy_vm(vm, pool, backingservice) Thread.new do - $redis.srem('vmpooler__completed__' + pool, vm) - $redis.hdel('vmpooler__active__' + pool, vm) - $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) - - # Auto-expire metadata key - $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) - - host = vsphere.find_vm(vm) - - if host - start = Time.now - - if - (host.runtime) && - (host.runtime.powerState) && - (host.runtime.powerState == 'poweredOn') - - $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down") - host.PowerOffVM_Task.wait_for_completion - end - - host.Destroy_Task.wait_for_completion - finish = '%.2f' % (Time.now - start) - - $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") - $metrics.timing("destroy.#{pool}", finish) + begin + _destroy_vm(vm, pool, backingservice) + rescue => err + $logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{err}") + raise end end end + # Destroy a VM + # DONE + def _destroy_vm(vm, pool, backingservice) + $redis.srem('vmpooler__completed__' + pool, vm) + $redis.hdel('vmpooler__active__' + pool, vm) + $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now) + + # Auto-expire metadata key + $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) + + start = Time.now + + backingservice.destroy_vm(vm,pool) + + finish = '%.2f' % (Time.now - start) + $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds") + $metrics.timing("destroy.#{pool}", finish) + end + def create_vm_disk(vm, disk_size, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific Thread.new do _create_vm_disk(vm, disk_size, vsphere) end end def _create_vm_disk(vm, disk_size, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0)) @@ -358,12 +317,16 @@ module Vmpooler end def create_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific Thread.new do _create_vm_snapshot(vm, snapshot_name, vsphere) end end def _create_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?)) @@ -387,12 +350,16 @@ module Vmpooler end def revert_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific Thread.new do _revert_vm_snapshot(vm, snapshot_name, vsphere) end end def _revert_vm_snapshot(vm, snapshot_name, vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific host = vsphere.find_vm(vm) if host @@ -413,6 +380,8 @@ module Vmpooler end def check_disk_queue +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific $logger.log('d', "[*] [disk_manager] starting worker thread") $vsphere['disk_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics @@ -426,6 +395,8 @@ module Vmpooler end def _check_disk_queue(vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__disk') unless vm.nil? @@ -439,6 +410,8 @@ module Vmpooler end def check_snapshot_queue +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific $logger.log('d', "[*] [snapshot_manager] starting worker thread") $vsphere['snapshot_manager'] ||= Vmpooler::VsphereHelper.new $config, $metrics @@ -452,6 +425,8 @@ module Vmpooler end def _check_snapshot_queue(vsphere) +fail "NOT YET REFACTORED" +# TODO This is all vSphere specific vm = $redis.spop('vmpooler__tasks__snapshot') unless vm.nil? @@ -475,56 +450,55 @@ module Vmpooler end end + # DONE def migration_limit(migration_limit) # Returns migration_limit setting when enabled return false if migration_limit == 0 || ! migration_limit migration_limit if migration_limit >= 1 end - def migrate_vm(vm, pool, vsphere) + # DONE + def migrate_vm(vm, pool, backingservice) Thread.new do - _migrate_vm(vm, pool, vsphere) + begin + _migrate_vm(vm, pool, backingservice) + rescue => err + $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") + remove_vmpooler_migration_vm(pool, vm) + end end end - def _migrate_vm(vm, pool, vsphere) - begin - $redis.srem('vmpooler__migrating__' + pool, vm) - vm_object = vsphere.find_vm(vm) - parent_host, parent_host_name = get_vm_host_info(vm_object) - migration_limit = migration_limit $config[:config]['migration_limit'] - migration_count = $redis.scard('vmpooler__migration') + # DONE + def _migrate_vm(vm, pool, backingservice) + $redis.srem('vmpooler__migrating__' + pool, vm) - if ! migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + parent_host_name = backingservice.get_vm_host(vm) + migration_limit = migration_limit $config[:config]['migration_limit'] + migration_count = $redis.scard('vmpooler__migration') + + if ! migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") + return + else + if migration_count >= migration_limit + $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") return else - if migration_count >= migration_limit - $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached") - return + $redis.sadd('vmpooler__migration', vm) + host_name = backingservice.find_least_used_compatible_host(vm) + if host_name == parent_host_name + $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") else - $redis.sadd('vmpooler__migration', vm) - host, host_name = vsphere.find_least_used_compatible_host(vm_object) - if host == parent_host - $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}") - else - finish = migrate_vm_and_record_timing(vm_object, vm, pool, host, parent_host_name, host_name, vsphere) - $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") - end - remove_vmpooler_migration_vm(pool, vm) + finish = migrate_vm_and_record_timing(vm, pool, parent_host_name, host_name, backingservice) + $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds") end + remove_vmpooler_migration_vm(pool, vm) end - rescue => err - $logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}") - remove_vmpooler_migration_vm(pool, vm) end end - def get_vm_host_info(vm_object) - parent_host = vm_object.summary.runtime.host - [parent_host, parent_host.name] - end - + # DONE def remove_vmpooler_migration_vm(pool, vm) begin $redis.srem('vmpooler__migration', vm) @@ -533,9 +507,10 @@ module Vmpooler end end - def migrate_vm_and_record_timing(vm_object, vm_name, pool, host, source_host_name, dest_host_name, vsphere) + # DONE + def migrate_vm_and_record_timing(vm_name, pool, source_host_name, dest_host_name, backingservice) start = Time.now - vsphere.migrate_vm_host(vm_object, host) + backingservice.migrate_vm_to_host(vm_name, dest_host_name) finish = '%.2f' % (Time.now - start) $metrics.timing("migrate.#{pool}", finish) $metrics.increment("migrate_from.#{source_host_name}") @@ -546,26 +521,34 @@ module Vmpooler finish end + # DONE def check_pool(pool) $logger.log('d', "[*] [#{pool['name']}] starting worker thread") - $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new $config, $metrics + case pool['backingservice'] + when 'vsphere' + # TODO what about the helper + $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Vsphere.new({ 'metrics' => $metrics}) # TODO Vmpooler::VsphereHelper.new $config[:vsphere], $metrics + when 'dummy' + $backing_services[pool['name']] ||= Vmpooler::PoolManager::BackingService::Dummy.new($config[:backingservice][:dummy]) + else + $logger.log('s', "[!] backing service #{pool['backingservice']} is unknown for pool [#{pool['name']}]") + end $threads[pool['name']] = Thread.new do loop do - _check_pool(pool, $vsphere[pool['name']]) - sleep(5) + _check_pool(pool, $backing_services[pool['name']]) +# TODO Should this be configurable? + sleep(2) # Should be 5 end end end - def _check_pool(pool, vsphere) + def _check_pool(pool,backingservice) # INVENTORY inventory = {} begin - base = vsphere.find_folder(pool['folder']) - - base.childEntity.each do |vm| + backingservice.vms_in_pool(pool).each do |vm| if (! $redis.sismember('vmpooler__running__' + pool['name'], vm['name'])) && (! $redis.sismember('vmpooler__ready__' + pool['name'], vm['name'])) && @@ -590,7 +573,7 @@ module Vmpooler if inventory[vm] begin vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12 - check_running_vm(vm, pool['name'], vm_lifetime, vsphere) + check_running_vm(vm, pool['name'], vm_lifetime, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool with an error while evaluating running VMs: #{err}") end @@ -601,7 +584,7 @@ module Vmpooler $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm| if inventory[vm] begin - check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere) + check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}") end @@ -613,7 +596,7 @@ module Vmpooler pool_timeout = pool['timeout'] || $config[:config]['timeout'] || 15 if inventory[vm] begin - check_pending_vm(vm, pool['name'], pool_timeout, vsphere) + check_pending_vm(vm, pool['name'], pool_timeout, backingservice) rescue => err $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating pending VMs: #{err}") end @@ -626,7 +609,7 @@ module Vmpooler $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm| if inventory[vm] begin - destroy_vm(vm, pool['name'], vsphere) + destroy_vm(vm, pool['name'], backingservice) rescue => err $redis.srem("vmpooler__completed__#{pool['name']}", vm) $redis.hdel("vmpooler__active__#{pool['name']}", vm) @@ -663,7 +646,7 @@ module Vmpooler $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm| if inventory[vm] begin - migrate_vm(vm, pool['name'], vsphere) + migrate_vm(vm, pool['name'], backingservice) rescue => err $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}") end @@ -693,14 +676,7 @@ module Vmpooler if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i begin $redis.incr('vmpooler__tasks__clone') - - clone_vm( - pool['template'], - pool['folder'], - pool['datastore'], - pool['clone_target'], - vsphere - ) + clone_vm(pool,backingservice) rescue => err $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}") $redis.decr('vmpooler__tasks__clone') @@ -721,21 +697,26 @@ module Vmpooler $redis.set('vmpooler__tasks__clone', 0) # Clear out vmpooler__migrations since stale entries may be left after a restart $redis.del('vmpooler__migration') + # Set default backingservice for all pools that do not have one defined + $config[:pools].each do |pool| + pool['backingservice'] = 'vsphere' if pool['backingservice'].nil? + end loop do - if ! $threads['disk_manager'] - check_disk_queue - elsif ! $threads['disk_manager'].alive? - $logger.log('d', "[!] [disk_manager] worker thread died, restarting") - check_disk_queue - end + # DEBUG TO DO + # if ! $threads['disk_manager'] + # check_disk_queue + # elsif ! $threads['disk_manager'].alive? + # $logger.log('d', "[!] [disk_manager] worker thread died, restarting") + # check_disk_queue + # end - if ! $threads['snapshot_manager'] - check_snapshot_queue - elsif ! $threads['snapshot_manager'].alive? - $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting") - check_snapshot_queue - end + # if ! $threads['snapshot_manager'] + # check_snapshot_queue + # elsif ! $threads['snapshot_manager'].alive? + # $logger.log('d', "[!] [snapshot_manager] worker thread died, restarting") + # check_snapshot_queue + # end $config[:pools].each do |pool| if ! $threads[pool['name']] diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb index cc93250..3b7f230 100644 --- a/lib/vmpooler/vsphere_helper.rb +++ b/lib/vmpooler/vsphere_helper.rb @@ -1,417 +1,417 @@ -require 'rubygems' unless defined?(Gem) - -module Vmpooler - class VsphereHelper - ADAPTER_TYPE = 'lsiLogic' - DISK_TYPE = 'thin' - DISK_MODE = 'persistent' - - def initialize(config, metrics) - $credentials = config[:vsphere] - $conf = config[:config] - $metrics = metrics - end - - def ensure_connected(connection, credentials) - connection.serviceInstance.CurrentTime - rescue - $metrics.increment("connect.open") - connect_to_vsphere $credentials - end - - def connect_to_vsphere(credentials) - max_tries = $conf['max_tries'] || 3 - retry_factor = $conf['retry_factor'] || 10 - try = 1 - begin - @connection = RbVmomi::VIM.connect host: credentials['server'], - user: credentials['username'], - password: credentials['password'], - insecure: credentials['insecure'] || true - $metrics.increment("connect.open") - rescue => err - try += 1 - $metrics.increment("connect.fail") - raise err if try == max_tries - sleep(try * retry_factor) - retry - end - end - - def add_disk(vm, size, datastore) - ensure_connected @connection, $credentials - - return false unless size.to_i > 0 - - vmdk_datastore = find_datastore(datastore) - vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" - - controller = find_disk_controller(vm) - - vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( - capacityKb: size.to_i * 1024 * 1024, - adapterType: ADAPTER_TYPE, - diskType: DISK_TYPE - ) - - vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( - datastore: vmdk_datastore, - diskMode: DISK_MODE, - fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" - ) - - device = RbVmomi::VIM::VirtualDisk( - backing: vmdk_backing, - capacityInKB: size.to_i * 1024 * 1024, - controllerKey: controller.key, - key: -1, - unitNumber: find_disk_unit_number(vm, controller) - ) - - device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( - device: device, - operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') - ) - - vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( - deviceChange: [device_config_spec] - ) - - @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( - datacenter: @connection.serviceInstance.find_datacenter, - name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", - spec: vmdk_spec - ).wait_for_completion - - vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion - - true - end - - def find_datastore(datastorename) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - datacenter.find_datastore(datastorename) - end - - def find_device(vm, deviceName) - ensure_connected @connection, $credentials - - vm.config.hardware.device.each do |device| - return device if device.deviceInfo.label == deviceName - end - - nil - end - - def find_disk_controller(vm) - ensure_connected @connection, $credentials - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |device| - if devices[device]['children'].length < 15 - return find_device(vm, devices[device]['device'].deviceInfo.label) - end - end - - nil - end - - def find_disk_devices(vm) - ensure_connected @connection, $credentials - - devices = {} - - vm.config.hardware.device.each do |device| - if device.is_a? RbVmomi::VIM::VirtualSCSIController - if devices[device.controllerKey].nil? - devices[device.key] = {} - devices[device.key]['children'] = [] - end - - devices[device.key]['device'] = device - end - - if device.is_a? RbVmomi::VIM::VirtualDisk - if devices[device.controllerKey].nil? - devices[device.controllerKey] = {} - devices[device.controllerKey]['children'] = [] - end - - devices[device.controllerKey]['children'].push(device) - end - end - - devices - end - - def find_disk_unit_number(vm, controller) - ensure_connected @connection, $credentials - - used_unit_numbers = [] - available_unit_numbers = [] - - devices = find_disk_devices(vm) - - devices.keys.sort.each do |c| - next unless controller.key == devices[c]['device'].key - used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) - devices[c]['children'].each do |disk| - used_unit_numbers.push(disk.unitNumber) - end - end - - (0..15).each do |scsi_id| - if used_unit_numbers.grep(scsi_id).length <= 0 - available_unit_numbers.push(scsi_id) - end - end - - available_unit_numbers.sort[0] - end - - def find_folder(foldername) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.vmFolder - folders = foldername.split('/') - folders.each do |folder| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == folder } - else - abort "Unexpected object type encountered (#{base.class}) while finding folder" - end - end - - base - end - - # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference - # Params: - # +model+:: CPU arch version to match on - # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments - def get_host_utilization(host, model=nil, limit=90) - if model - return nil unless host_has_cpu_model? host, model - end - return nil if host.runtime.inMaintenanceMode - return nil unless host.overallStatus == 'green' - - cpu_utilization = cpu_utilization_for host - memory_utilization = memory_utilization_for host - - return nil if cpu_utilization > limit - return nil if memory_utilization > limit - - [ cpu_utilization + memory_utilization, host ] - end - - def host_has_cpu_model?(host, model) - get_host_cpu_arch_version(host) == model - end - - def get_host_cpu_arch_version(host) - cpu_model = host.hardware.cpuPkg[0].description - cpu_model_parts = cpu_model.split() - arch_version = cpu_model_parts[4] - arch_version - end - - def cpu_utilization_for(host) - cpu_usage = host.summary.quickStats.overallCpuUsage - cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores - (cpu_usage.to_f / cpu_size.to_f) * 100 - end - - def memory_utilization_for(host) - memory_usage = host.summary.quickStats.overallMemoryUsage - memory_size = host.summary.hardware.memorySize / 1024 / 1024 - (memory_usage.to_f / memory_size.to_f) * 100 - end - - def find_least_used_host(cluster) - ensure_connected @connection, $credentials - - cluster_object = find_cluster(cluster) - target_hosts = get_cluster_host_utilization(cluster_object) - least_used_host = target_hosts.sort[0][1] - least_used_host - end - - def find_cluster(cluster) - datacenter = @connection.serviceInstance.find_datacenter - datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } - end - - def get_cluster_host_utilization(cluster) - cluster_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host) - cluster_hosts << host_usage if host_usage - end - cluster_hosts - end - - def find_least_used_compatible_host(vm) - ensure_connected @connection, $credentials - - source_host = vm.summary.runtime.host - model = get_host_cpu_arch_version(source_host) - cluster = source_host.parent - target_hosts = [] - cluster.host.each do |host| - host_usage = get_host_utilization(host, model) - target_hosts << host_usage if host_usage - end - target_host = target_hosts.sort[0][1] - [target_host, target_host.name] - end - - def find_pool(poolname) - ensure_connected @connection, $credentials - - datacenter = @connection.serviceInstance.find_datacenter - base = datacenter.hostFolder - pools = poolname.split('/') - pools.each do |pool| - case base - when RbVmomi::VIM::Folder - base = base.childEntity.find { |f| f.name == pool } - when RbVmomi::VIM::ClusterComputeResource - base = base.resourcePool.resourcePool.find { |f| f.name == pool } - when RbVmomi::VIM::ResourcePool - base = base.resourcePool.find { |f| f.name == pool } - else - abort "Unexpected object type encountered (#{base.class}) while finding resource pool" - end - end - - base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) - base - end - - def find_snapshot(vm, snapshotname) - if vm.snapshot - get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) - end - end - - def find_vm(vmname) - ensure_connected @connection, $credentials - find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] - end - - def find_vm_light(vmname) - ensure_connected @connection, $credentials - - @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) - end - - def find_vm_heavy(vmname) - ensure_connected @connection, $credentials - - vmname = vmname.is_a?(Array) ? vmname : [vmname] - containerView = get_base_vm_container_from @connection - propertyCollector = @connection.propertyCollector - - objectSet = [{ - obj: containerView, - skip: true, - selectSet: [RbVmomi::VIM::TraversalSpec.new( - name: 'gettingTheVMs', - path: 'view', - skip: false, - type: 'ContainerView' - )] - }] - - propSet = [{ - pathSet: ['name'], - type: 'VirtualMachine' - }] - - results = propertyCollector.RetrievePropertiesEx( - specSet: [{ - objectSet: objectSet, - propSet: propSet - }], - options: { maxObjects: nil } - ) - - vms = {} - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - - while results.token - results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) - results.objects.each do |result| - name = result.propSet.first.val - next unless vmname.include? name - vms[name] = result.obj - end - end - - vms - end - - def find_vmdks(vmname, datastore) - ensure_connected @connection, $credentials - - disks = [] - - vmdk_datastore = find_datastore(datastore) - - vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' - vm_files.keys.each do |f| - vm_files[f]['layoutEx.file'].each do |l| - if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) - disks.push(l) - end - end - end - - disks - end - - def get_base_vm_container_from(connection) - ensure_connected @connection, $credentials - - viewManager = connection.serviceContent.viewManager - viewManager.CreateContainerView( - container: connection.serviceContent.rootFolder, - recursive: true, - type: ['VirtualMachine'] - ) - end - - def get_snapshot_list(tree, snapshotname) - snapshot = nil - - tree.each do |child| - if child.name == snapshotname - snapshot ||= child.snapshot - else - snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) - end - end - - snapshot - end - - def migrate_vm_host(vm, host) - relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) - vm.RelocateVM_Task(spec: relospec).wait_for_completion - end - - def close - @connection.close - end - end -end +# require 'rubygems' unless defined?(Gem) + +# module Vmpooler +# class VsphereHelper +# ADAPTER_TYPE = 'lsiLogic' +# DISK_TYPE = 'thin' +# DISK_MODE = 'persistent' + +# def initialize(config, metrics) +# $credentials = config[:vsphere] +# $conf = config[:config] +# $metrics = metrics +# end + +# def ensure_connected(connection, credentials) +# connection.serviceInstance.CurrentTime +# rescue +# $metrics.increment("connect.open") +# connect_to_vsphere $credentials +# end + +# def connect_to_vsphere(credentials) +# max_tries = $conf['max_tries'] || 3 +# retry_factor = $conf['retry_factor'] || 10 +# try = 1 +# begin +# @connection = RbVmomi::VIM.connect host: credentials['server'], +# user: credentials['username'], +# password: credentials['password'], +# insecure: credentials['insecure'] || true +# $metrics.increment("connect.open") +# rescue => err +# try += 1 +# $metrics.increment("connect.fail") +# raise err if try == max_tries +# sleep(try * retry_factor) +# retry +# end +# end + +# def add_disk(vm, size, datastore) +# ensure_connected @connection, $credentials + +# return false unless size.to_i > 0 + +# vmdk_datastore = find_datastore(datastore) +# vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore).length + 1}.vmdk" + +# controller = find_disk_controller(vm) + +# vmdk_spec = RbVmomi::VIM::FileBackedVirtualDiskSpec( +# capacityKb: size.to_i * 1024 * 1024, +# adapterType: ADAPTER_TYPE, +# diskType: DISK_TYPE +# ) + +# vmdk_backing = RbVmomi::VIM::VirtualDiskFlatVer2BackingInfo( +# datastore: vmdk_datastore, +# diskMode: DISK_MODE, +# fileName: "[#{vmdk_datastore.name}] #{vmdk_file_name}" +# ) + +# device = RbVmomi::VIM::VirtualDisk( +# backing: vmdk_backing, +# capacityInKB: size.to_i * 1024 * 1024, +# controllerKey: controller.key, +# key: -1, +# unitNumber: find_disk_unit_number(vm, controller) +# ) + +# device_config_spec = RbVmomi::VIM::VirtualDeviceConfigSpec( +# device: device, +# operation: RbVmomi::VIM::VirtualDeviceConfigSpecOperation('add') +# ) + +# vm_config_spec = RbVmomi::VIM::VirtualMachineConfigSpec( +# deviceChange: [device_config_spec] +# ) + +# @connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task( +# datacenter: @connection.serviceInstance.find_datacenter, +# name: "[#{vmdk_datastore.name}] #{vmdk_file_name}", +# spec: vmdk_spec +# ).wait_for_completion + +# vm.ReconfigVM_Task(spec: vm_config_spec).wait_for_completion + +# true +# end + +# def find_datastore(datastorename) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# datacenter.find_datastore(datastorename) +# end + +# def find_device(vm, deviceName) +# ensure_connected @connection, $credentials + +# vm.config.hardware.device.each do |device| +# return device if device.deviceInfo.label == deviceName +# end + +# nil +# end + +# def find_disk_controller(vm) +# ensure_connected @connection, $credentials + +# devices = find_disk_devices(vm) + +# devices.keys.sort.each do |device| +# if devices[device]['children'].length < 15 +# return find_device(vm, devices[device]['device'].deviceInfo.label) +# end +# end + +# nil +# end + +# def find_disk_devices(vm) +# ensure_connected @connection, $credentials + +# devices = {} + +# vm.config.hardware.device.each do |device| +# if device.is_a? RbVmomi::VIM::VirtualSCSIController +# if devices[device.controllerKey].nil? +# devices[device.key] = {} +# devices[device.key]['children'] = [] +# end + +# devices[device.key]['device'] = device +# end + +# if device.is_a? RbVmomi::VIM::VirtualDisk +# if devices[device.controllerKey].nil? +# devices[device.controllerKey] = {} +# devices[device.controllerKey]['children'] = [] +# end + +# devices[device.controllerKey]['children'].push(device) +# end +# end + +# devices +# end + +# def find_disk_unit_number(vm, controller) +# ensure_connected @connection, $credentials + +# used_unit_numbers = [] +# available_unit_numbers = [] + +# devices = find_disk_devices(vm) + +# devices.keys.sort.each do |c| +# next unless controller.key == devices[c]['device'].key +# used_unit_numbers.push(devices[c]['device'].scsiCtlrUnitNumber) +# devices[c]['children'].each do |disk| +# used_unit_numbers.push(disk.unitNumber) +# end +# end + +# (0..15).each do |scsi_id| +# if used_unit_numbers.grep(scsi_id).length <= 0 +# available_unit_numbers.push(scsi_id) +# end +# end + +# available_unit_numbers.sort[0] +# end + +# def find_folder(foldername) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# base = datacenter.vmFolder +# folders = foldername.split('/') +# folders.each do |folder| +# case base +# when RbVmomi::VIM::Folder +# base = base.childEntity.find { |f| f.name == folder } +# else +# abort "Unexpected object type encountered (#{base.class}) while finding folder" +# end +# end + +# base +# end + +# # Returns an array containing cumulative CPU and memory utilization of a host, and its object reference +# # Params: +# # +model+:: CPU arch version to match on +# # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments +# def get_host_utilization(host, model=nil, limit=90) +# if model +# return nil unless host_has_cpu_model? host, model +# end +# return nil if host.runtime.inMaintenanceMode +# return nil unless host.overallStatus == 'green' + +# cpu_utilization = cpu_utilization_for host +# memory_utilization = memory_utilization_for host + +# return nil if cpu_utilization > limit +# return nil if memory_utilization > limit + +# [ cpu_utilization + memory_utilization, host ] +# end + +# def host_has_cpu_model?(host, model) +# get_host_cpu_arch_version(host) == model +# end + +# def get_host_cpu_arch_version(host) +# cpu_model = host.hardware.cpuPkg[0].description +# cpu_model_parts = cpu_model.split() +# arch_version = cpu_model_parts[4] +# arch_version +# end + +# def cpu_utilization_for(host) +# cpu_usage = host.summary.quickStats.overallCpuUsage +# cpu_size = host.summary.hardware.cpuMhz * host.summary.hardware.numCpuCores +# (cpu_usage.to_f / cpu_size.to_f) * 100 +# end + +# def memory_utilization_for(host) +# memory_usage = host.summary.quickStats.overallMemoryUsage +# memory_size = host.summary.hardware.memorySize / 1024 / 1024 +# (memory_usage.to_f / memory_size.to_f) * 100 +# end + +# def find_least_used_host(cluster) +# ensure_connected @connection, $credentials + +# cluster_object = find_cluster(cluster) +# target_hosts = get_cluster_host_utilization(cluster_object) +# least_used_host = target_hosts.sort[0][1] +# least_used_host +# end + +# def find_cluster(cluster) +# datacenter = @connection.serviceInstance.find_datacenter +# datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster } +# end + +# def get_cluster_host_utilization(cluster) +# cluster_hosts = [] +# cluster.host.each do |host| +# host_usage = get_host_utilization(host) +# cluster_hosts << host_usage if host_usage +# end +# cluster_hosts +# end + +# def find_least_used_compatible_host(vm) +# ensure_connected @connection, $credentials + +# source_host = vm.summary.runtime.host +# model = get_host_cpu_arch_version(source_host) +# cluster = source_host.parent +# target_hosts = [] +# cluster.host.each do |host| +# host_usage = get_host_utilization(host, model) +# target_hosts << host_usage if host_usage +# end +# target_host = target_hosts.sort[0][1] +# [target_host, target_host.name] +# end + +# def find_pool(poolname) +# ensure_connected @connection, $credentials + +# datacenter = @connection.serviceInstance.find_datacenter +# base = datacenter.hostFolder +# pools = poolname.split('/') +# pools.each do |pool| +# case base +# when RbVmomi::VIM::Folder +# base = base.childEntity.find { |f| f.name == pool } +# when RbVmomi::VIM::ClusterComputeResource +# base = base.resourcePool.resourcePool.find { |f| f.name == pool } +# when RbVmomi::VIM::ResourcePool +# base = base.resourcePool.find { |f| f.name == pool } +# else +# abort "Unexpected object type encountered (#{base.class}) while finding resource pool" +# end +# end + +# base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) && base.respond_to?(:resourcePool) +# base +# end + +# def find_snapshot(vm, snapshotname) +# if vm.snapshot +# get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) +# end +# end + +# def find_vm(vmname) +# ensure_connected @connection, $credentials +# find_vm_light(vmname) || find_vm_heavy(vmname)[vmname] +# end + +# def find_vm_light(vmname) +# ensure_connected @connection, $credentials + +# @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) +# end + +# def find_vm_heavy(vmname) +# ensure_connected @connection, $credentials + +# vmname = vmname.is_a?(Array) ? vmname : [vmname] +# containerView = get_base_vm_container_from @connection +# propertyCollector = @connection.propertyCollector + +# objectSet = [{ +# obj: containerView, +# skip: true, +# selectSet: [RbVmomi::VIM::TraversalSpec.new( +# name: 'gettingTheVMs', +# path: 'view', +# skip: false, +# type: 'ContainerView' +# )] +# }] + +# propSet = [{ +# pathSet: ['name'], +# type: 'VirtualMachine' +# }] + +# results = propertyCollector.RetrievePropertiesEx( +# specSet: [{ +# objectSet: objectSet, +# propSet: propSet +# }], +# options: { maxObjects: nil } +# ) + +# vms = {} +# results.objects.each do |result| +# name = result.propSet.first.val +# next unless vmname.include? name +# vms[name] = result.obj +# end + +# while results.token +# results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token) +# results.objects.each do |result| +# name = result.propSet.first.val +# next unless vmname.include? name +# vms[name] = result.obj +# end +# end + +# vms +# end + +# def find_vmdks(vmname, datastore) +# ensure_connected @connection, $credentials + +# disks = [] + +# vmdk_datastore = find_datastore(datastore) + +# vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file' +# vm_files.keys.each do |f| +# vm_files[f]['layoutEx.file'].each do |l| +# if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/) +# disks.push(l) +# end +# end +# end + +# disks +# end + +# def get_base_vm_container_from(connection) +# ensure_connected @connection, $credentials + +# viewManager = connection.serviceContent.viewManager +# viewManager.CreateContainerView( +# container: connection.serviceContent.rootFolder, +# recursive: true, +# type: ['VirtualMachine'] +# ) +# end + +# def get_snapshot_list(tree, snapshotname) +# snapshot = nil + +# tree.each do |child| +# if child.name == snapshotname +# snapshot ||= child.snapshot +# else +# snapshot ||= get_snapshot_list(child.childSnapshotList, snapshotname) +# end +# end + +# snapshot +# end + +# def migrate_vm_host(vm, host) +# relospec = RbVmomi::VIM.VirtualMachineRelocateSpec(host: host) +# vm.RelocateVM_Task(spec: relospec).wait_for_completion +# end + +# def close +# @connection.close +# end +# end +# end diff --git a/spec/vmpooler/backingservice/base_spec.rb b/spec/vmpooler/backingservice/base_spec.rb new file mode 100644 index 0000000..5256c28 --- /dev/null +++ b/spec/vmpooler/backingservice/base_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' + +# This spec does not really exercise code paths but is merely used +# to enforce that certain methods are defined in the base classes + +describe 'Vmpooler::PoolManager::BackingService::Base' do + let(:config) { {} } + let(:fake_vm) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm + } + + subject { Vmpooler::PoolManager::BackingService::Base.new(config) } + + describe '#name' do + it 'should be base' do + expect(subject.name).to eq('base') + end + end + + describe '#vms_in_pool' do + it 'should raise error' do + expect{subject.vms_in_pool('pool')}.to raise_error(/does not implement vms_in_pool/) + end + end + + describe '#get_vm_host' do + it 'should raise error' do + expect{subject.get_vm_host('vm')}.to raise_error(/does not implement get_vm_host/) + end + end + + describe '#find_least_used_compatible_host' do + it 'should raise error' do + expect{subject.find_least_used_compatible_host('vm')}.to raise_error(/does not implement find_least_used_compatible_host/) + end + end + + describe '#migrate_vm_to_host' do + it 'should raise error' do + expect{subject.migrate_vm_to_host('vm','host')}.to raise_error(/does not implement migrate_vm_to_host/) + end + end + + describe '#get_vm' do + it 'should raise error' do + expect{subject.get_vm('vm')}.to raise_error(/does not implement get_vm/) + end + end + + describe '#create_vm' do + it 'should raise error' do + expect{subject.create_vm('pool','newname')}.to raise_error(/does not implement create_vm/) + end + end + + describe '#destroy_vm' do + it 'should raise error' do + expect{subject.destroy_vm('vm','pool')}.to raise_error(/does not implement destroy_vm/) + end + end + + describe '#is_vm_ready?' do + it 'should raise error' do + expect{subject.is_vm_ready?('vm','pool','timeout')}.to raise_error(/does not implement is_vm_ready?/) + end + end + + describe '#vm_exists?' do + it 'should raise error' do + expect{subject.vm_exists?('vm')}.to raise_error(/does not implement/) + end + + it 'should return true when get_vm is returns an object' do + allow(subject).to receive(:get_vm).with('vm').and_return(fake_vm) + + expect(subject.vm_exists?('vm')).to eq(true) + end + + it 'should return false when get_vm is returns nil' do + allow(subject).to receive(:get_vm).with('vm').and_return(nil) + + expect(subject.vm_exists?('vm')).to eq(false) + end + end +end diff --git a/spec/vmpooler/backingservice/dummy_spec.rb b/spec/vmpooler/backingservice/dummy_spec.rb new file mode 100644 index 0000000..1d01d40 --- /dev/null +++ b/spec/vmpooler/backingservice/dummy_spec.rb @@ -0,0 +1,408 @@ +require 'spec_helper' + +describe 'Vmpooler::PoolManager::BackingService::Dummy' do + let(:logger) { double('logger') } + + let (:pool_hash) { + pool = {} + pool['name'] = 'pool1' + + pool + } + + let (:default_config) { + # Construct an initial state for testing + dummylist = {} + dummylist['pool'] = {} + # pool1 is a pool of "normal" VMs + dummylist['pool']['pool1'] = [] + # A normal running VM + vm = {} + vm['name'] = 'vm1' + vm['hostname'] = 'vm1' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool1' + vm['poolname'] = 'pool1' + vm['ready'] = true + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'RUNNING' + dummylist['pool']['pool1'] << vm + + # pool2 is a pool of "abnormal" VMs e.g. PoweredOff etc. + dummylist['pool']['pool2'] = [] + # A freshly provisioned VM that is not ready + vm = {} + vm['name'] = 'vm2' + vm['hostname'] = 'vm2' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool2' + vm['poolname'] = 'pool2' + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'UNKNOWN' + dummylist['pool']['pool2'] << vm + # A freshly provisioned VM that is running but not ready + vm = {} + vm['name'] = 'vm3' + vm['hostname'] = 'vm3' + vm['domain'] = 'dummy.local' + vm['vm_template'] = 'template1' + vm['template'] = 'pool2' + vm['poolname'] = 'pool2' + vm['ready'] = false + vm['boottime'] = Time.now + vm['powerstate'] = 'PoweredOn' + vm['vm_host'] = 'HOST1' + vm['dummy_state'] = 'RUNNING' + dummylist['pool']['pool2'] << vm + + config = {} + config['initial_state'] = dummylist + + config + } + + let (:config) { default_config } + + subject { Vmpooler::PoolManager::BackingService::Dummy.new(config) } + + before do + allow(logger).to receive(:log) + $logger = logger + end + + describe '#name' do + it 'should be dummy' do + expect(subject.name).to eq('dummy') + end + end + + describe '#vms_in_pool' do + it 'should return [] when pool does not exist' do + pool = pool_hash + pool['name'] = 'pool_does_not_exist' + + vm_list = subject.vms_in_pool(pool) + + expect(vm_list).to eq([]) + end + + it 'should return an array of VMs when pool exists' do + pool = pool_hash + pool['name'] = 'pool1' + + vm_list = subject.vms_in_pool(pool) + + expect(vm_list.count).to eq(1) + end + end + + describe '#get_vm_host' do + it 'should return the hostname when VM exists' do + expect(subject.get_vm_host('vm1')).to eq('HOST1') + end + + it 'should error when VM does not exist' do + expect{subject.get_vm_host('doesnotexist')}.to raise_error(RuntimeError) + end + end + + describe '#find_least_used_compatible_host' do + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to eq('HOST1') + end + + context 'using migratevm_couldmove_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['migratevm_couldmove_percent'] = 0 + config + } + + it 'should return the current host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to eq('HOST1') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['migratevm_couldmove_percent'] = 100 + config + } + + it 'should return a different host' do + new_host = subject.find_least_used_compatible_host('vm1') + expect(new_host).to_not eq('HOST1') + end + end + + end + end + + describe '#migrate_vm_to_host' do + it 'should move to the new host' do + expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host('vm1')).to eq('NEWHOST') + end + + context 'using migratevm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['migratevm_fail_percent'] = 0 + config + } + + it 'should move to the new host' do + expect(subject.migrate_vm_to_host('vm1','NEWHOST')).to eq(true) + expect(subject.get_vm_host('vm1')).to eq('NEWHOST') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['migratevm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.migrate_vm_to_host('vm1','NEWHOST')}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#get_vm' do + it 'should return the VM when VM exists' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOn') + expect(vm['hostname']).to eq(vm['name']) + end + + it 'should return nil when VM does not exist' do + expect(subject.get_vm('doesnotexist')).to eq(nil) + end + + context 'using getvm_poweroff_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['getvm_poweroff_percent'] = 0 + config + } + + it 'will not power off a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOn') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['getvm_poweroff_percent'] = 100 + config + } + + it 'will power off a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['powerstate']).to eq('PoweredOff') + end + end + end + + context 'using getvm_rename_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['getvm_rename_percent'] = 0 + config + } + + it 'will not rename a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['hostname']).to eq(vm['name']) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['getvm_rename_percent'] = 100 + config + } + + it 'will rename a VM' do + vm = subject.get_vm('vm1') + expect(vm['name']).to eq('vm1') + expect(vm['hostname']).to_not eq(vm['name']) + end + end + end + end + + describe '#create_vm' do + it 'should return a new VM' do + expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm') + end + + it 'should increase the number of VMs in the pool' do + pool = pool_hash + old_pool_count = subject.vms_in_pool(pool).count + new_vm = subject.create_vm(pool_hash,'newvm') + expect(subject.vms_in_pool(pool).count).to eq(old_pool_count + 1) + end + + context 'using createvm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['createvm_fail_percent'] = 0 + config + } + + it 'should return a new VM' do + expect(subject.create_vm(pool_hash,'newvm')['name']).to eq('newvm') + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['createvm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.create_vm(pool_hash,'newvm')}.to raise_error(/createvm_fail_percent/) + end + + it 'new VM should not exist' do + begin + subject.create_vm(pool_hash,'newvm') + rescue + end + expect(subject.get_vm('newvm')).to eq(nil) + end + end + end + end + + describe '#destroy_vm' do + it 'should return true when destroyed' do + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + + it 'should log if the VM is powered off' do + expect(logger).to receive(:log).with('d', "[ ] [pool1] 'vm1' is being shut down") + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.destroy_vm('doesnotexist','pool1')).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.destroy_vm('vm1','differentpool')).to eq(false) + end + + context 'using destroyvm_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['destroyvm_fail_percent'] = 0 + config + } + + it 'should return true when destroyed' do + expect(subject.destroy_vm('vm1','pool1')).to eq(true) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['destroyvm_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.destroy_vm('vm1','pool1')}.to raise_error(/migratevm_fail_percent/) + end + end + end + end + + describe '#is_vm_ready?' do + it 'should return true if ready' do + expect(subject.is_vm_ready?('vm1','pool1',0)).to eq(true) + end + + it 'should return false if VM does not exist' do + expect(subject.is_vm_ready?('doesnotexist','pool1',0)).to eq(false) + end + + it 'should return false if VM is not in the correct pool' do + expect(subject.is_vm_ready?('vm1','differentpool',0)).to eq(false) + end + + it 'should raise an error if timeout expires' do + expect{subject.is_vm_ready?('vm2','pool2',1)}.to raise_error(Timeout::Error) + end + + it 'should return true if VM becomes ready' do + expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true) + end + + context 'using vmready_fail_percent' do + describe 'of zero' do + let (:config) { + config = default_config + config['vmready_fail_percent'] = 0 + config + } + + it 'should return true if VM becomes ready' do + expect(subject.is_vm_ready?('vm3','pool2',1)).to eq(true) + end + end + + describe 'of 100' do + let (:config) { + config = default_config + config['vmready_fail_percent'] = 100 + config + } + + it 'should raise an error' do + expect{subject.is_vm_ready?('vm3','pool2',1)}.to raise_error(/vmready_fail_percent/) + end + end + end + end + + describe '#vm_exists?' do + it 'should return true when VM exists' do + expect(subject.vm_exists?('vm1')).to eq(true) + end + + it 'should return true when VM does not exist' do + expect(subject.vm_exists?('doesnotexist')).to eq(false) + end + end +end diff --git a/spec/vmpooler/pool_manager_migration_spec.rb b/spec/vmpooler/pool_manager_migration_spec.rb index 9fd491b..a4904ab 100644 --- a/spec/vmpooler/pool_manager_migration_spec.rb +++ b/spec/vmpooler/pool_manager_migration_spec.rb @@ -3,84 +3,96 @@ require 'mock_redis' require 'time' describe 'Pool Manager' do - let(:logger) { double('logger') } let(:redis) { MockRedis.new } + let(:logger) { double('logger') } let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { - 'site_name' => 'test pooler', 'migration_limit' => 2, - vsphere: { - 'server' => 'vsphere.puppet.com', - 'username' => 'vmpooler@vsphere.local', - 'password' => '', - 'insecure' => true - }, - pools: [ {'name' => 'pool1', 'size' => 5, 'folder' => 'pool1_folder'} ], - statsd: { 'prefix' => 'stats_prefix'}, - pool_names: [ 'pool1' ] } } } - let(:pool) { config[:config][:pools][0]['name'] } - let(:vm) { - { - 'name' => 'vm1', - 'host' => 'host1', - 'template' => pool, - } + let(:backingservice) { double('backingservice') } + let(:pool) { 'pool1' } + let(:vm) { 'vm1' } + let(:timeout) { 5 } + let(:host) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm } + let(:vm_host_hostname) { 'host1' } + + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + + describe "#migration_limit" do + it 'return false if config is nil' do + expect(subject.migration_limit(nil)).to equal(false) + end + it 'return false if config is 0' do + expect(subject.migration_limit(0)).to equal(false) + end + it 'return nil if config is -1' do + expect(subject.migration_limit(-1)).to equal(nil) + end + it 'return 1 if config is 1' do + expect(subject.migration_limit(1)).to equal(1) + end + it 'return 100 if config is 100' do + expect(subject.migration_limit(100)).to equal(100) + end + end describe '#_migrate_vm' do - let(:vsphere) { double(pool) } - let(:pooler) { Vmpooler::PoolManager.new(config, logger, redis, metrics) } context 'evaluates VM for migration and logs host' do before do - create_migrating_vm vm['name'], pool, redis - allow(vsphere).to receive(:find_vm).and_return(vm) - allow(pooler).to receive(:get_vm_host_info).and_return([{'name' => 'host1'}, 'host1']) + create_migrating_vm vm, pool, redis + allow(backingservice).to receive(:get_vm_host).with(vm).and_return(vm_host_hostname) end it 'logs VM host when migration is disabled' do config[:config]['migration_limit'] = nil - expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be true - expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}") + expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be true + expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) - expect(redis.sismember("vmpooler__migrating__#{pool}", vm['name'])).to be false + expect(redis.sismember("vmpooler__migrating__#{pool}", vm)).to be false end it 'verifies that migration_limit greater than or equal to migrations in progress and logs host' do - add_vm_to_migration_set vm['name'], redis + add_vm_to_migration_set vm, redis add_vm_to_migration_set 'vm2', redis - expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm['name']}' is running on #{vm['host']}. No migration will be evaluated since the migration_limit has been reached") + expect(logger).to receive(:log).with('s', "[ ] [#{pool}] '#{vm}' is running on #{vm_host_hostname}. No migration will be evaluated since the migration_limit has been reached") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) end it 'verifies that migration_limit is less than migrations in progress and logs old host, new host and migration time' do - allow(vsphere).to receive(:find_least_used_compatible_host).and_return([{'name' => 'host2'}, 'host2']) - allow(vsphere).to receive(:migrate_vm_host) + allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2') + allow(backingservice).to receive(:migrate_vm_to_host).and_return(true) expect(redis.hget("vmpooler__vm__#{vm['name']}", 'migration_time')) expect(redis.hget("vmpooler__vm__#{vm['name']}", 'checkout_to_migration')) - expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm['name']}' migrated from #{vm['host']} to host2 in 0.00 seconds") + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' migrated from #{vm_host_hostname} to host2 in 0.00 seconds") - pooler._migrate_vm(vm['name'], pool, vsphere) + subject._migrate_vm(vm, pool, backingservice) end it 'fails when no suitable host can be found' do error = 'ArgumentError: No target host found' - allow(vsphere).to receive(:find_least_used_compatible_host) - allow(vsphere).to receive(:migrate_vm_host).and_raise(error) + allow(backingservice).to receive(:find_least_used_compatible_host).and_return('host2') + allow(backingservice).to receive(:migrate_vm_to_host).and_raise(error) - expect(logger).to receive(:log).with('s', "[x] [#{pool}] '#{vm['name']}' migration failed with an error: #{error}") - - pooler._migrate_vm(vm['name'], pool, vsphere) + expect{subject._migrate_vm(vm, pool, backingservice)}.to raise_error(error) end end end diff --git a/spec/vmpooler/pool_manager_snapshot_spec.rb b/spec/vmpooler/pool_manager_snapshot_spec.rb new file mode 100644 index 0000000..c55257d --- /dev/null +++ b/spec/vmpooler/pool_manager_snapshot_spec.rb @@ -0,0 +1,83 @@ +require 'spec_helper' +require 'time' + +describe 'Pool Manager' do + let(:logger) { double('logger') } + let(:redis) { double('redis') } + let(:metrics) { Vmpooler::DummyStatsd.new } + let(:config) { {} } + let(:pool) { 'pool1' } + let(:vm) { 'vm1' } + let(:timeout) { 5 } + let(:host) { double('host') } + + subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } + + describe '#_create_vm_snapshot' do + let(:snapshot_manager) { 'snapshot_manager' } + let(:pool_helper) { double('snapshot_manager') } + let(:backingservice) { {snapshot_manager => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + context '(valid host)' do + let(:vm_host) { double('vmhost') } + + it 'creates a snapshot' do + expect(backingservice).to receive(:get_vm).and_return vm_host + expect(logger).to receive(:log) + expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion) + expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s) + expect(logger).to receive(:log) + + subject._create_vm_snapshot('testvm', 'testsnapshot', backingservice) + end + end + end + + describe '#_revert_vm_snapshot' do + let(:snapshot_manager) { 'snapshot_manager' } + let(:pool_helper) { double('snapshot_manager') } + let(:backingservice) { {snapshot_manager => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + context '(valid host)' do + let(:vm_host) { double('vmhost') } + let(:vm_snapshot) { double('vmsnapshot') } + + it 'reverts a snapshot' do + expect(backingservice).to receive(:get_vm).and_return vm_host + expect(backingservice).to receive(:find_snapshot).and_return vm_snapshot + expect(logger).to receive(:log) + expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion) + expect(logger).to receive(:log) + + subject._revert_vm_snapshot('testvm', 'testsnapshot', backingservice) + end + end + end + + describe '#_check_snapshot_queue' do + let(:pool_helper) { double('pool') } + let(:backingservice) { {pool => pool_helper} } + + before do + expect(subject).not_to be_nil + $backingservice = backingservice + end + + it 'checks appropriate redis queues' do + expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot') + expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert') + + subject._check_snapshot_queue(backingservice) + end + end +end diff --git a/spec/vmpooler/pool_manager_spec.rb b/spec/vmpooler/pool_manager_spec.rb index 6bf00bd..96b87fa 100644 --- a/spec/vmpooler/pool_manager_spec.rb +++ b/spec/vmpooler/pool_manager_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require 'time' -describe 'Pool Manager' do +describe 'Vmpooler::PoolManager' do let(:logger) { double('logger') } let(:redis) { double('redis') } let(:metrics) { Vmpooler::DummyStatsd.new } @@ -9,82 +9,397 @@ describe 'Pool Manager' do let(:pool) { 'pool1' } let(:vm) { 'vm1' } let(:timeout) { 5 } - let(:host) { double('host') } + let(:default_host) { + fake_vm = {} + fake_vm['name'] = 'vm1' + fake_vm['hostname'] = 'vm1' + fake_vm['template'] = 'pool1' + fake_vm['boottime'] = Time.now + fake_vm['powerstate'] = 'PoweredOn' + + fake_vm + } + let(:host) { default_host } + let(:max_int) { 4611686018427387903 } # A really big integer (64bit) subject { Vmpooler::PoolManager.new(config, logger, redis, metrics) } - describe '#_check_pending_vm' do - let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + describe '#check_pending_vm' do + let(:backingservice) { double('backingservice') } before do expect(subject).not_to be_nil - $vsphere = vsphere end - context 'host not in pool' do + it 'calls _check_pending_vm' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice) + + subject.check_pending_vm(vm, pool, timeout, backingservice) + end + + it 'calls fail_pending_vm if an error is raised' do + expect(Thread).to receive(:new).and_yield + allow(logger).to receive(:log) + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error') + expect(subject).to receive(:fail_pending_vm).with(vm, pool, timeout) + + expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/) + end + + it 'logs a message if an error is raised' do + expect(Thread).to receive(:new).and_yield + expect(logger).to receive(:log) + expect(subject).to receive(:_check_pending_vm).with(vm,pool,timeout,backingservice).and_raise('an_error') + allow(subject).to receive(:fail_pending_vm).with(vm, pool, timeout) + + expect{subject.check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/an_error/) + end + end + + describe '#_check_pending_vm' do + let(:backingservice) { double('backingservice') } + + before do + expect(subject).not_to be_nil + end + + context 'VM does not exist' do it 'calls fail_pending_vm' do - allow(vsphere).to receive(:find_vm).and_return(nil) + allow(backingservice).to receive(:get_vm).and_return(nil) allow(redis).to receive(:hget) - subject._check_pending_vm(vm, pool, timeout, vsphere) + subject._check_pending_vm(vm, pool, timeout, backingservice) end end - context 'host is in pool' do - let(:vm_finder) { double('vm_finder') } - let(:tcpsocket) { double('TCPSocket') } - + context 'VM is in pool and ready' do it 'calls move_pending_vm_to_ready' do - allow(subject).to receive(:open_socket).and_return(true) - allow(vsphere).to receive(:find_vm).and_return(vm_finder) - allow(vm_finder).to receive(:summary).and_return(nil) + allow(backingservice).to receive(:get_vm).with(vm).and_return(host) + allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(true) + allow(subject).to receive(:move_pending_vm_to_ready) - expect(vm_finder).to receive(:summary).once - expect(redis).not_to receive(:hget).with(String, 'clone') + subject._check_pending_vm(vm, pool, timeout, backingservice) + end + end - subject._check_pending_vm(vm, pool, timeout, vsphere) + context 'VM is in pool but not ready' do + it 'raises an error' do + allow(backingservice).to receive(:get_vm).with(vm).and_return(host) + allow(backingservice).to receive(:is_vm_ready?).with(vm,pool,Integer).and_return(false) + allow(subject).to receive(:move_pending_vm_to_ready) + + expect{subject._check_pending_vm(vm, pool, timeout, backingservice)}.to raise_error(/VM is not ready/) end end end + describe '#remove_nonexistent_vm' do + before do + expect(subject).not_to be_nil + end + + it 'removes VM from pending in redis' do + allow(logger).to receive(:log) + expect(redis).to receive(:srem).with("vmpooler__pending__#{pool}", vm) + + subject.remove_nonexistent_vm(vm, pool) + end + + it 'logs msg' do + allow(redis).to receive(:srem) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.") + + subject.remove_nonexistent_vm(vm, pool) + end + end + + describe '#fail_pending_vm' do + before do + expect(subject).not_to be_nil + allow(logger).to receive(:log) + end + + it 'takes no action if VM is not cloning' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil) + + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) + end + + it 'takes no action if VM is within timeout' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s) + + expect(subject.fail_pending_vm(vm, pool, timeout)).to eq(true) + end + + it 'moves VM to completed queue if VM has exceeded timeout and exists' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm) + + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) + end + + it 'logs message if VM has exceeded timeout and exists' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes") + + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(true) + end + + it 'calls remove_nonexistent_vm if VM has exceeded timeout and does not exist' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Date.new(2001,1,1).to_s) + expect(subject).to receive(:remove_nonexistent_vm).with(vm, pool) + + expect(subject.fail_pending_vm(vm, pool, timeout,false)).to eq(true) + end + + it 'returns false and swallows error if an error is raised' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + expect(subject.fail_pending_vm(vm, pool, timeout,true)).to eq(false) + end + + it 'logs message if an error is raised' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + expect(logger).to receive(:log).with('d', String) + + subject.fail_pending_vm(vm, pool, timeout,true) + end + end + + describe 'move_pending_vm_to_ready' do + before do + expect(subject).not_to be_nil + allow(logger).to receive(:log) + allow(Socket).to receive(:getaddrinfo) + end + + context 'when hostname does not match VM name' do + it 'should not take any action' do + expect(logger).to receive(:log).exactly(0).times + expect(Socket).to receive(:getaddrinfo).exactly(0).times + + bad_host = host + bad_host['hostname'] = 'different_name' + + subject.move_pending_vm_to_ready(vm, pool, bad_host) + end + end + + context 'when hostname matches VM name' do + it 'should use the pool in smove' do + allow(redis).to receive(:hget) + allow(redis).to receive(:hset) + expect(redis).to receive(:smove).with("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm) + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should log a message' do + allow(redis).to receive(:hget) + allow(redis).to receive(:hset) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should use clone start time to determine boot timespan' do + allow(redis).to receive(:smove) + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(Time.now.to_s) + expect(redis).to receive(:hset).with(String,pool + ':' + vm,String) + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should not determine boot timespan if clone start time not set' do + allow(redis).to receive(:smove) + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return(nil) + expect(redis).to receive(:hset).with(String,pool + ':' + vm,String).exactly(0).times + + subject.move_pending_vm_to_ready(vm, pool, host) + end + + it 'should raise error if clone start time is not parsable' do + expect(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'clone').and_return('iamnotparsable_asdate') + + expect{subject.move_pending_vm_to_ready(vm, pool, host)}.to raise_error(/iamnotparsable_asdate/) + end + end + end + + describe '#check_ready_vm' do + let(:backingservice) { double('backingservice') } + let (:ttl) { 5 } + + before do + expect(subject).not_to be_nil + end + + it 'calls _check_pending_vm' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice) + + subject.check_ready_vm(vm, pool, ttl, backingservice) + end + + it 'logs a message if an error is raised' do + expect(Thread).to receive(:new).and_yield + expect(subject).to receive(:_check_ready_vm).with(vm, pool, ttl, backingservice).and_raise('an_error') + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' failed while checking a ready vm : an_error") + + expect{subject.check_ready_vm(vm, pool, ttl, backingservice)}.to raise_error(/an_error/) + end + end + + + + + + describe '#_check_ready_vm' do + let(:backingservice) { double('backingservice') } + let (:ttl) { 5 } + + let(:config) { + config = { + 'config': {} + } + # Use the configuration defaults + config[:config]['vm_checktime'] = 15 + + config + } + + before do + expect(subject).not_to be_nil + allow(backingservice).to receive(:get_vm).and_return(host) + allow(logger).to receive(:log) + end + + context 'a VM that does not need to be checked' do + it 'should do nothing' do + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + + context 'a VM that does not exist' do + it 'should log a message' do + allow(backingservice).to receive(:get_vm).and_return(nil) + allow(redis).to receive(:srem) + expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vm}' not found in inventory for pool #{pool}, removed from 'ready' queue") + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + + it 'should remove the VM from the ready queue' do + allow(backingservice).to receive(:get_vm).and_return(nil) + allow(redis).to receive(:srem).with("vmpooler__ready__#{pool}", vm) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + + context 'an old VM' do + let (:host) { + fake_vm = default_host + fake_vm['boottime'] = Time.new(2001,1,1) + fake_vm + } + + context 'with a TTL of zero' do + it 'should do nothing' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, 0, backingservice) + end + end + + context 'with a TTL longer than the VM lifetime' do + it 'should do nothing' do + # allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + + subject._check_ready_vm(vm, pool, max_int, backingservice) + end + end + + context 'with a TTL shorter than the VM lifetime' do + it 'should move the VM to the completed queue' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(Time.now.to_s) + expect(redis).to receive(:smove).with("vmpooler__ready__#{pool}", "vmpooler__completed__#{pool}", vm) + + subject._check_ready_vm(vm, pool, 1, backingservice) + end + + it 'should log a message' do + #allow(backingservice).to receive(:get_vm).and_return(host) + allow(redis).to receive(:smove) + expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue") + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + end + end + + context 'a VM that has not previously been checked' do + before do + allow(redis).to receive(:hget).with("vmpooler__vm__#{vm}", 'check').and_return(nil) + end + + it 'sets the last check time' do + expect(redis).to receive(:hset).with("vmpooler__vm__#{vm}", 'check', Time) + allow(backingservice).to receive(:is_vm_ready?).and_return(true) + + subject._check_ready_vm(vm, pool, ttl, backingservice) + end + + end + + # TODO Need to test everything inside the check if statement + + end + + + + + + + + + + + + + + + + + + describe '#move_vm_to_ready' do before do expect(subject).not_to be_nil end context 'a host without correct summary' do - it 'does nothing when summary is nil' do - allow(host).to receive(:summary).and_return nil - subject.move_pending_vm_to_ready(vm, pool, host) - end - - it 'does nothing when guest is nil' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return nil - subject.move_pending_vm_to_ready(vm, pool, host) - end - it 'does nothing when hostName is nil' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return nil + host['hostname'] = nil + subject.move_pending_vm_to_ready(vm, pool, host) end it 'does nothing when hostName does not match vm' do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return 'adifferentvm' + host['hostname'] = 'adifferentvm' + subject.move_pending_vm_to_ready(vm, pool, host) end end context 'a host with proper summary' do before do - allow(host).to receive(:summary).and_return true - allow(host).to receive_message_chain(:summary, :guest).and_return true - allow(host).to receive_message_chain(:summary, :guest, :hostName).and_return vm - allow(redis).to receive(:hget) allow(redis).to receive(:smove) allow(redis).to receive(:hset) @@ -145,24 +460,24 @@ describe 'Pool Manager' do describe '#_check_running_vm' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice end it 'does nothing with nil host' do - allow(vsphere).to receive(:find_vm).and_return(nil) + allow(backingservice).to receive(:get_vm).and_return(nil) expect(redis).not_to receive(:smove) - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end context 'valid host' do let(:vm_host) { double('vmhost') } it 'does not move vm when not poweredOn' do - allow(vsphere).to receive(:find_vm).and_return vm_host + allow(backingservice).to receive(:get_vm).and_return vm_host allow(vm_host).to receive(:runtime).and_return true allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOff' @@ -170,11 +485,11 @@ describe 'Pool Manager' do expect(redis).not_to receive(:smove) expect(logger).not_to receive(:log).with('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead") - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end it 'moves vm when poweredOn, but past TTL' do - allow(vsphere).to receive(:find_vm).and_return vm_host + allow(backingservice).to receive(:get_vm).and_return vm_host allow(vm_host).to receive(:runtime).and_return true allow(vm_host).to receive_message_chain(:runtime, :powerState).and_return 'poweredOn' @@ -182,7 +497,7 @@ describe 'Pool Manager' do expect(redis).to receive(:smove) expect(logger).to receive(:log).with('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{timeout} hours") - subject._check_running_vm(vm, pool, timeout, vsphere) + subject._check_running_vm(vm, pool, timeout, backingservice) end end end @@ -209,7 +524,7 @@ describe 'Pool Manager' do describe '#_check_pool' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } let(:config) { { config: { task_limit: 10 }, pools: [ {'name' => 'pool1', 'size' => 5} ] @@ -217,7 +532,7 @@ describe 'Pool Manager' do before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice allow(logger).to receive(:log) allow(pool_helper).to receive(:find_folder) allow(redis).to receive(:smembers).with('vmpooler__pending__pool1').and_return([]) @@ -238,14 +553,14 @@ describe 'Pool Manager' do allow(redis).to receive(:scard).with('vmpooler__running__pool1').and_return(0) expect(logger).to receive(:log).with('s', "[!] [pool1] is empty") - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end end end describe '#_stats_running_ready' do let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } + let(:backingservice) { {pool => pool_helper} } let(:metrics) { Vmpooler::DummyStatsd.new } let(:config) { { config: { task_limit: 10 }, @@ -255,7 +570,7 @@ describe 'Pool Manager' do before do expect(subject).not_to be_nil - $vsphere = vsphere + $backingservice = backingservice allow(logger).to receive(:log) allow(pool_helper).to receive(:find_folder) allow(redis).to receive(:smembers).and_return([]) @@ -275,7 +590,7 @@ describe 'Pool Manager' do expect(metrics).to receive(:gauge).with('ready.pool1', 1) expect(metrics).to receive(:gauge).with('running.pool1', 5) - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end it 'increments metrics when ready with 0 when pool empty' do @@ -286,76 +601,8 @@ describe 'Pool Manager' do expect(metrics).to receive(:gauge).with('ready.pool1', 0) expect(metrics).to receive(:gauge).with('running.pool1', 5) - subject._check_pool(config[:pools][0], vsphere) + subject._check_pool(config[:pools][0], backingservice) end end end - - describe '#_create_vm_snapshot' do - let(:snapshot_manager) { 'snapshot_manager' } - let(:pool_helper) { double('snapshot_manager') } - let(:vsphere) { {snapshot_manager => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - context '(valid host)' do - let(:vm_host) { double('vmhost') } - - it 'creates a snapshot' do - expect(vsphere).to receive(:find_vm).and_return vm_host - expect(logger).to receive(:log) - expect(vm_host).to receive_message_chain(:CreateSnapshot_Task, :wait_for_completion) - expect(redis).to receive(:hset).with('vmpooler__vm__testvm', 'snapshot:testsnapshot', Time.now.to_s) - expect(logger).to receive(:log) - - subject._create_vm_snapshot('testvm', 'testsnapshot', vsphere) - end - end - end - - describe '#_revert_vm_snapshot' do - let(:snapshot_manager) { 'snapshot_manager' } - let(:pool_helper) { double('snapshot_manager') } - let(:vsphere) { {snapshot_manager => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - context '(valid host)' do - let(:vm_host) { double('vmhost') } - let(:vm_snapshot) { double('vmsnapshot') } - - it 'reverts a snapshot' do - expect(vsphere).to receive(:find_vm).and_return vm_host - expect(vsphere).to receive(:find_snapshot).and_return vm_snapshot - expect(logger).to receive(:log) - expect(vm_snapshot).to receive_message_chain(:RevertToSnapshot_Task, :wait_for_completion) - expect(logger).to receive(:log) - - subject._revert_vm_snapshot('testvm', 'testsnapshot', vsphere) - end - end - end - - describe '#_check_snapshot_queue' do - let(:pool_helper) { double('pool') } - let(:vsphere) { {pool => pool_helper} } - - before do - expect(subject).not_to be_nil - $vsphere = vsphere - end - - it 'checks appropriate redis queues' do - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot') - expect(redis).to receive(:spop).with('vmpooler__tasks__snapshot-revert') - - subject._check_snapshot_queue(vsphere) - end - end end diff --git a/vmpooler b/vmpooler index 20eba53..30fe321 100755 --- a/vmpooler +++ b/vmpooler @@ -5,7 +5,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) require 'rubygems' unless defined?(Gem) require 'lib/vmpooler' -config = Vmpooler.config +config = Vmpooler.config(ENV['VMPOOLER_CONFIG'] || 'vmpooler.yaml') redis_host = config[:redis]['server'] logger_file = config[:config]['logfile'] @@ -26,4 +26,11 @@ manager = Thread.new { ).execute! } +if ENV['VMPOOLER_DEBUG'] + trap("INT") { + puts "Shutting down." + [api, manager].each { |t| t.exit } + } +end + [api, manager].each { |t| t.join } diff --git a/vmpooler-new.yaml b/vmpooler-new.yaml new file mode 100644 index 0000000..2f4dd6c --- /dev/null +++ b/vmpooler-new.yaml @@ -0,0 +1,418 @@ +--- +# TODO KILL ME +:vsphere: + server: '127.0.0.1' + username: 'vmpooler' + password: 'swimsw1msw!m' + + +:backingservice: +# :backingservice: +# +# This section contains the backing services for VMs and Pools +# The currently supported backing services are: +# - vsphere +# - dummy + +# :vsphere: +# +# This section contains the server hostname and authentication credentials +# needed for vmpooler to connect to VMware vSphere. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the VMware vSphere server. +# (required) +# +# - username +# The username used to authenticate VMware vSphere. +# (required) +# +# - password +# The password used to authenticate VMware vSphere. +# (required) + +# Example: + + :vsphere: + server: '127.0.0.1' + username: 'vmpooler' + password: 'swimsw1msw!m' + +# :dummy: +# +# The dummy backing service is a simple text file service that can be used +# to test vmpooler operations in a development or test environment +# +# Available configuration parameters: +# +# - filename +# The filename used to store the backing text file + +# Example: + + :dummy: + filename: 'C:/source/vmpooler/dummy-backing.yaml' + + # createvm_max_time: 30 + # createvm_fail_percent: 5 + # migratevm_couldmove_percent: 25 + # migratevm_max_time: 10 + # migratevm_fail_percent: 5 + # vmready_fail_percent: 5 + # destroyvm_max_shutdown_time: 10 + # destroyvm_fail_percent: 50 + # destroyvm_max_time: 10 + # getvm_poweroff_percent: 10 + # getvm_rename_percent: 10 + +# :redis: +# +# This section contains the server hostname and authentication credentials +# needed for vmpooler to connect to Redis. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the Redis server. +# (optional; default: 'localhost') +# +# - username +# The username used to authenticate Redis. +# (optional) +# +# - password +# The password used to authenticate Redis. +# (optional) +# +# - data_ttl +# How long (in hours) to retain metadata in Redis after VM destruction. +# (optional; default: '168') + +# Example: + +:redis: + server: 'localhost' + + + # :graphs: + # + # This section contains the server and prefix information for a graphite- + # compatible web front-end where graphs may be viewed. This is used by the + # vmpooler dashboard to retrieve statistics and graphs for a given instance. + # + # NOTE: This is not the endpoint for publishing metrics data. See `graphite:` + # and `statsd:` below. + # + # NOTE: If `graphs:` is not set, for legacy compatibility, `graphite:` will be + # consulted for `server`/`prefix` information to use in locating a + # graph server for our dashboard. `graphs:` is recommended over + # `graphite:` + # + # + # Available configuration parameters: + # + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + + # :statsd: + # + # This section contains the connection information required to store + # historical data via statsd. This is mutually exclusive with graphite + # and takes precedence. + # + # Available configuration parameters: + # + # - server + # The FQDN hostname of the statsd daemon. + # (required) + # + # - prefix + # The prefix to use while storing statsd data. + # (optional; default: 'vmpooler') + # + # - port + # The UDP port to communicate with the statsd daemon. + # (optional; default: 8125) + + # Example: + + :statsd: + server: 'localhost' + prefix: 'vmpooler' + port: 8125 + +# :graphite: +# +# This section contains the connection information required to store +# historical data in an external Graphite database. This is mutually exclusive +# with statsd. +# +# Available configuration parameters: +# +# - server +# The FQDN hostname of the Graphite server. +# (required) +# +# - prefix +# The prefix to use while storing Graphite data. +# (optional; default: 'vmpooler') +# +# - port +# The TCP port to communicate with the graphite server. +# (optional; default: 2003) + +# Example: + +:graphite: + server: 'graphite.company.com' + +# :auth: +# +# This section contains information related to authenticating users +# for token operations. +# +# Supported Auth Providers: +# - Dummy +# - LDAP +# +# - Dummy Auth Provider +# The Dummy Authentication provider should only be used during development or testing +# If the Username and Password are different then validation succeeds +# If the Username and Password are the same then validation fails +# +# Example: +# :auth: +# provider: 'dummy' +# +# - LDAP Auth Provider +# The LDAP Authentication provider will validate usernames and passwords against an +# existing LDAP service +# +# Available configuration parameters: +# +# - host +# The FQDN hostname of the LDAP server. +# +# - port +# The port used to connect to the LDAP service. +# (optional; default: '389') +# +# - base +# The base DN used for LDAP searches. +# +# - user_object +# The LDAP object-type used to designate a user object. +# +# Example: +# :auth: +# provider: 'ldap' +# :ldap: +# host: 'localhost' +# port: 389 +# base: 'ou=users,dc=company,dc=com' +# user_object: 'uid' + +:auth: + provider: 'dummy' + +# :tagfilter: +# +# Filter tags by regular expression. + +# Example: +# +# This example demonstrates discarding everything after a '/' character for +# the 'url' tag, transforming 'foo.com/something.html' to 'foo.com'. + +:tagfilter: + url: '(.*)\/' + +# :config: +# +# This section contains global configuration information. +# +# Available configuration parameters: +# +# - site_name +# The name of your deployment. +# (optional; default: 'vmpooler') +# +# - logfile +# The path to vmpooler's log file. +# (optional; default: '/var/log/vmpooler.log') +# +# - clone_target TODO +# The target cluster VMs are cloned into (host with least VMs chosen) +# (optional; default: same cluster/host as origin template) +# +# - task_limit TODO +# The number of concurrent VMware vSphere tasks to perform. +# (optional; default: '10') +# +# - timeout +# How long (in minutes) before marking a clone as 'failed' and retrying. +# (optional; default: '15') +# +# - vm_checktime +# How often (in minutes) to check the sanity of VMs in 'ready' queues. +# (optional; default: '15') +# +# - vm_lifetime +# How long (in hours) to keep VMs in 'running' queues before destroying. +# (optional; default: '24') +# +# - vm_lifetime_auth +# Same as vm_lifetime, but applied if a valid authentication token is +# included during the request. +# +# - allowed_tags +# If set, restricts tags to those specified in this array. +# +# - domain +# If set, returns a top-level 'domain' JSON key in POST requests +# +# - prefix +# If set, prefixes all created VMs with this string. This should include +# a separator. +# (optional; default: '') +# +# - migration_limit +# When set to any value greater than 0 enable VM migration at checkout. +# When enabled this capability will evaluate a VM for migration when it is requested +# in an effort to maintain a more even distribution of load across compute resources. +# The migration_limit ensures that no more than n migrations will be evaluated at any one time +# and greatly reduces the possibilty of VMs ending up bunched together on a particular host. + +# Example: + +:config: + site_name: 'vmpooler' + logfile: 'c:/temp/vmpooler.log' + task_limit: 10 + timeout: 15 + vm_checktime: 15 + vm_lifetime: 12 + vm_lifetime_auth: 24 + # allowed_tags: + # - 'created_by' + # - 'project' + domain: 'company.com' + prefix: 'poolvm-' + migration_limit: 5 + +# :pools: +# +# This section contains a list of virtual machine 'pools' for vmpooler to +# create and maintain. +# +# Available configuration parameters (per-pool): +# +# - name +# The name of the pool. +# (required) +# +# - alias +# Other names this pool can be requested as. +# (optional) +# +# - backingservice +# The backing service which will be used to provision this pool. Default is vsphere +# (optional) +# +# - timeout +# How long (in minutes) before marking a clone as 'failed' and retrying. +# This setting overrides any globally-configured timeout setting. +# (optional; default: '15') +# +# - ready_ttl +# How long (in minutes) to keep VMs in 'ready' queues before destroying. +# (optional) +# +# - template +# The template or virtual machine target to spawn clones from. +# (required) +# +# - size +# The number of waiting VMs to keep in a pool. +# (required) + +# VSphere Backing Service configuration Options +# +# - folder +# The vSphere 'folder' destination for spawned clones. +# (required) +# +# - datastore +# The vSphere 'datastore' destination for spawned clones. +# (required) +# +# - clone_target +# Per-pool option to override the global 'clone_target' cluster. +# (optional) +# + +# Example: + +:pools: + # VSPHERE + # - name: 'debian-7-i386' + # alias: [ 'debian-7-32' ] + # template: 'Templates/debian-7-i386' + # folder: 'Pooled VMs/debian-7-i386' + # datastore: 'vmstorage' + # size: 5 + # timeout: 15 + # ready_ttl: 1440 + + # - name: 'debian-7-x86_64' + # alias: [ 'debian-7-64', 'debian-7-amd64' ] + # template: 'Templates/debian-7-x86_64' + # folder: 'Pooled VMs/debian-7-x86_64' + # datastore: 'vmstorage' + # size: 2 + # timeout: 15 + # ready_ttl: 1440 + + # DUMMY + - name: 'debian-7-i386' + template: 'test1' + backingservice: dummy + size: 1 + timeout: 15 + ready_ttl: 1440 + + - name: 'debian-7-x86_64' + template: 'test1' + backingservice: dummy + size: 1 + timeout: 15 + ready_ttl: 1440 + + # - name: 'win-2008r2-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 + + # - name: 'win-2016-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 + + # - name: 'win-2012r2-x86_64' + # template: 'test1' + # backingservice: dummy + # size: 5 + # timeout: 15 + # ready_ttl: 1440 diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example index 76dcb25..c128b5d 100644 --- a/vmpooler.yaml.example +++ b/vmpooler.yaml.example @@ -137,8 +137,22 @@ # This section contains information related to authenticating users # for token operations. # -# Currently the only supported provider is LDAP; the following parameters -# will all be under an ':ldap:' subsection (see example below). +# Supported Auth Providers: +# - Dummy +# - LDAP +# +# - Dummy Auth Provider +# The Dummy Authentication provider should only be used during development or testing +# If the Username and Password are different then validation succeeds +# If the Username and Password are the same then validation fails +# +# Example: +# :auth: +# provider: 'dummy' +# +# - LDAP Auth Provider +# The LDAP Authentication provider will validate usernames and passwords against an +# existing LDAP service # # Available configuration parameters: # @@ -154,8 +168,15 @@ # # - user_object # The LDAP object-type used to designate a user object. - +# # Example: +# :auth: +# provider: 'ldap' +# :ldap: +# host: 'localhost' +# port: 389 +# base: 'ou=users,dc=company,dc=com' +# user_object: 'uid' :auth: provider: 'ldap'