(POOLER-70) Add Pool Manager based functions to vSphere Provider

Previously the vSphere provider did not implement any of the required methods
from the base class.  This commit modifies the vSphere provider so that in
can properly implement the following methods:

- name
- vms_in_pool
- get_vm_host
- find_least_used_compatible_host
- migrate_vm_to_host
- get_vm
- create_vm
- destroy_vm
- vm_ready?
- vm_exists?
- create_disk
- create_snapshot
- revert_snapshot

This commit also includes changes to syntax for rubocop violations.

lib/vmpooler/providers/vsphere.rb

@@ -2,33 +2,287 @@ module Vmpooler
   class PoolManager
     class Provider
       class VSphere < Vmpooler::PoolManager::Provider::Base
-        def initialize(options)
+        def initialize(config, logger, metrics, name, options)
-          super(options)
+          super(config, logger, metrics, name, options)
-
+          @credentials = provider_config
-          # options will be a hash with the following keys:
+          @conf = global_config[:config]
-          #  :config  => The whole configuration object
-          #  :metrics => A metrics object
-
-          @credentials = options[:config][:vsphere]
-          @conf = options[:config][:config]
-          @metrics = options[:metrics]
         end
 
         def name
           'vsphere'
         end
 
+        def vms_in_pool(pool_name)
+          connection = get_connection
+
+          foldername = pool_config(pool_name)['folder']
+          folder_object = find_folder(foldername, connection)
+
+          vms = []
+
+          return vms if folder_object.nil?
+
+          folder_object.childEntity.each do |vm|
+            vms << { 'name' => vm.name }
+          end
+
+          vms
+        end
+
+        def get_vm_host(_pool_name, vm_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          return nil if vm_object.nil?
+
+          host_name = nil
+          host_name = vm_object.summary.runtime.host.name if vm_object.summary && vm_object.summary.runtime && vm_object.summary.runtime.host
+
+          host_name
+        end
+
+        def find_least_used_compatible_host(_pool_name, vm_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+
+          return nil if vm_object.nil?
+          host_object = find_least_used_vpshere_compatible_host(vm_object)
+
+          return nil if host_object.nil?
+          host_object[0].name
+        end
+
+        def migrate_vm_to_host(pool_name, vm_name, dest_host_name)
+          pool = pool_config(pool_name)
+          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
+
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil?
+
+          target_cluster_name = get_target_cluster_from_config(pool_name)
+          cluster = find_cluster(target_cluster_name, connection)
+          raise("Pool #{pool_name} specifies cluster #{target_cluster_name} which does not exist for the provider #{name}") if cluster.nil?
+
+          # Go through each host and initiate a migration when the correct host name is found
+          cluster.host.each do |host|
+            if host.name == dest_host_name
+              migrate_vm_host(vm_object, host)
+              return true
+            end
+          end
+
+          false
+        end
+
+        def get_vm(_pool_name, vm_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          return nil if vm_object.nil?
+
+          vm_folder_path = get_vm_folder_path(vm_object)
+          # Find the pool name based on the folder path
+          pool_name = nil
+          template_name = nil
+          global_config[:pools].each do |pool|
+            if pool['folder'] == vm_folder_path
+              pool_name = pool['name']
+              template_name = pool['template']
+            end
+          end
+
+          generate_vm_hash(vm_object, template_name, pool_name)
+        end
+
+        def create_vm(pool_name, new_vmname)
+          pool = pool_config(pool_name)
+          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
+
+          connection = get_connection
+
+          # Assume all pool config is valid i.e. not missing
+          template_path = pool['template']
+          target_folder_path = pool['folder']
+          target_datastore = pool['datastore']
+          target_cluster_name = get_target_cluster_from_config(pool_name)
+
+          # Extract the template VM name from the full path
+          raise("Pool #{pool_name} did specify a full path for the template for the provider #{name}") unless template_path =~ /\//
+          templatefolders = template_path.split('/')
+          template_name = templatefolders.pop
+
+          # Get the actual objects from vSphere
+          template_folder_object = find_folder(templatefolders.join('/'), connection)
+          raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil?
+
+          template_vm_object = template_folder_object.find(template_name)
+          raise("Pool #{pool_name} specifies a template VM of #{template_name} which does not exist for the provider #{name}") if template_vm_object.nil?
+
+          # Annotate with creation time, origin template, etc.
+          # Add extraconfig options that can be queried by vmtools
+          config_spec = RbVmomi::VIM.VirtualMachineConfigSpec(
+            annotation: JSON.pretty_generate(
+              name: new_vmname,
+              created_by: provider_config['username'],
+              base_template: template_path,
+              creation_timestamp: Time.now.utc
+            ),
+            extraConfig: [
+              { key: 'guestinfo.hostname', value: new_vmname }
+            ]
+          )
+
+          # Choose a cluster/host to place the new VM on
+          target_host_object = find_least_used_host(target_cluster_name, connection)
+
+          # Put the VM in the specified folder and resource pool
+          relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+            datastore: find_datastore(target_datastore, connection),
+            host: target_host_object,
+            diskMoveType: :moveChildMostDiskBacking
+          )
+
+          # Create a clone spec
+          clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec(
+            location: relocate_spec,
+            config: config_spec,
+            powerOn: true,
+            template: false
+          )
+
+          # Create the new VM
+          new_vm_object = template_vm_object.CloneVM_Task(
+            folder: find_folder(target_folder_path, connection),
+            name: new_vmname,
+            spec: clone_spec
+          ).wait_for_completion
+
+          generate_vm_hash(new_vm_object, template_path, pool_name)
+        end
+
+        def create_disk(pool_name, vm_name, disk_size)
+          pool = pool_config(pool_name)
+          raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
+
+          datastore_name = pool['datastore']
+          raise("Pool #{pool_name} does not have a datastore defined for the provider #{name}") if datastore_name.nil?
+
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
+
+          add_disk(vm_object, disk_size, datastore_name, connection)
+
+          true
+        end
+
+        def create_snapshot(pool_name, vm_name, new_snapshot_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
+
+          old_snap = find_snapshot(vm_object, new_snapshot_name)
+          raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil?
+
+          vm_object.CreateSnapshot_Task(
+            name: new_snapshot_name,
+            description: 'vmpooler',
+            memory: true,
+            quiesce: true
+          ).wait_for_completion
+
+          true
+        end
+
+        def revert_snapshot(pool_name, vm_name, snapshot_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
+
+          snapshot_object = find_snapshot(vm_object, snapshot_name)
+          raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil?
+
+          snapshot_object.RevertToSnapshot_Task.wait_for_completion
+
+          true
+        end
+
+        def destroy_vm(_pool_name, vm_name)
+          connection = get_connection
+
+          vm_object = find_vm(vm_name, connection)
+          # If a VM doesn't exist then it is effectively deleted
+          return true if vm_object.nil?
+
+          # Poweroff the VM if it's running
+          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
+
+          # Kill it with fire
+          vm_object.Destroy_Task.wait_for_completion
+
+          true
+        end
+
+        def vm_ready?(_pool_name, vm_name)
+          begin
+            open_socket(vm_name)
+          rescue => _err
+            return false
+          end
+
+          true
+        end
+
+        def provider_config
+          # The vSphere configuration is currently in it's own root.  This will
+          # eventually shift into the same location base expects it
+          global_config[:vsphere]
+        end
+
+        # VSphere Helper methods
+
+        def get_target_cluster_from_config(pool_name)
+          pool = pool_config(pool_name)
+          return nil if pool.nil?
+
+          return pool['clone_target'] unless pool['clone_target'].nil?
+          return global_config[:config]['clone_target'] unless global_config[:config]['clone_target'].nil?
+
+          nil
+        end
+
+        def generate_vm_hash(vm_object, template_name, pool_name)
+          hash = { 'name' => nil, 'hostname' => nil, 'template' => nil, 'poolname' => nil, 'boottime' => nil, 'powerstate' => nil }
+
+          hash['name']     = vm_object.name
+          hash['hostname'] = vm_object.summary.guest.hostName if vm_object.summary && vm_object.summary.guest && vm_object.summary.guest.hostName
+          hash['template'] = template_name
+          hash['poolname'] = pool_name
+          hash['boottime']   = vm_object.runtime.bootTime if vm_object.runtime && vm_object.runtime.bootTime
+          hash['powerstate'] = vm_object.runtime.powerState if vm_object.runtime && vm_object.runtime.powerState
+
+          hash
+        end
+
         # vSphere helper methods
-        ADAPTER_TYPE = 'lsiLogic'
+        ADAPTER_TYPE = 'lsiLogic'.freeze
-        DISK_TYPE = 'thin'
+        DISK_TYPE = 'thin'.freeze
-        DISK_MODE = 'persistent'
+        DISK_MODE = 'persistent'.freeze
 
         def get_connection
+          begin
             @connection.serviceInstance.CurrentTime
           rescue
             @connection = connect_to_vsphere @credentials
-        ensure
+          end
-          return @connection
+
+          @connection
         end
 
         def connect_to_vsphere(credentials)
@@ -40,17 +294,55 @@ module Vmpooler
                                               user: credentials['username'],
                                               password: credentials['password'],
                                               insecure: credentials['insecure'] || true
-            @metrics.increment("connect.open")
+            metrics.increment('connect.open')
             return connection
           rescue => err
             try += 1
-            @metrics.increment("connect.fail")
+            metrics.increment('connect.fail')
             raise err if try == max_tries
             sleep(try * retry_factor)
             retry
           end
         end
 
+        # This should supercede the open_socket method in the Pool Manager
+        def open_socket(host, domain = nil, timeout = 5, port = 22, &_block)
+          Timeout.timeout(timeout) do
+            target_host = host
+            target_host = "#{host}.#{domain}" if domain
+            sock = TCPSocket.new target_host, port
+            begin
+              yield sock if block_given?
+            ensure
+              sock.close
+            end
+          end
+        end
+
+        def get_vm_folder_path(vm_object)
+          # This gives an array starting from the root Datacenters folder all the way to the VM
+          # [ [Object, String], [Object, String ] ... ]
+          # It's then reversed so that it now goes from the VM to the Datacenter
+          full_path = vm_object.path.reverse
+
+          # Find the Datacenter object
+          dc_index = full_path.index { |p| p[0].is_a?(RbVmomi::VIM::Datacenter) }
+          return nil if dc_index.nil?
+          # The Datacenter should be at least 2 otherwise there's something
+          # wrong with the array passed in
+          # This is the minimum:
+          # [ VM (0), VM ROOT FOLDER (1), DC (2)]
+          return nil if dc_index <= 1
+
+          # Remove the VM name (Starting position of 1 in the slice)
+          # Up until the Root VM Folder of DataCenter Node (dc_index - 2)
+          full_path = full_path.slice(1..dc_index - 2)
+
+          # Reverse the array back to normal and
+          # then convert the array of paths into a '/' seperated string
+          (full_path.reverse.map { |p| p[1] }).join('/')
+        end
+
         def add_disk(vm, size, datastore, connection)
           return false unless size.to_i > 0
 
@@ -104,9 +396,9 @@ module Vmpooler
           datacenter.find_datastore(datastorename)
         end
 
-        def find_device(vm, deviceName)
+        def find_device(vm, device_name)
           vm.config.hardware.device.each do |device|
-            return device if device.deviceInfo.label == deviceName
+            return device if device.deviceInfo.label == device_name
           end
 
           nil
@@ -176,13 +468,11 @@ module Vmpooler
         def find_folder(foldername, connection)
           datacenter = connection.serviceInstance.find_datacenter
           base = datacenter.vmFolder
+
           folders = foldername.split('/')
           folders.each do |folder|
-            if base.is_a? RbVmomi::VIM::Folder
+            raise("Unexpected object type encountered (#{base.class}) while finding folder") unless base.is_a? RbVmomi::VIM::Folder
             base = base.childEntity.find { |f| f.name == folder }
-            else
-              raise(RuntimeError, "Unexpected object type encountered (#{base.class}) while finding folder")
-            end
           end
 
           base
@@ -192,7 +482,7 @@ module Vmpooler
         # Params:
         # +model+:: CPU arch version to match on
         # +limit+:: Hard limit for CPU or memory utilization beyond which a host is excluded for deployments
-        def get_host_utilization(host, model=nil, limit=90)
+        def get_host_utilization(host, model = nil, limit = 90)
           if model
             return nil unless host_has_cpu_model?(host, model)
           end
@@ -205,7 +495,7 @@ module Vmpooler
           return nil if cpu_utilization > limit
           return nil if memory_utilization > limit
 
-          [ cpu_utilization + memory_utilization, host ]
+          [cpu_utilization + memory_utilization, host]
         end
 
         def host_has_cpu_model?(host, model)
@@ -214,7 +504,7 @@ module Vmpooler
 
         def get_host_cpu_arch_version(host)
           cpu_model = host.hardware.cpuPkg[0].description
-          cpu_model_parts = cpu_model.split()
+          cpu_model_parts = cpu_model.split
           arch_version = cpu_model_parts[4]
           arch_version
         end
@@ -270,15 +560,14 @@ module Vmpooler
           base = datacenter.hostFolder
           pools = poolname.split('/')
           pools.each do |pool|
-            case
+            if base.is_a?(RbVmomi::VIM::Folder)
-              when base.is_a?(RbVmomi::VIM::Folder)
               base = base.childEntity.find { |f| f.name == pool }
-              when base.is_a?(RbVmomi::VIM::ClusterComputeResource)
+            elsif base.is_a?(RbVmomi::VIM::ClusterComputeResource)
               base = base.resourcePool.resourcePool.find { |f| f.name == pool }
-              when base.is_a?(RbVmomi::VIM::ResourcePool)
+            elsif base.is_a?(RbVmomi::VIM::ResourcePool)
               base = base.resourcePool.find { |f| f.name == pool }
             else
-                raise(RuntimeError, "Unexpected object type encountered (#{base.class}) while finding resource pool")
+              raise("Unexpected object type encountered (#{base.class}) while finding resource pool")
             end
           end
 
@@ -287,9 +576,7 @@ module Vmpooler
         end
 
         def find_snapshot(vm, snapshotname)
-          if vm.snapshot
+          get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname) if vm.snapshot
-            get_snapshot_list(vm.snapshot.rootSnapshotList, snapshotname)
-          end
         end
 
         def find_vm(vmname, connection)
@@ -302,11 +589,11 @@ module Vmpooler
 
         def find_vm_heavy(vmname, connection)
           vmname = vmname.is_a?(Array) ? vmname : [vmname]
-          containerView = get_base_vm_container_from(connection)
+          container_view = get_base_vm_container_from(connection)
-          propertyCollector = connection.propertyCollector
+          property_collector = connection.propertyCollector
 
-          objectSet = [{
+          object_set = [{
-            obj: containerView,
+            obj: container_view,
             skip: true,
             selectSet: [RbVmomi::VIM::TraversalSpec.new(
               name: 'gettingTheVMs',
@@ -316,15 +603,15 @@ module Vmpooler
             )]
           }]
 
-          propSet = [{
+          prop_set = [{
             pathSet: ['name'],
             type: 'VirtualMachine'
           }]
 
-          results = propertyCollector.RetrievePropertiesEx(
+          results = property_collector.RetrievePropertiesEx(
             specSet: [{
-              objectSet: objectSet,
+              objectSet: object_set,
-              propSet: propSet
+              propSet: prop_set
             }],
             options: { maxObjects: nil }
           )
@@ -337,7 +624,7 @@ module Vmpooler
           end
 
           while results.token
-            results = propertyCollector.ContinueRetrievePropertiesEx(token: results.token)
+            results = property_collector.ContinueRetrievePropertiesEx(token: results.token)
             results.objects.each do |result|
               name = result.propSet.first.val
               next unless vmname.include? name
@@ -356,7 +643,7 @@ module Vmpooler
           vm_files = vmdk_datastore._connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file'
           vm_files.keys.each do |f|
             vm_files[f]['layoutEx.file'].each do |l|
-              if l.name.match(/^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/)
+              if l.name =~ /^\[#{vmdk_datastore.name}\] #{vmname}\/#{vmname}_([0-9]+).vmdk/
                 disks.push(l)
               end
             end
@@ -366,8 +653,8 @@ module Vmpooler
         end
 
         def get_base_vm_container_from(connection)
-          viewManager = connection.serviceContent.viewManager
+          view_manager = connection.serviceContent.viewManager
-          viewManager.CreateContainerView(
+          view_manager.CreateContainerView(
             container: connection.serviceContent.rootFolder,
             recursive: true,
             type: ['VirtualMachine']

spec/rbvmomi_helper.rb

@@ -60,6 +60,22 @@ MockFolder = Struct.new(
   def children
     childEntity
   end
+
+  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/Folder.rb#L9-L12
+  def find(name, type=Object)
+    # Fake the searchIndex
+    childEntity.each do |child|
+      if child.name == name
+        if child.kind_of?(type)
+          return child
+        else
+          return nil
+        end
+      end
+    end
+
+    nil
+  end
 end
 
 MockHostSystem = Struct.new(
@@ -103,7 +119,7 @@ MockServiceInstance = Struct.new(
     # In our mocked instance, DataCenters are always in the root Folder.
     # If path is nil the first DC is returned otherwise match by name
     content.rootFolder.childEntity.each do |child|
-      if child.is_a?(MockDatacenter)
+      if child.is_a?(RbVmomi::VIM::Datacenter)
         return child if path.nil? || child.name == path
       end
     end
@@ -143,9 +159,12 @@ MockVirtualDiskManager = Object
 MockVirtualMachine = Struct.new(
   # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.VirtualMachine.html
   # From VirtualMachine
-  :config, :snapshot, :summary,
+  :config, :runtime, :snapshot, :summary,
   # From ManagedEntity
-  :name
+  :name,
+  # From RbVmomi::VIM::ManagedEntity
+  # https://github.com/vmware/rbvmomi/blob/master/lib/rbvmomi/vim/ManagedEntity.rb
+  :path
 )
 
 MockVirtualMachineSnapshot = Struct.new(
@@ -256,6 +275,12 @@ MockVirtualMachineFileLayoutExFileInfo = Struct.new(
   :key, :name, :size, :type, :uniqueSize
 )
 
+MockVirtualMachineGuestSummary = Struct.new(
+  # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.Summary.GuestSummary.html
+  # From VirtualMachineGuestSummary
+  :hostName
+)
+
 MockVirtualMachineRuntimeInfo = Struct.new(
   # https://www.vmware.com/support/developer/vc-sdk/visdk400pubs/ReferenceGuide/vim.vm.RuntimeInfo.html
   # From VirtualMachineRuntimeInfo
@@ -399,6 +424,10 @@ def mock_RbVmomi_VIM_Datacenter(options = {})
     mock.datastore << mock_ds
   end
 
+  allow(mock).to receive(:is_a?) do |expected_type|
+    expected_type == RbVmomi::VIM::Datacenter
+  end
+
   mock
 end
 
@@ -626,14 +655,21 @@ end
 def mock_RbVmomi_VIM_VirtualMachine(options = {})
   options[:snapshot_tree] = nil if options[:snapshot_tree].nil?
   options[:name] = 'VM' + rand(65536).to_s if options[:name].nil?
+  options[:path] = [] if options[:path].nil?
 
   mock = MockVirtualMachine.new()
   mock.config = MockVirtualMachineConfigInfo.new()
   mock.config.hardware = MockVirtualHardware.new([])
   mock.summary = MockVirtualMachineSummary.new()
   mock.summary.runtime = MockVirtualMachineRuntimeInfo.new()
+  mock.summary.guest = MockVirtualMachineGuestSummary.new()
+  mock.runtime = mock.summary.runtime
 
   mock.name = options[:name]
+  mock.summary.guest.hostName = options[:hostname]
+  mock.runtime.bootTime = options[:boottime]
+  mock.runtime.powerState = options[:powerstate]
+
   unless options[:snapshot_tree].nil?
     mock.snapshot = MockVirtualMachineSnapshotInfo.new()
     mock.snapshot.rootSnapshotList = []
@@ -642,6 +678,24 @@ def mock_RbVmomi_VIM_VirtualMachine(options = {})
     # Create a recursive snapshot tree
     recurse_snapshot_tree(options[:snapshot_tree],mock.snapshot.rootSnapshotList,index)
   end
+
+  # Create an array of items that describe the path of the VM from the root folder
+  # all the way to the VM itself
+  mock.path = []
+  options[:path].each do |path_item|
+    mock_item = nil
+    case path_item[:type]
+    when 'folder'
+      mock_item = mock_RbVmomi_VIM_Folder({ :name => path_item[:name] })
+    when 'datacenter'
+      mock_item = mock_RbVmomi_VIM_Datacenter({ :name => path_item[:name] })
+    else
+      raise("Unknown mock type #{path_item[:type]} for mock_RbVmomi_VIM_VirtualMachine")
+    end
+    mock.path << [mock_item,path_item[:name]]
+  end
+  mock.path << [mock,options[:name]]
+
   allow(mock).to receive(:is_a?) do |expected_type|
     expected_type == RbVmomi::VIM::VirtualMachine
   end