From 7077eadc65c24587d5fd6d1dbd19d3a3124a3d6b Mon Sep 17 00:00:00 2001
From: Glenn Sarti <glennsarti@users.noreply.github.com>
Date: Wed, 14 Jun 2017 17:39:14 -0700
Subject: [PATCH] (POOLER-83) Add ability to specify a datacenter for vsphere

Previously the vsphere provider assumed that there was one and only one
datacenter (DC) in the vsphere instance.  However this is simply not true for
many vSphere installations.  This commit:
- Adds the ability to define a vSphere datacenter at the Pool or Provider level
  whereby the Pool setting takes precedence
- If no datacenter is specified the default behaviour of picking the first DC
  in the vSphere instance
- Updated all tests for the new setting
- Update the vmpooler configuration file example with relevant setting name
  and expected behaviour
- Fixed a bug in the rvmomi_helper whereby if no DC was found it would return
  all DCs.  This is opposite behaviour of the real RBVMOMI library as it returns
  nil
---
 lib/vmpooler/providers/vsphere.rb   |  62 ++--
 spec/rbvmomi_helper.rb              |   1 +
 spec/unit/providers/vsphere_spec.rb | 547 +++++++++++++++++++++-------
 vmpooler.yaml.example               |  48 ++-
 4 files changed, 483 insertions(+), 175 deletions(-)

diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb
index c77f651..88be86d 100644
--- a/lib/vmpooler/providers/vsphere.rb
+++ b/lib/vmpooler/providers/vsphere.rb
@@ -45,7 +45,7 @@ module Vmpooler
           @connection_pool.with_metrics do |pool_object|
             connection = ensured_vsphere_connection(pool_object)
             foldername = pool_config(pool_name)['folder']
-            folder_object = find_folder(foldername, connection)
+            folder_object = find_folder(foldername, connection, get_target_datacenter_from_config(pool_name))
 
             return vms if folder_object.nil?
 
@@ -94,7 +94,7 @@ module Vmpooler
             raise("VM #{vm_name} does not exist in Pool #{pool_name} for the provider #{name}") if vm_object.nil?
 
             target_cluster_name = get_target_cluster_from_config(pool_name)
-            cluster = find_cluster(target_cluster_name, connection)
+            cluster = find_cluster(target_cluster_name, connection, get_target_datacenter_from_config(pool_name))
             raise("Pool #{pool_name} specifies cluster #{target_cluster_name} which does not exist for the provider #{name}") if cluster.nil?
 
             # Go through each host and initiate a migration when the correct host name is found
@@ -142,6 +142,7 @@ module Vmpooler
             target_folder_path = pool['folder']
             target_datastore = pool['datastore']
             target_cluster_name = get_target_cluster_from_config(pool_name)
+            target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
             # Extract the template VM name from the full path
             raise("Pool #{pool_name} did specify a full path for the template for the provider #{name}") unless template_path =~ /\//
@@ -149,7 +150,7 @@ module Vmpooler
             template_name = templatefolders.pop
 
             # Get the actual objects from vSphere
-            template_folder_object = find_folder(templatefolders.join('/'), connection)
+            template_folder_object = find_folder(templatefolders.join('/'), connection, target_datacenter_name)
             raise("Pool #{pool_name} specifies a template folder of #{templatefolders.join('/')} which does not exist for the provider #{name}") if template_folder_object.nil?
 
             template_vm_object = template_folder_object.find(template_name)
@@ -170,11 +171,11 @@ module Vmpooler
             )
 
             # Choose a cluster/host to place the new VM on
-            target_host_object = find_least_used_host(target_cluster_name, connection)
+            target_host_object = find_least_used_host(target_cluster_name, connection, target_datacenter_name)
 
             # Put the VM in the specified folder and resource pool
             relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-              datastore: find_datastore(target_datastore, connection),
+              datastore: find_datastore(target_datastore, connection, target_datacenter_name),
               host: target_host_object,
               diskMoveType: :moveChildMostDiskBacking
             )
@@ -189,7 +190,7 @@ module Vmpooler
 
             # Create the new VM
             new_vm_object = template_vm_object.CloneVM_Task(
-              folder: find_folder(target_folder_path, connection),
+              folder: find_folder(target_folder_path, connection, target_datacenter_name),
               name: new_vmname,
               spec: clone_spec
             ).wait_for_completion
@@ -211,7 +212,7 @@ module Vmpooler
             vm_object = find_vm(vm_name, connection)
             raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil?
 
-            add_disk(vm_object, disk_size, datastore_name, connection)
+            add_disk(vm_object, disk_size, datastore_name, connection, get_target_datacenter_from_config(pool_name))
           end
           true
         end
@@ -287,6 +288,16 @@ module Vmpooler
           nil
         end
 
+        def get_target_datacenter_from_config(pool_name)
+          pool = pool_config(pool_name)
+          return nil if pool.nil?
+
+          return pool['datacenter']            unless pool['datacenter'].nil?
+          return provider_config['datacenter'] unless provider_config['datacenter'].nil?
+
+          nil
+        end
+
         def generate_vm_hash(vm_object, template_name, pool_name)
           hash = { 'name' => nil, 'hostname' => nil, 'template' => nil, 'poolname' => nil, 'boottime' => nil, 'powerstate' => nil }
 
@@ -375,11 +386,12 @@ module Vmpooler
           (full_path.reverse.map { |p| p[1] }).join('/')
         end
 
-        def add_disk(vm, size, datastore, connection)
+        def add_disk(vm, size, datastore, connection, datacentername)
           return false unless size.to_i > 0
 
-          vmdk_datastore = find_datastore(datastore, connection)
-          vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore, connection).length + 1}.vmdk"
+          vmdk_datastore = find_datastore(datastore, connection, datacentername)
+          raise("Datastore '#{datastore}' does not exist in datacenter '#{datacentername}'") if vmdk_datastore.nil?
+          vmdk_file_name = "#{vm['name']}/#{vm['name']}_#{find_vmdks(vm['name'], datastore, connection, datacentername).length + 1}.vmdk"
 
           controller = find_disk_controller(vm)
 
@@ -413,7 +425,7 @@ module Vmpooler
           )
 
           connection.serviceContent.virtualDiskManager.CreateVirtualDisk_Task(
-            datacenter: connection.serviceInstance.find_datacenter,
+            datacenter: connection.serviceInstance.find_datacenter(datacentername),
             name: "[#{vmdk_datastore.name}] #{vmdk_file_name}",
             spec: vmdk_spec
           ).wait_for_completion
@@ -423,8 +435,9 @@ module Vmpooler
           true
         end
 
-        def find_datastore(datastorename, connection)
-          datacenter = connection.serviceInstance.find_datacenter
+        def find_datastore(datastorename, connection, datacentername)
+          datacenter = connection.serviceInstance.find_datacenter(datacentername)
+          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
           datacenter.find_datastore(datastorename)
         end
 
@@ -497,8 +510,9 @@ module Vmpooler
           available_unit_numbers.sort[0]
         end
 
-        def find_folder(foldername, connection)
-          datacenter = connection.serviceInstance.find_datacenter
+        def find_folder(foldername, connection, datacentername)
+          datacenter = connection.serviceInstance.find_datacenter(datacentername)
+          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
           base = datacenter.vmFolder
 
           folders = foldername.split('/')
@@ -558,16 +572,17 @@ module Vmpooler
           (memory_usage.to_f / memory_size.to_f) * 100
         end
 
-        def find_least_used_host(cluster, connection)
-          cluster_object = find_cluster(cluster, connection)
+        def find_least_used_host(cluster, connection, datacentername)
+          cluster_object = find_cluster(cluster, connection, datacentername)
           target_hosts = get_cluster_host_utilization(cluster_object)
           raise("There is no host candidate in vcenter that meets all the required conditions, check that the cluster has available hosts in a 'green' status, not in maintenance mode and not overloaded CPU and memory'") if target_hosts.empty?
           least_used_host = target_hosts.sort[0][1]
           least_used_host
         end
 
-        def find_cluster(cluster, connection)
-          datacenter = connection.serviceInstance.find_datacenter
+        def find_cluster(cluster, connection, datacentername)
+          datacenter = connection.serviceInstance.find_datacenter(datacentername)
+          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
           datacenter.hostFolder.children.find { |cluster_object| cluster_object.name == cluster }
         end
 
@@ -590,8 +605,9 @@ module Vmpooler
           [target_host, target_host.name]
         end
 
-        def find_pool(poolname, connection)
-          datacenter = connection.serviceInstance.find_datacenter
+        def find_pool(poolname, connection, datacentername)
+          datacenter = connection.serviceInstance.find_datacenter(datacentername)
+          raise("Datacenter #{datacentername} does not exist") if datacenter.nil?
           base = datacenter.hostFolder
           pools = poolname.split('/')
           pools.each do |pool|
@@ -670,10 +686,10 @@ module Vmpooler
           vms
         end
 
-        def find_vmdks(vmname, datastore, connection)
+        def find_vmdks(vmname, datastore, connection, datacentername)
           disks = []
 
-          vmdk_datastore = find_datastore(datastore, connection)
+          vmdk_datastore = find_datastore(datastore, connection,datacentername)
 
           vm_files = connection.serviceContent.propertyCollector.collectMultiple vmdk_datastore.vm, 'layoutEx.file'
           vm_files.keys.each do |f|
diff --git a/spec/rbvmomi_helper.rb b/spec/rbvmomi_helper.rb
index 375fd56..dec86d1 100644
--- a/spec/rbvmomi_helper.rb
+++ b/spec/rbvmomi_helper.rb
@@ -123,6 +123,7 @@ MockServiceInstance = Struct.new(
         return child if path.nil? || child.name == path
       end
     end
+    nil
   end
 end
 
diff --git a/spec/unit/providers/vsphere_spec.rb b/spec/unit/providers/vsphere_spec.rb
index 224166b..68af47b 100644
--- a/spec/unit/providers/vsphere_spec.rb
+++ b/spec/unit/providers/vsphere_spec.rb
@@ -42,6 +42,7 @@ describe 'Vmpooler::PoolManager::Provider::VSphere' do
   let(:metrics) { Vmpooler::DummyStatsd.new }
   let(:poolname) { 'pool1'}
   let(:provider_options) { { 'param' => 'value' } }
+  let(:datacenter_name) { 'MockDC' }
   let(:config) { YAML.load(<<-EOT
 ---
 :config:
@@ -55,6 +56,7 @@ describe 'Vmpooler::PoolManager::Provider::VSphere' do
     insecure: true
     # Drop the connection pool timeout way down for spec tests so they fail fast
     connection_pool_timeout: 1
+    datacenter: MockDC
 :pools:
   - name: '#{poolname}'
     alias: [ 'mockpool' ]
@@ -95,7 +97,7 @@ EOT
 
     context 'Given a pool folder that is missing' do
       before(:each) do
-        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection).and_return(nil)
+        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection,datacenter_name).and_return(nil)
       end
 
       it 'should get a connection' do
@@ -113,7 +115,7 @@ EOT
 
     context 'Given an empty pool folder' do
       before(:each) do
-        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection).and_return(folder_object)
+        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection,datacenter_name).and_return(folder_object)
       end
 
       it 'should get a connection' do
@@ -142,7 +144,7 @@ EOT
           folder_object.childEntity << mock_vm
         end
 
-        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection).and_return(folder_object)
+        expect(subject).to receive(:find_folder).with(pool_config['folder'],connection,datacenter_name).and_return(folder_object)
       end
 
       it 'should get a connection' do
@@ -324,7 +326,7 @@ EOT
 
       before(:each) do
         config[:pools][0]['clone_target'] = cluster_name
-        expect(subject).to receive(:find_cluster).with(cluster_name,connection).and_return(nil)
+        expect(subject).to receive(:find_cluster).with(cluster_name,connection,datacenter_name).and_return(nil)
       end
 
       it 'should raise an error' do
@@ -338,7 +340,7 @@ EOT
       before(:each) do
         config[:pools][0]['clone_target'] = nil
         config[:config]['clone_target'] = cluster_name
-        expect(subject).to receive(:find_cluster).with(cluster_name,connection).and_return(nil)
+        expect(subject).to receive(:find_cluster).with(cluster_name,connection,datacenter_name).and_return(nil)
       end
 
       it 'should raise an error' do
@@ -352,7 +354,7 @@ EOT
         mock_cluster = mock_RbVmomi_VIM_ComputeResource({
           :hosts => [ { :name => 'HOST001' },{ :name => dest_host_name} ]
         })
-        expect(subject).to receive(:find_cluster).with(cluster_name,connection).and_return(mock_cluster)
+        expect(subject).to receive(:find_cluster).with(cluster_name,connection,datacenter_name).and_return(mock_cluster)
         expect(subject).to receive(:migrate_vm_host).exactly(0).times
       end
 
@@ -367,7 +369,7 @@ EOT
         mock_cluster = mock_RbVmomi_VIM_ComputeResource({
           :hosts => [ { :name => 'HOST001' },{ :name => dest_host_name} ]
         })
-        expect(subject).to receive(:find_cluster).with(cluster_name,connection).and_return(mock_cluster)
+        expect(subject).to receive(:find_cluster).with(cluster_name,connection,datacenter_name).and_return(mock_cluster)
         expect(subject).to receive(:migrate_vm_host).with(Object,Object).and_raise(RuntimeError,'MockMigrationError')
       end
 
@@ -382,7 +384,7 @@ EOT
         mock_cluster = mock_RbVmomi_VIM_ComputeResource({
           :hosts => [ { :name => 'HOST001' },{ :name => dest_host_name} ]
         })
-        expect(subject).to receive(:find_cluster).with(cluster_name,connection).and_return(mock_cluster)
+        expect(subject).to receive(:find_cluster).with(cluster_name,connection,datacenter_name).and_return(mock_cluster)
         expect(subject).to receive(:migrate_vm_host).with(Object,Object).and_return(nil)
       end
 
@@ -558,7 +560,7 @@ EOT
 
     context 'Given a successful creation' do
       before(:each) do
-        template_vm = subject.find_folder('Templates',connection).find('pool1')
+        template_vm = subject.find_folder('Templates',connection,datacenter_name).find('pool1')
         allow(template_vm).to receive(:CloneVM_Task).and_return(clone_vm_task)
         allow(clone_vm_task).to receive(:wait_for_completion).and_return(new_vm_object)
       end
@@ -570,7 +572,7 @@ EOT
       end
 
       it 'should use the appropriate Create_VM spec' do
-        template_vm = subject.find_folder('Templates',connection).find('pool1')
+        template_vm = subject.find_folder('Templates',connection,datacenter_name).find('pool1')
         expect(template_vm).to receive(:CloneVM_Task)
           .with(create_vm_spec(vmname,'pool1','datastore0'))
           .and_return(clone_vm_task)
@@ -633,7 +635,7 @@ EOT
 
     context 'when adding the disk succeeds' do
       before(:each) do
-        expect(subject).to receive(:add_disk).with(vm_object, disk_size, datastorename, connection)
+        expect(subject).to receive(:add_disk).with(vm_object, disk_size, datastorename, connection, datacenter_name)
       end
 
       it 'should return true' do
@@ -881,6 +883,92 @@ EOT
     end
   end
 
+  # vSphere helper methods
+  describe '#get_target_datacenter_from_config' do
+    let(:pool_dc) { 'PoolDC'}
+    let(:provider_dc) { 'ProvDC'}
+
+    context 'when not specified' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :vsphere:
+    server: "vcenter.domain.local"
+    username: "vcenter_user"
+    password: "vcenter_password"
+:pools:
+  - name: '#{poolname}'
+EOT
+        )
+      }
+      it 'returns nil' do
+        expect(subject.get_target_datacenter_from_config(poolname)).to be_nil
+      end
+    end
+
+    context 'when specified only in the pool' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :vsphere:
+    server: "vcenter.domain.local"
+    username: "vcenter_user"
+    password: "vcenter_password"
+:pools:
+  - name: '#{poolname}'
+    datacenter: '#{pool_dc}'
+EOT
+        )
+      }
+      it 'returns the pool datacenter' do
+        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
+      end
+    end
+
+    context 'when specified only in the provider' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :vsphere:
+    server: "vcenter.domain.local"
+    username: "vcenter_user"
+    password: "vcenter_password"
+    datacenter: '#{provider_dc}'
+:pools:
+  - name: '#{poolname}'
+EOT
+        )
+      }
+      it 'returns the provider datacenter' do
+        expect(subject.get_target_datacenter_from_config(poolname)).to eq(provider_dc)
+      end
+    end
+
+    context 'when specified in the provider and pool' do
+      let(:config) { YAML.load(<<-EOT
+---
+:config:
+:providers:
+  :vsphere:
+    server: "vcenter.domain.local"
+    username: "vcenter_user"
+    password: "vcenter_password"
+    datacenter: '#{provider_dc}'
+:pools:
+  - name: '#{poolname}'
+    datacenter: '#{pool_dc}'
+EOT
+        )
+      }
+      it 'returns the pool datacenter' do
+        expect(subject.get_target_datacenter_from_config(poolname)).to eq(pool_dc)
+      end
+    end
+  end
+
   # vSphere helper methods
   describe '#ensured_vsphere_connection' do
     let(:config) { YAML.load(<<-EOT
@@ -1227,7 +1315,7 @@ EOT
     let(:connection_options) {{
       :serviceContent => {
         :datacenters => [
-          { :name => 'MockDC', :datastores => [datastorename] }
+          { :name => datacenter_name, :datastores => [datastorename] }
         ]
       }
     }}
@@ -1240,7 +1328,11 @@ EOT
       allow(connection.serviceContent.propertyCollector).to receive(:collectMultiple).and_return(collectMultiple_response)
 
       # Mocking for creating the disk
-      allow(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task).and_return(create_virtual_disk_task)
+      allow(connection.serviceContent.virtualDiskManager).to receive(:CreateVirtualDisk_Task) do |options|
+        if options[:datacenter][:name] == datacenter_name
+          create_virtual_disk_task
+        end
+      end
       allow(create_virtual_disk_task).to receive(:wait_for_completion).and_return(true)
 
       # Mocking for adding disk to the VM
@@ -1250,7 +1342,7 @@ EOT
 
     context 'Succesfully addding disk' do
       it 'should return true' do
-        expect(subject.add_disk(vm_object,disk_size,datastorename,connection)).to be true
+        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
       end
 
       it 'should request a disk of appropriate size' do
@@ -1259,13 +1351,13 @@ EOT
           .and_return(create_virtual_disk_task)
 
 
-        subject.add_disk(vm_object,disk_size,datastorename,connection)
+        subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)
       end
     end
 
     context 'Requested disk size is 0' do
       it 'should raise an error' do
-        expect(subject.add_disk(vm_object,0,datastorename,connection)).to be false
+        expect(subject.add_disk(vm_object,0,datastorename,connection,datacenter_name)).to be false
       end
     end
 
@@ -1273,13 +1365,28 @@ EOT
       let(:connection_options) {{
         :serviceContent => {
           :datacenters => [
-            { :name => 'MockDC', :datastores => ['missing_datastore'] }
+            { :name => datacenter_name, :datastores => ['missing_datastore'] }
           ]
         }
       }}
 
-      it 'should return false' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection) }.to raise_error(NoMethodError)
+      it 'should raise error' do
+        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(/does not exist/)
+      end
+    end
+
+    context 'Multiple datacenters with multiple datastores' do
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => 'AnotherDC', :datastores => ['dc1','dc2'] },
+            { :name => datacenter_name, :datastores => ['dc3',datastorename,'dc4'] },
+          ]
+        }
+      }}
+
+      it 'should return true' do
+        expect(subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name)).to be true
       end
     end
 
@@ -1293,7 +1400,7 @@ EOT
       }
 
       it 'should raise an error' do
-        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection) }.to raise_error(NoMethodError)
+        expect{ subject.add_disk(vm_object,disk_size,datastorename,connection,datacenter_name) }.to raise_error(NoMethodError)
       end
     end
   end
@@ -1306,13 +1413,13 @@ EOT
       let(:connection_options) {{
         :serviceContent => {
           :datacenters => [
-            { :name => 'MockDC', :datastores => [] }
+            { :name => datacenter_name, :datastores => [] }
           ]
         }
       }}
 
       it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore(datastorename,connection)
+        result = subject.find_datastore(datastorename,connection,datacenter_name)
         expect(result).to be_nil
       end
     end
@@ -1321,18 +1428,42 @@ EOT
       let(:connection_options) {{
         :serviceContent => {
           :datacenters => [
-            { :name => 'MockDC', :datastores => ['ds1','ds2',datastorename,'ds3'] }
+            { :name => datacenter_name, :datastores => ['ds1','ds2',datastorename,'ds3'] }
           ]
         }
       }}
 
       it 'should return nil if the datastore is not found' do
-        result = subject.find_datastore('missing_datastore',connection)
+        result = subject.find_datastore('missing_datastore',connection,datacenter_name)
         expect(result).to be_nil
       end
 
       it 'should find the datastore in the datacenter' do
-        result = subject.find_datastore(datastorename,connection)
+        result = subject.find_datastore(datastorename,connection,datacenter_name)
+        
+        expect(result).to_not be_nil
+        expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
+        expect(result.name).to eq(datastorename)
+      end
+    end
+
+    context 'Many datastores in many datacenters' do
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => 'AnotherDC', :datastores => ['ds1','ds2','ds3'] },
+            { :name => datacenter_name, :datastores => ['ds3','ds4',datastorename,'ds5'] },
+          ]
+        }
+      }}
+
+      it 'should return nil if the datastore is not found' do
+        result = subject.find_datastore(datastorename,connection,'AnotherDC')
+        expect(result).to be_nil
+      end
+
+      it 'should find the datastore in the datacenter' do
+        result = subject.find_datastore(datastorename,connection,datacenter_name)
         
         expect(result).to_not be_nil
         expect(result.is_a?(RbVmomi::VIM::Datastore)).to be true
@@ -1558,54 +1689,102 @@ EOT
     let(:foldername) { 'folder'}
     let(:missing_foldername) { 'missing_folder'}
 
-    before(:each) do
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
     context 'with no folder hierarchy' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name }
+          ]
+        }
+      }}
 
       it 'should return nil if the folder is not found' do
-        expect(subject.find_folder(missing_foldername,connection)).to be_nil
+        expect(subject.find_folder(missing_foldername,connection,datacenter_name)).to be_nil
       end
     end
 
     context 'with a single layer folder hierarchy' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :vmfolder_tree => {
-          'folder1' => nil,
-          'folder2' => nil,
-          foldername => nil,
-          'folder3' => nil,
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :vmfolder_tree => {
+                'folder1' => nil,
+                'folder2' => nil,
+                foldername => nil,
+                'folder3' => nil,
+              }
+            }
+          ]
         }
-      }) }
+      }}
 
       it 'should return the folder when found' do
-        result = subject.find_folder(foldername,connection)
+        result = subject.find_folder(foldername,connection,datacenter_name)
         expect(result).to_not be_nil
         expect(result.name).to eq(foldername)
       end
 
       it 'should return nil if the folder is not found' do
-        expect(subject.find_folder(missing_foldername,connection)).to be_nil
+        expect(subject.find_folder(missing_foldername,connection,datacenter_name)).to be_nil
+      end
+    end
+
+    context 'with a single layer folder hierarchy in many datacenters' do
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => 'AnotherDC',
+              :vmfolder_tree => {
+                'folder1' => nil,
+                'folder2' => nil,
+                'folder3' => nil,
+              }
+            },
+            { :name => datacenter_name,
+              :vmfolder_tree => {
+                'folder4' => nil,
+                'folder5' => nil,
+                foldername => nil,
+                'folder6' => nil,
+              }
+            }
+          ]
+        }
+      }}
+
+      it 'should return the folder when found' do
+        result = subject.find_folder(foldername,connection,datacenter_name)
+        expect(result).to_not be_nil
+        expect(result.name).to eq(foldername)
+      end
+
+      it 'should return nil if the folder is not found' do
+        expect(subject.find_folder(missing_foldername,connection,'AnotherDC')).to be_nil
       end
     end
 
     context 'with a VM with the same name as a folder in a single layer folder hierarchy' do
       # The folder hierarchy should include a VM with same name as folder, and appear BEFORE the
       # folder in the child list.
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :vmfolder_tree => {
-          'folder1' => nil,
-          'vm1' => { :object_type => 'vm', :name => foldername },
-          foldername => nil,
-          'folder3' => nil,
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :vmfolder_tree => {
+                'folder1' => nil,
+                'vm1' => { :object_type => 'vm', :name => foldername },
+                foldername => nil,
+                'folder3' => nil,
+              }
+            }
+          ]
         }
-      }) }
+      }}
 
       it 'should not return a VM' do
         pending('https://github.com/puppetlabs/vmpooler/issues/204')
-        result = subject.find_folder(foldername,connection)
+        result = subject.find_folder(foldername,connection,datacenter_name)
         expect(result).to_not be_nil
         expect(result.name).to eq(foldername)
         expect(result.is_a? RbVmomi::VIM::VirtualMachine).to be false
@@ -1615,31 +1794,37 @@ EOT
     context 'with a multi layer folder hierarchy' do
       let(:end_folder_name) { 'folder'}
       let(:foldername) { 'folder2/folder4/' + end_folder_name}
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :vmfolder_tree => {
-          'folder1' => nil,
-          'folder2' => {
-            :children => {
-              'folder3' => nil,
-              'folder4' => {
-                :children => {
-                  end_folder_name => nil,
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :vmfolder_tree => {
+                'folder1' => nil,
+                'folder2' => {
+                  :children => {
+                    'folder3' => nil,
+                    'folder4' => {
+                      :children => {
+                        end_folder_name => nil,
+                      },
+                    }
+                  },
                 },
+                'folder5' => nil,
               }
-            },
-          },
-          'folder5' => nil,
+            }
+          ]
         }
-      }) }
+      }}
 
       it 'should return the folder when found' do
-        result = subject.find_folder(foldername,connection)
+        result = subject.find_folder(foldername,connection,datacenter_name)
         expect(result).to_not be_nil
         expect(result.name).to eq(end_folder_name)
       end
 
       it 'should return nil if the folder is not found' do
-        expect(subject.find_folder(missing_foldername,connection)).to be_nil
+        expect(subject.find_folder(missing_foldername,connection,datacenter_name)).to be_nil
       end
     end
 
@@ -1648,30 +1833,36 @@ EOT
       # and appear BEFORE the folder in the child list.
       let(:end_folder_name) { 'folder'}
       let(:foldername) { 'folder2/folder4/' + end_folder_name}
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :vmfolder_tree => {
-          'folder1' => nil,
-          'folder2' => {
-            :children => {
-              'folder3' => nil,
-              'vm1' => { :object_type => 'vm', :name => 'folder4' },
-              'folder4' => {
-                :children => {
-                  end_folder_name => nil,
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :vmfolder_tree => {
+                'folder1' => nil,
+                'folder2' => {
+                  :children => {
+                    'folder3' => nil,
+                    'vm1' => { :object_type => 'vm', :name => 'folder4' },
+                    'folder4' => {
+                      :children => {
+                        end_folder_name => nil,
+                      },
+                    }
+                  },
                 },
+                'folder5' => nil,
               }
-            },
-          },
-          'folder5' => nil,
+            }
+          ]
         }
-      }) }
+      }}
 
       it 'should not return a VM' do
         pending('https://github.com/puppetlabs/vmpooler/issues/204')
-        result = subject.find_folder(foldername,connection)
+        result = subject.find_folder(foldername,connection,datacenter_name)
         expect(result).to_not be_nil
         expect(result.name).to eq(foldername)
-        expect(result.is_a? RbVmomi::VIM::VirtualMachine).to be false
+        expect(result.is_a? RbVmomi::VIM::VirtualMachine,datacenter_name).to be false
       end
     end
   end
@@ -1920,9 +2111,9 @@ EOT
           :name => cluster_name,
       }]})}
       let(:expected_host) { cluster_object.host[0] }
-
+#,datacenter_name
       it 'should raise an error' do
-        expect{subject.find_least_used_host(missing_cluster_name,connection)}.to raise_error(NoMethodError,/undefined method/)
+        expect{subject.find_least_used_host(missing_cluster_name,connection,datacenter_name)}.to raise_error(NoMethodError,/undefined method/)
       end
     end
 
@@ -1935,7 +2126,7 @@ EOT
       let(:expected_host) { cluster_object.host[0] }
 
       it 'should return the standalone host' do
-        result = subject.find_least_used_host(cluster_name,connection)
+        result = subject.find_least_used_host(cluster_name,connection,datacenter_name)
 
         expect(result).to be(expected_host)
       end
@@ -1951,7 +2142,7 @@ EOT
       let(:expected_host) { cluster_object.host[0] }
 
       it 'should raise an error' do
-        expect{subject.find_least_used_host(missing_cluster_name,connection)}.to raise_error(NoMethodError,/undefined method/)
+        expect{subject.find_least_used_host(missing_cluster_name,connection,datacenter_name)}.to raise_error(NoMethodError,/undefined method/)
       end
     end
 
@@ -1966,7 +2157,7 @@ EOT
       let(:expected_host) { cluster_object.host[1] }
 
       it 'should return the standalone host' do
-        result = subject.find_least_used_host(cluster_name,connection)
+        result = subject.find_least_used_host(cluster_name,connection,datacenter_name)
 
         expect(result).to be(expected_host)
       end
@@ -1983,7 +2174,7 @@ EOT
       let(:expected_host) { cluster_object.host[1] }
 
       it 'should raise an error' do
-        expect{subject.find_least_used_host(missing_cluster_name,connection)}.to raise_error(NoMethodError,/undefined method/)
+        expect{subject.find_least_used_host(missing_cluster_name,connection,datacenter_name)}.to raise_error(NoMethodError,/undefined method/)
       end
     end
 
@@ -2000,7 +2191,7 @@ EOT
       let(:expected_host) { cluster_object.host[1] }
 
       it 'should return the standalone host' do
-        result = subject.find_least_used_host(cluster_name,connection)
+        result = subject.find_least_used_host(cluster_name,connection,datacenter_name)
 
         expect(result).to be(expected_host)
       end
@@ -2018,7 +2209,7 @@ EOT
 
       it 'should return a host' do
         pending('https://github.com/puppetlabs/vmpooler/issues/206')
-        result = subject.find_least_used_host(missing_cluster_name,connection)
+        result = subject.find_least_used_host(missing_cluster_name,connection,datacenter_name)
         expect(result).to_not be_nil
       end
     end
@@ -2028,66 +2219,109 @@ EOT
     let(:cluster) {'cluster'}
     let(:missing_cluster) {'missing_cluster'}
 
-    before(:each) do
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
     context 'no clusters in the datacenter' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
-
-      before(:each) do
-      end
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name }
+          ]
+        }
+      }}
 
       it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection)).to be_nil
+        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
       end
     end
 
     context 'with a single layer folder hierarchy' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :hostfolder_tree => {
-          'cluster1' =>  {:object_type => 'compute_resource'},
-          'cluster2' => {:object_type => 'compute_resource'},
-          cluster => {:object_type => 'compute_resource'},
-          'cluster3' => {:object_type => 'compute_resource'},
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :hostfolder_tree => {
+                'cluster1' =>  {:object_type => 'compute_resource'},
+                'cluster2' => {:object_type => 'compute_resource'},
+                cluster => {:object_type => 'compute_resource'},
+                'cluster3' => {:object_type => 'compute_resource'},
+              }
+            }
+          ]
         }
-      }) }
+      }}
 
       it 'should return the cluster when found' do
-        result = subject.find_cluster(cluster,connection)
+        result = subject.find_cluster(cluster,connection,datacenter_name)
 
         expect(result).to_not be_nil
         expect(result.name).to eq(cluster)
       end
 
       it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection)).to be_nil
+        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
+      end
+    end
+
+    context 'with a single layer folder hierarchy with multiple datacenters' do
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => 'AnotherDC',
+              :hostfolder_tree => {
+                'cluster1' =>  {:object_type => 'compute_resource'},
+                'cluster2' => {:object_type => 'compute_resource'},
+              }
+            },
+            { :name => datacenter_name,
+              :hostfolder_tree => {
+                cluster => {:object_type => 'compute_resource'},
+                'cluster3' => {:object_type => 'compute_resource'},
+              }
+            }
+          ]
+        }
+      }}
+
+      it 'should return the cluster when found' do
+        result = subject.find_cluster(cluster,connection,datacenter_name)
+
+        expect(result).to_not be_nil
+        expect(result.name).to eq(cluster)
+      end
+
+      it 'should return nil if the cluster is not found' do
+        expect(subject.find_cluster(missing_cluster,connection,'AnotherDC')).to be_nil
       end
     end
 
     context 'with a multi layer folder hierarchy' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({
-        :hostfolder_tree => {
-          'cluster1' =>  {:object_type => 'compute_resource'},
-          'folder2' => {
-            :children => {
-              cluster => {:object_type => 'compute_resource'},
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name,
+              :hostfolder_tree => {
+                'cluster1' =>  {:object_type => 'compute_resource'},
+                'folder2' => {
+                  :children => {
+                    cluster => {:object_type => 'compute_resource'},
+                  }
+                },
+                'cluster3' => {:object_type => 'compute_resource'},
+              }
             }
-          },
-          'cluster3' => {:object_type => 'compute_resource'},
+          ]
         }
-      }) }
+      }}
 
       it 'should return the cluster when found' do
         pending('https://github.com/puppetlabs/vmpooler/issues/205')
-        result = subject.find_cluster(cluster,connection)
+        result = subject.find_cluster(cluster,connection,datacenter_name)
 
         expect(result).to_not be_nil
         expect(result.name).to eq(cluster)
       end
 
       it 'should return nil if the cluster is not found' do
-        expect(subject.find_cluster(missing_cluster,connection)).to be_nil
+        expect(subject.find_cluster(missing_cluster,connection,datacenter_name)).to be_nil
       end
     end
   end
@@ -2287,23 +2521,56 @@ EOT
     let(:poolname) { 'pool'}
     let(:missing_poolname) { 'missing_pool'}
 
-    before(:each) do
-      allow(connection.serviceInstance).to receive(:find_datacenter).and_return(datacenter_object)
-    end
-
     context 'with empty folder hierarchy' do
-      let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter() }
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => datacenter_name }
+          ] 
+        }
+      }}
 
       it 'should ensure the connection' do
         pending('https://github.com/puppetlabs/vmpooler/issues/209')
         expect(subject).to receive(:ensure_connected)
 
-        subject.find_pool(poolname,connection)
+        subject.find_pool(poolname,connection,datacenter_name)
       end
 
       it 'should return nil if the pool is not found' do
         pending('https://github.com/puppetlabs/vmpooler/issues/209')
-        expect(subject.find_pool(missing_poolname,connection)).to be_nil
+        expect(subject.find_pool(missing_poolname,connection,datacenter_name)).to be_nil
+      end
+    end
+
+    context 'with multiple datacenters' do
+      let(:poolpath) { 'pool' }
+      let(:connection_options) {{
+        :serviceContent => {
+          :datacenters => [
+            { :name => 'AnotherDC',
+              :hostfolder_tree => {
+                'folder1' => nil,
+                'folder2' => nil,
+              },
+            },
+            { :name => datacenter_name,
+              :hostfolder_tree => {
+                'folder3' => nil,
+                'pool' => {:object_type => 'resource_pool'},
+                'folder4' => nil,
+              },
+            }
+          ]
+        }
+      }}
+
+      it 'should return the pool when found' do
+        result = subject.find_pool(poolpath,connection, datacenter_name)
+
+        expect(result).to_not be_nil
+        expect(result.name).to eq('pool')
+        expect(result.is_a?(RbVmomi::VIM::ResourcePool)).to be true
       end
     end
 
@@ -2398,10 +2665,18 @@ EOT
     },
     ].each do |testcase|
       context testcase[:context] do
-        let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ :hostfolder_tree => testcase[:hostfolder_tree]}) }
+        let(:connection_options) {{
+          :serviceContent => {
+            :datacenters => [
+              { :name => datacenter_name,
+                :hostfolder_tree => testcase[:hostfolder_tree],
+              }
+            ]
+          }
+        }}
 
         it 'should return the pool when found' do
-          result = subject.find_pool(testcase[:poolpath],connection)
+          result = subject.find_pool(testcase[:poolpath],connection,datacenter_name)
 
           expect(result).to_not be_nil
           expect(result.name).to eq(testcase[:poolname])
@@ -2410,7 +2685,7 @@ EOT
 
         it 'should return nil if the poolname is not found' do
           pending('https://github.com/puppetlabs/vmpooler/issues/209')
-          expect(subject.find_pool(missing_poolname,connection)).to be_nil
+          expect(subject.find_pool(missing_poolname,connection,datacenter_name)).to be_nil
         end
       end
     end
@@ -2455,18 +2730,26 @@ EOT
     },
     ].each do |testcase|
       context testcase[:context] do
-        let(:datacenter_object) { mock_RbVmomi_VIM_Datacenter({ :hostfolder_tree => testcase[:hostfolder_tree]}) }
+        let(:connection_options) {{
+          :serviceContent => {
+            :datacenters => [
+              { :name => datacenter_name,
+                :hostfolder_tree => testcase[:hostfolder_tree],
+              }
+            ]
+          }
+        }}
 
         it 'should ensure the connection' do
           pending('https://github.com/puppetlabs/vmpooler/issues/210')
           expect(subject).to receive(:ensure_connected)
 
-          subject.find_pool(testcase[:poolpath])
+          subject.find_pool(testcase[:poolpath],connection,datacenter_name)
         end
 
         it 'should return the pool when found' do
           pending('https://github.com/puppetlabs/vmpooler/issues/210')
-          result = subject.find_pool(testcase[:poolpath])
+          result = subject.find_pool(testcase[:poolpath],connection,datacenter_name)
 
           expect(result).to_not be_nil
           expect(result.name).to eq(testcase[:poolname])
@@ -2688,7 +2971,7 @@ EOT
     let(:connection_options) {{
       :serviceContent => {
         :datacenters => [
-          { :name => 'MockDC', :datastores => [datastorename] }
+          { :name => datacenter_name, :datastores => [datastorename] }
         ]
       }
     }}
@@ -2719,11 +3002,11 @@ EOT
       } }
 
       it 'should return empty array if no VMDKs match the VM name' do
-        expect(subject.find_vmdks('missing_vm_name',datastorename,connection)).to eq([])
+        expect(subject.find_vmdks('missing_vm_name',datastorename,connection,datacenter_name)).to eq([])
       end
 
       it 'should return matching VMDKs for the VM' do
-        result = subject.find_vmdks(vmname,datastorename,connection)
+        result = subject.find_vmdks(vmname,datastorename,connection,datacenter_name)
         expect(result).to_not be_nil
         expect(result.count).to eq(2)
         # The keys for each VMDK should be less that 100 as per the mocks
diff --git a/vmpooler.yaml.example b/vmpooler.yaml.example
index 6b0d4c6..e6686e1 100644
--- a/vmpooler.yaml.example
+++ b/vmpooler.yaml.example
@@ -6,6 +6,27 @@
 # The currently supported backing services are:
 # - vsphere
 # - dummy
+#
+#   - provider_class
+#     For multiple providers, specify one of the supported backing services (vsphere or dummy)
+#     (optional: will default to it's parent :key: name eg. 'vsphere')
+#
+# If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
+# backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
+# the provider to use.
+#
+# Multiple providers example:
+
+  :vsphere-pdx:
+    server: 'vsphere.pdx.company.com'
+    username: 'vmpooler-pdx'
+    password: 'swimsw1msw!m'
+    provider_class: 'vsphere'
+  :vsphere-bfs:
+    server: 'vsphere.bfs.company.com'
+    username: 'vmpooler-bfs'
+    password: 'swimsw1msw!m'
+    provider_class: 'vsphere'
 
 # :vsphere:
 #
@@ -33,9 +54,9 @@
 #     Whether to ignore any HTTPS negotiation errors (e.g. untrusted self-signed certificates)
 #     (optional: default true)
 #
-#   - provider_class
-#     For multiple providers, specify one of the supported backing services (vsphere or dummy)
-#     (optional: will default to it's parent :key: name eg. 'vsphere')
+#   - datacenter
+#     The datacenter within vCenter to manage VMs.  This can be overridden in the pool configuration
+#     (optional: default is the first datacenter in vSphere)
 #
 # Example:
 
@@ -44,23 +65,6 @@
     username: 'vmpooler'
     password: 'swimsw1msw!m'
 
-# If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
-# backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
-# the provider to use.
-#
-# Multiple providers example:
-
-  :vsphere-pdx:
-    server: 'vsphere.pdx.company.com'
-    username: 'vmpooler-pdx'
-    password: 'swimsw1msw!m'
-    provider_class: 'vsphere'
-  :vsphere-bfs:
-    server: 'vsphere.bfs.company.com'
-    username: 'vmpooler-bfs'
-    password: 'swimsw1msw!m'
-    provider_class: 'vsphere'
-
 # :dummy:
 #
 # The dummy backing service is a simple text file service that can be used
@@ -465,6 +469,10 @@
 #       The vSphere 'datastore' destination for spawned clones.
 #       (required)
 #
+#     - datacenter
+#       The datacenter within vCenter to manage VMs.
+#       (optional: default is the first datacenter in vSphere)
+#
 # Example:
 
 :pools: