Merge pull request #274 from mattkirby/cleanup_unused_folders_and_vms

(POOLER-66) Purge vms and folders no longer configured

spec/unit/pool_manager_spec.rb

@@ -762,6 +762,140 @@ EOT
     end
   end
 
+  describe '#purge_unused_vms_and_folders' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock: {}
+:pools:
+  - name: '#{pool}'
+    size: 1
+EOT
+      )
+    }
+
+    it 'should return when purging is not enabled' do
+      expect(subject.purge_unused_vms_and_folders).to be_nil
+    end
+
+    context 'with purging enabled globally' do
+      before(:each) do
+        config[:config]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for each provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+
+      it 'should log when purging fails' do
+        expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
+        expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+
+    context 'with purging enabled on the provider' do
+      before(:each) do
+        config[:providers][:mock]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for the provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+  end
+
+  describe '#pool_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:datacenter) { 'dc1' }
+    let(:provider_name) { 'mock_provider' }
+    let(:expected_response) {
+      {
+        folder_name => "#{datacenter}/vm/#{folder_base}"
+      }
+    }
+    let(:config) { YAML.load(<<-EOT
+---
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+  - name: '#{pool}2'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}2'
+EOT
+      )
+    }
+
+    it 'should return a list of pool folders' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
+
+      expect(subject.pool_folders(provider)).to eq(expected_response)
+    end
+
+    it 'should raise an error when the provider fails to get the datacenter' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
+
+      expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
+  describe '#purge_vms_and_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:datacenter) { 'dc1' }
+    let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
+    let(:configured_folders) { { folder_name => full_folder_path } }
+    let(:base_folders) { [ full_folder_path ] }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:provider_name) { 'mock_provider' }
+    let(:whitelist) { nil }
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock_provider: {}
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+EOT
+      )
+    }
+
+    it 'should run purge_unconfigured_folders' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
+
+      subject.purge_vms_and_folders(provider)
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
+
+      expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
   describe '#create_vm_disk' do
     let(:provider) { double('provider') }
     let(:disk_size) { 15 }
@@ -2040,6 +2174,8 @@ EOT
         let(:config) {
         YAML.load(<<-EOT
 ---
+:providers:
+  :vsphere: {}
 :pools:
   - name: #{pool}
   - name: 'dummy'

spec/unit/providers/vsphere_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'mock_redis'
 
 RSpec::Matchers.define :relocation_spec_with_host do |value|
   match { |actual| actual[:spec].host == value }
@@ -87,6 +88,289 @@ EOT
     end
   end
 
+  describe '#folder_configured?' do
+    let(:folder_title) { 'folder1' }
+    let(:other_folder) { 'folder2' }
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:configured_folders) { { folder_title => base_folder } }
+    let(:whitelist) { nil }
+    it 'should return true when configured_folders includes the folder_title' do
+      expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true
+    end
+
+    it 'should return false when title is not in configured_folders' do
+      expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false
+    end
+
+    context 'with another base folder' do
+      let(:base_folder) { 'dc2/vm/base' }
+      let(:configured_folders) { { folder_title => 'dc1/vm/base' } }
+      it 'should return false' do
+        expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false
+      end
+    end
+
+    context 'with a whitelist set' do
+      let(:whitelist) { [ other_folder ] }
+      it 'should return true' do
+        expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true
+      end
+    end
+
+    context 'with string whitelist value' do
+      let(:whitelist) { 'whitelist' }
+      it 'should raise an error' do
+        expect(whitelist).to receive(:include?).and_raise('mockerror')
+
+        expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror')
+      end
+    end
+  end
+
+  describe '#destroy_vm_and_log' do
+    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
+        :name => vmname,
+        :powerstate => 'poweredOn',
+      })
+    }
+    let(:pool) { 'pool1' }
+    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
+    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
+    let(:data_ttl) { 1 }
+    let(:redis) { MockRedis.new }
+    let(:finish) { '0.00' }
+    let(:now) { Time.now }
+
+    context 'when destroying a vm' do
+      before(:each) do
+        allow(power_off_task).to receive(:wait_for_completion)
+        allow(destroy_task).to receive(:wait_for_completion)
+        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
+        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+        $redis = redis
+      end
+
+      it 'should remove redis data and expire the vm key' do
+        allow(Time).to receive(:now).and_return(now)
+        expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname)
+        expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname)
+        expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now)
+        expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should log a message that the vm is destroyed' do
+        expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds")
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should record metrics' do
+        expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should power off and destroy the vm' do
+        allow(destroy_task).to receive(:wait_for_completion)
+        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
+        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+    end
+
+    context 'with a powered off vm' do
+      before(:each) do
+        vm_object.runtime.powerState = 'poweredOff'
+      end
+
+      it 'should destroy the vm without attempting to power it off' do
+        allow(destroy_task).to receive(:wait_for_completion)
+        expect(vm_object).to_not receive(:PowerOffVM_Task)
+        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+    end
+
+    context 'with a folder object' do
+      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) }
+
+      it 'should log that a folder object was received' do
+        expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying")
+
+        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
+      end
+
+      it 'should raise an error' do
+        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
+      end
+    end
+
+    context 'with an error that is not a RuntimeError' do
+      it 'should retry three times' do
+        expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times
+
+        expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/)
+      end
+    end
+  end
+
+  describe '#destroy_folder_and_children' do
+    let(:data_ttl) { 1 }
+    let(:config) {
+      {
+        redis: {
+          'data_ttl' => data_ttl
+        }
+      }
+    }
+    let(:foldername) { 'pool1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+
+    before(:each) do
+      $config = config
+    end
+
+    context 'with an empty folder' do
+      it 'should destroy the folder' do
+        expect(subject).to_not receive(:destroy_vm_and_log)
+        expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil)
+
+        subject.destroy_folder_and_children(folder_object)
+      end
+    end
+
+    context 'with a folder containing vms' do
+      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
+      before(:each) do
+        folder_object.childEntity << vm_object
+      end
+
+      it 'should destroy the vms' do
+        allow(subject).to receive(:destroy_vm_and_log).and_return(nil)
+        allow(subject).to receive(:destroy_folder).and_return(nil)
+        expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl)
+
+        subject.destroy_folder_and_children(folder_object)
+      end
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:destroy_folder).and_throw('mockerror')
+
+      expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#destroy_folder' do
+    let(:foldername) { 'pool1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
+
+    before(:each) do
+      allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
+      allow(destroy_task).to receive(:wait_for_completion)
+    end
+
+    it 'should destroy the folder' do
+      expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+      subject.destroy_folder(folder_object)
+    end
+
+    it 'should log that the folder is being destroyed' do
+      expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder")
+
+      subject.destroy_folder(folder_object)
+    end
+
+    it 'should retry three times when failing' do
+      expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times
+
+      expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#purge_unconfigured_folders' do
+    let(:folder_title) { 'folder1' }
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
+    let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) }
+    let(:whitelist) { nil }
+    let(:base_folders) { [ base_folder ] }
+    let(:configured_folders) { { folder_title => base_folder } }
+    let(:folder_children) { [ folder_title => child_folder ] }
+    let(:empty_list) { [] }
+
+    before(:each) do
+      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
+    end
+
+    context 'with an empty folder' do
+      it 'should not attempt to destroy any folders' do
+        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list)
+        expect(subject).to_not receive(:destroy_folder_and_children)
+
+        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      end
+    end
+
+    it 'should retrieve the folder children' do
+      expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
+      allow(subject).to receive(:folder_configured?).and_return(true)
+
+      subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+    end
+
+    context 'with a folder that is not configured' do
+      before(:each) do
+        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
+        allow(subject).to receive(:folder_configured?).and_return(false)
+      end
+
+      it 'should destroy the folder and children' do
+        expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil)
+
+        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      end
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:get_folder_children).and_throw('mockerror')
+
+      expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#get_folder_children' do
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
+    let(:foldername) { 'folder1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+    let(:folder_return) { [ { foldername => folder_object } ] }
+
+    before(:each) do
+      base_folder_object.childEntity << folder_object
+    end
+
+    it 'should return an array of configured folder hashes' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object)
+
+      result = subject.get_folder_children(foldername, connection)
+
+      expect(result).to eq(folder_return)
+    end
+
+    it 'should raise any errors' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror')
+
+      expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/)
+    end
+  end
+
   describe '#vms_in_pool' do
     let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
     let(:pool_config) { config[:pools][0] }