(POOLER-66) Purge vms and folders no longer configured

This commit enables optional purging for vms and folders when they are
not configured in the vmpooler provided configuration. Base folders are
determined from folders specified in the pool configuration. Then,
anything not configured in that folder for that provider and is not a
whitelisted folder title will be destroyed. Without this change vmpooler
will leave unconfigured vms and folders behind and any vms will be left
running forever without manual intervention. Additionally, any
associated redis data will never be expired.

lib/vmpooler/pool_manager.rb

@@ -287,6 +287,54 @@ module Vmpooler
       end
     end
 
+    def purge_unused_vms_and_folders
+      global_purge = $config[:config]['purge_unconfigured_folders']
+      providers = $config[:providers].keys
+      providers.each do |provider|
+        provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
+        provider_purge = global_purge if provider_purge.nil?
+        if provider_purge
+          Thread.new do
+            begin
+              purge_vms_and_folders($providers[provider.to_s])
+            rescue => err
+              $logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}")
+            end
+          end
+        end
+      end
+      return
+    end
+
+    # Return a list of pool folders
+    def pool_folders(provider)
+      provider_name = provider.name
+      folders = {}
+      $config[:pools].each do |pool|
+        next unless pool['provider'] == provider_name
+        folder_parts = pool['folder'].split('/')
+        datacenter = provider.get_target_datacenter_from_config(pool['name'])
+        folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
+      end
+      folders
+    end
+
+    def get_base_folders(folders)
+      base = []
+      folders.each do |key, value|
+        base << value
+      end
+      base.uniq
+    end
+
+    def purge_vms_and_folders(provider)
+      provider_name = provider.name
+      configured_folders = pool_folders(provider)
+      base_folders = get_base_folders(configured_folders)
+      whitelist = $config[:providers][provider_name.to_sym]['folder_whitelist']
+      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+    end
+
     def create_vm_disk(pool_name, vm, disk_size, provider)
       Thread.new do
         begin
@@ -961,6 +1009,8 @@ module Vmpooler
         end
       end
 
+      purge_unused_vms_and_folders
+
       loop_count = 1
       loop do
         if !$threads['disk_manager']

lib/vmpooler/providers/base.rb

@@ -225,6 +225,18 @@ module Vmpooler
         def create_template_delta_disks(pool)
           raise("#{self.class.name} does not implement create_template_delta_disks")
         end
+
+        # inputs
+        #   [String] provider_name : Name of the provider
+        # returns
+        #   Hash of folders
+        def get_target_datacenter_from_config(provider_name)
+          raise("#{self.class.name} does not implement get_target_datacenter_from_config")
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        end
       end
     end
   end

lib/vmpooler/providers/vsphere.rb

@@ -42,6 +42,109 @@ module Vmpooler
           'vsphere'
         end
 
+        def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+          if whitelist
+            return true if whitelist.include?(folder_title)
+          end
+          return false unless configured_folders.keys.include?(folder_title)
+          return false unless configured_folders[folder_title] == base_folder
+          return true
+        end
+
+        def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
+          try = 0 if try.nil?
+          max_tries = 3
+          $redis.srem("vmpooler__completed__#{pool}", vm_name)
+          $redis.hdel("vmpooler__active__#{pool}", vm_name)
+          $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
+
+          # Auto-expire metadata key
+          $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
+
+          start = Time.now
+
+          if vm_object.is_a? RbVmomi::VIM::Folder
+            logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
+            raise('Expected VM, but received a folder object')
+          end
+          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
+          vm_object.Destroy_Task.wait_for_completion
+
+          finish = format('%.2f', Time.now - start)
+          logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
+          metrics.timing("destroy.#{pool}", finish)
+        rescue RuntimeError
+          raise
+        rescue => err
+          try += 1
+          logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}")
+          try >= max_tries ? raise : retry
+        end
+
+        def destroy_folder_and_children(folder_object)
+          vms = {}
+          data_ttl = $config[:redis]['data_ttl'].to_i
+          folder_name = folder_object.name
+          unless folder_object.childEntity.count == 0
+            folder_object.childEntity.each do |vm|
+              vms[vm.name] = vm
+            end
+
+            vms.each do |vm_name, vm_object|
+              destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
+            end
+          end
+          destroy_folder(folder_object)
+        end
+
+        def destroy_folder(folder_object)
+          try = 0 if try.nil?
+          max_tries = 3
+          logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
+          folder_object.Destroy_Task.wait_for_completion
+        rescue
+          try += 1
+          try >= max_tries ? raise : retry
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+
+            base_folders.each do |base_folder|
+              folder_children = get_folder_children(base_folder, connection)
+              unless folder_children.empty?
+                folder_children.each do |folder_hash|
+                  folder_hash.each do |folder_title, folder_object|
+                    unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+                      destroy_folder_and_children(folder_object)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def get_folder_children(folder_name, connection)
+          folders = []
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => folder_name
+          }
+          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+
+          return folders if folder_object.nil?
+
+          folder_object.childEntity.each do |folder|
+            next unless folder.is_a? RbVmomi::VIM::Folder
+            folders << { folder.name => folder }
+          end
+
+          folders
+        end
+
         def vms_in_pool(pool_name)
           vms = []
           @connection_pool.with_metrics do |pool_object|

spec/unit/pool_manager_spec.rb

@@ -802,6 +802,140 @@ EOT
     end
   end
 
+  describe '#purge_unused_vms_and_folders' do
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock: {}
+:pools:
+  - name: '#{pool}'
+    size: 1
+EOT
+      )
+    }
+
+    it 'should return when purging is not enabled' do
+      expect(subject.purge_unused_vms_and_folders).to be_nil
+    end
+
+    context 'with purging enabled globally' do
+      before(:each) do
+        config[:config]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for each provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+
+      it 'should log when purging fails' do
+        expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
+        expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+
+    context 'with purging enabled on the provider' do
+      before(:each) do
+        config[:providers][:mock]['purge_unconfigured_folders'] = true
+        expect(Thread).to receive(:new).and_yield
+      end
+
+      it 'should run a purge for the provider' do
+        expect(subject).to receive(:purge_vms_and_folders)
+
+        subject.purge_unused_vms_and_folders
+      end
+    end
+  end
+
+  describe '#pool_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:datacenter) { 'dc1' }
+    let(:provider_name) { 'mock_provider' }
+    let(:expected_response) {
+      {
+        folder_name => "#{datacenter}/vm/#{folder_base}"
+      }
+    }
+    let(:config) { YAML.load(<<-EOT
+---
+:providers:
+  :mock:
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+  - name: '#{pool}2'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}2'
+EOT
+      )
+    }
+
+    it 'should return a list of pool folders' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
+
+      expect(subject.pool_folders(provider)).to eq(expected_response)
+    end
+
+    it 'should raise an error when the provider fails to get the datacenter' do
+      expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
+
+      expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
+  describe '#purge_vms_and_folders' do
+    let(:folder_name) { 'myinstance' }
+    let(:folder_base) { 'vmpooler' }
+    let(:datacenter) { 'dc1' }
+    let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
+    let(:configured_folders) { { folder_name => full_folder_path } }
+    let(:base_folders) { [ full_folder_path ] }
+    let(:folder) { [folder_base,folder_name].join('/') }
+    let(:provider_name) { 'mock_provider' }
+    let(:whitelist) { nil }
+    let(:config) { YAML.load(<<-EOT
+---
+:config: {}
+:providers:
+  :mock_provider: {}
+:pools:
+  - name: '#{pool}'
+    folder: '#{folder}'
+    size: 1
+    datacenter: '#{datacenter}'
+    provider: '#{provider_name}'
+EOT
+      )
+    }
+
+    it 'should run purge_unconfigured_folders' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
+
+      subject.purge_vms_and_folders(provider)
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:pool_folders).and_return(configured_folders)
+      expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
+
+      expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
+    end
+  end
+
   describe '#create_vm_disk' do
     let(:provider) { double('provider') }
     let(:disk_size) { 15 }
@@ -2080,6 +2214,8 @@ EOT
         let(:config) {
         YAML.load(<<-EOT
 ---
+:providers:
+  :vsphere: {}
 :pools:
   - name: #{pool}
   - name: 'dummy'

spec/unit/providers/vsphere_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'mock_redis'
 
 RSpec::Matchers.define :relocation_spec_with_host do |value|
   match { |actual| actual[:spec].host == value }
@@ -87,6 +88,289 @@ EOT
     end
   end
 
+  describe '#folder_configured?' do
+    let(:folder_title) { 'folder1' }
+    let(:other_folder) { 'folder2' }
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:configured_folders) { { folder_title => base_folder } }
+    let(:whitelist) { nil }
+    it 'should return true when configured_folders includes the folder_title' do
+      expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true
+    end
+
+    it 'should return false when title is not in configured_folders' do
+      expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false
+    end
+
+    context 'with another base folder' do
+      let(:base_folder) { 'dc2/vm/base' }
+      let(:configured_folders) { { folder_title => 'dc1/vm/base' } }
+      it 'should return false' do
+        expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false
+      end
+    end
+
+    context 'with a whitelist set' do
+      let(:whitelist) { [ other_folder ] }
+      it 'should return true' do
+        expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true
+      end
+    end
+
+    context 'with string whitelist value' do
+      let(:whitelist) { 'whitelist' }
+      it 'should raise an error' do
+        expect(whitelist).to receive(:include?).and_raise('mockerror')
+
+        expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror')
+      end
+    end
+  end
+
+  describe '#destroy_vm_and_log' do
+    let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
+        :name => vmname,
+        :powerstate => 'poweredOn',
+      })
+    }
+    let(:pool) { 'pool1' }
+    let(:power_off_task) { mock_RbVmomi_VIM_Task() }
+    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
+    let(:data_ttl) { 1 }
+    let(:redis) { MockRedis.new }
+    let(:finish) { '0.00' }
+    let(:now) { Time.now }
+
+    context 'when destroying a vm' do
+      before(:each) do
+        allow(power_off_task).to receive(:wait_for_completion)
+        allow(destroy_task).to receive(:wait_for_completion)
+        allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
+        allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+        $redis = redis
+      end
+
+      it 'should remove redis data and expire the vm key' do
+        allow(Time).to receive(:now).and_return(now)
+        expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname)
+        expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname)
+        expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now)
+        expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should log a message that the vm is destroyed' do
+        expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds")
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should record metrics' do
+        expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+
+      it 'should power off and destroy the vm' do
+        allow(destroy_task).to receive(:wait_for_completion)
+        expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
+        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+    end
+
+    context 'with a powered off vm' do
+      before(:each) do
+        vm_object.runtime.powerState = 'poweredOff'
+      end
+
+      it 'should destroy the vm without attempting to power it off' do
+        allow(destroy_task).to receive(:wait_for_completion)
+        expect(vm_object).to_not receive(:PowerOffVM_Task)
+        expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+        subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
+      end
+    end
+
+    context 'with a folder object' do
+      let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) }
+
+      it 'should log that a folder object was received' do
+        expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying")
+
+        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
+      end
+
+      it 'should raise an error' do
+        expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
+      end
+    end
+
+    context 'with an error that is not a RuntimeError' do
+      it 'should retry three times' do
+        expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times
+
+        expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/)
+      end
+    end
+  end
+
+  describe '#destroy_folder_and_children' do
+    let(:data_ttl) { 1 }
+    let(:config) {
+      {
+        redis: {
+          'data_ttl' => data_ttl
+        }
+      }
+    }
+    let(:foldername) { 'pool1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+
+    before(:each) do
+      $config = config
+    end
+
+    context 'with an empty folder' do
+      it 'should destroy the folder' do
+        expect(subject).to_not receive(:destroy_vm_and_log)
+        expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil)
+
+        subject.destroy_folder_and_children(folder_object)
+      end
+    end
+
+    context 'with a folder containing vms' do
+      let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
+      before(:each) do
+        folder_object.childEntity << vm_object
+      end
+
+      it 'should destroy the vms' do
+        allow(subject).to receive(:destroy_vm_and_log).and_return(nil)
+        allow(subject).to receive(:destroy_folder).and_return(nil)
+        expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl)
+
+        subject.destroy_folder_and_children(folder_object)
+      end
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:destroy_folder).and_throw('mockerror')
+
+      expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#destroy_folder' do
+    let(:foldername) { 'pool1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+    let(:destroy_task) { mock_RbVmomi_VIM_Task() }
+
+    before(:each) do
+      allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
+      allow(destroy_task).to receive(:wait_for_completion)
+    end
+
+    it 'should destroy the folder' do
+      expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
+
+      subject.destroy_folder(folder_object)
+    end
+
+    it 'should log that the folder is being destroyed' do
+      expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder")
+
+      subject.destroy_folder(folder_object)
+    end
+
+    it 'should retry three times when failing' do
+      expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times
+
+      expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#purge_unconfigured_folders' do
+    let(:folder_title) { 'folder1' }
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
+    let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) }
+    let(:whitelist) { nil }
+    let(:base_folders) { [ base_folder ] }
+    let(:configured_folders) { { folder_title => base_folder } }
+    let(:folder_children) { [ folder_title => child_folder ] }
+    let(:empty_list) { [] }
+
+    before(:each) do
+      allow(subject).to receive(:connect_to_vsphere).and_return(connection)
+    end
+
+    context 'with an empty folder' do
+      it 'should not attempt to destroy any folders' do
+        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list)
+        expect(subject).to_not receive(:destroy_folder_and_children)
+
+        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      end
+    end
+
+    it 'should retrieve the folder children' do
+      expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
+      allow(subject).to receive(:folder_configured?).and_return(true)
+
+      subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+    end
+
+    context 'with a folder that is not configured' do
+      before(:each) do
+        expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
+        allow(subject).to receive(:folder_configured?).and_return(false)
+      end
+
+      it 'should destroy the folder and children' do
+        expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil)
+
+        subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+      end
+    end
+
+    it 'should raise any errors' do
+      expect(subject).to receive(:get_folder_children).and_throw('mockerror')
+
+      expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/)
+    end
+  end
+
+  describe '#get_folder_children' do
+    let(:base_folder) { 'dc1/vm/base' }
+    let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
+    let(:foldername) { 'folder1' }
+    let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
+    let(:folder_return) { [ { foldername => folder_object } ] }
+
+    before(:each) do
+      base_folder_object.childEntity << folder_object
+    end
+
+    it 'should return an array of configured folder hashes' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object)
+
+      result = subject.get_folder_children(foldername, connection)
+
+      expect(result).to eq(folder_return)
+    end
+
+    it 'should raise any errors' do
+      expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror')
+
+      expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/)
+    end
+  end
+
   describe '#vms_in_pool' do
     let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
     let(:pool_config) { config[:pools][0] }