Merge pull request #274 from mattkirby/cleanup_unused_folders_and_vms

(POOLER-66) Purge vms and folders no longer configured
This commit is contained in:
Kevin Imber 2018-07-19 13:19:13 -07:00 committed by GitHub
commit 099b53f348
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 609 additions and 0 deletions

View file

@ -315,6 +315,53 @@ module Vmpooler
end end
end end
def purge_unused_vms_and_folders
global_purge = $config[:config]['purge_unconfigured_folders']
providers = $config[:providers].keys
providers.each do |provider|
provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
provider_purge = global_purge if provider_purge.nil?
if provider_purge
Thread.new do
begin
purge_vms_and_folders($providers[provider.to_s])
rescue => err
$logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}")
end
end
end
end
return
end
# Return a list of pool folders
def pool_folders(provider)
provider_name = provider.name
folders = {}
$config[:pools].each do |pool|
next unless pool['provider'] == provider_name
folder_parts = pool['folder'].split('/')
datacenter = provider.get_target_datacenter_from_config(pool['name'])
folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
end
folders
end
def get_base_folders(folders)
base = []
folders.each do |key, value|
base << value
end
base.uniq
end
def purge_vms_and_folders(provider)
configured_folders = pool_folders(provider)
base_folders = get_base_folders(configured_folders)
whitelist = provider.provider_config['folder_whitelist']
provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
end
def create_vm_disk(pool_name, vm, disk_size, provider) def create_vm_disk(pool_name, vm, disk_size, provider)
Thread.new do Thread.new do
begin begin
@ -992,6 +1039,8 @@ module Vmpooler
end end
end end
purge_unused_vms_and_folders
loop_count = 1 loop_count = 1
loop do loop do
if !$threads['disk_manager'] if !$threads['disk_manager']

View file

@ -225,6 +225,18 @@ module Vmpooler
def create_template_delta_disks(pool) def create_template_delta_disks(pool)
raise("#{self.class.name} does not implement create_template_delta_disks") raise("#{self.class.name} does not implement create_template_delta_disks")
end end
# inputs
# [String] provider_name : Name of the provider
# returns
# Hash of folders
def get_target_datacenter_from_config(provider_name)
raise("#{self.class.name} does not implement get_target_datacenter_from_config")
end
def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
raise("#{self.class.name} does not implement purge_unconfigured_folders")
end
end end
end end
end end

View file

@ -42,6 +42,109 @@ module Vmpooler
'vsphere' 'vsphere'
end end
def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
if whitelist
return true if whitelist.include?(folder_title)
end
return false unless configured_folders.keys.include?(folder_title)
return false unless configured_folders[folder_title] == base_folder
return true
end
def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
try = 0 if try.nil?
max_tries = 3
$redis.srem("vmpooler__completed__#{pool}", vm_name)
$redis.hdel("vmpooler__active__#{pool}", vm_name)
$redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
# Auto-expire metadata key
$redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
start = Time.now
if vm_object.is_a? RbVmomi::VIM::Folder
logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
raise('Expected VM, but received a folder object')
end
vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
vm_object.Destroy_Task.wait_for_completion
finish = format('%.2f', Time.now - start)
logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
metrics.timing("destroy.#{pool}", finish)
rescue RuntimeError
raise
rescue => err
try += 1
logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}")
try >= max_tries ? raise : retry
end
def destroy_folder_and_children(folder_object)
vms = {}
data_ttl = $config[:redis]['data_ttl'].to_i
folder_name = folder_object.name
unless folder_object.childEntity.count == 0
folder_object.childEntity.each do |vm|
vms[vm.name] = vm
end
vms.each do |vm_name, vm_object|
destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
end
end
destroy_folder(folder_object)
end
def destroy_folder(folder_object)
try = 0 if try.nil?
max_tries = 3
logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
folder_object.Destroy_Task.wait_for_completion
rescue
try += 1
try >= max_tries ? raise : retry
end
def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
@connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object)
base_folders.each do |base_folder|
folder_children = get_folder_children(base_folder, connection)
unless folder_children.empty?
folder_children.each do |folder_hash|
folder_hash.each do |folder_title, folder_object|
unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
destroy_folder_and_children(folder_object)
end
end
end
end
end
end
end
def get_folder_children(folder_name, connection)
folders = []
propSpecs = {
:entity => self,
:inventoryPath => folder_name
}
folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
return folders if folder_object.nil?
folder_object.childEntity.each do |folder|
next unless folder.is_a? RbVmomi::VIM::Folder
folders << { folder.name => folder }
end
folders
end
def vms_in_pool(pool_name) def vms_in_pool(pool_name)
vms = [] vms = []
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|

View file

@ -762,6 +762,140 @@ EOT
end end
end end
describe '#purge_unused_vms_and_folders' do
let(:config) { YAML.load(<<-EOT
---
:config: {}
:providers:
:mock: {}
:pools:
- name: '#{pool}'
size: 1
EOT
)
}
it 'should return when purging is not enabled' do
expect(subject.purge_unused_vms_and_folders).to be_nil
end
context 'with purging enabled globally' do
before(:each) do
config[:config]['purge_unconfigured_folders'] = true
expect(Thread).to receive(:new).and_yield
end
it 'should run a purge for each provider' do
expect(subject).to receive(:purge_vms_and_folders)
subject.purge_unused_vms_and_folders
end
it 'should log when purging fails' do
expect(subject).to receive(:purge_vms_and_folders).and_raise(RuntimeError,'MockError')
expect(logger).to receive(:log).with('s', '[!] failed while purging provider mock VMs and folders with an error: MockError')
subject.purge_unused_vms_and_folders
end
end
context 'with purging enabled on the provider' do
before(:each) do
config[:providers][:mock]['purge_unconfigured_folders'] = true
expect(Thread).to receive(:new).and_yield
end
it 'should run a purge for the provider' do
expect(subject).to receive(:purge_vms_and_folders)
subject.purge_unused_vms_and_folders
end
end
end
describe '#pool_folders' do
let(:folder_name) { 'myinstance' }
let(:folder_base) { 'vmpooler' }
let(:folder) { [folder_base,folder_name].join('/') }
let(:datacenter) { 'dc1' }
let(:provider_name) { 'mock_provider' }
let(:expected_response) {
{
folder_name => "#{datacenter}/vm/#{folder_base}"
}
}
let(:config) { YAML.load(<<-EOT
---
:providers:
:mock:
:pools:
- name: '#{pool}'
folder: '#{folder}'
size: 1
datacenter: '#{datacenter}'
provider: '#{provider_name}'
- name: '#{pool}2'
folder: '#{folder}'
size: 1
datacenter: '#{datacenter}'
provider: '#{provider_name}2'
EOT
)
}
it 'should return a list of pool folders' do
expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_return(datacenter)
expect(subject.pool_folders(provider)).to eq(expected_response)
end
it 'should raise an error when the provider fails to get the datacenter' do
expect(provider).to receive(:get_target_datacenter_from_config).with(pool).and_raise('mockerror')
expect{ subject.pool_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
end
end
describe '#purge_vms_and_folders' do
let(:folder_name) { 'myinstance' }
let(:folder_base) { 'vmpooler' }
let(:datacenter) { 'dc1' }
let(:full_folder_path) { "#{datacenter}/vm/folder_base" }
let(:configured_folders) { { folder_name => full_folder_path } }
let(:base_folders) { [ full_folder_path ] }
let(:folder) { [folder_base,folder_name].join('/') }
let(:provider_name) { 'mock_provider' }
let(:whitelist) { nil }
let(:config) { YAML.load(<<-EOT
---
:config: {}
:providers:
:mock_provider: {}
:pools:
- name: '#{pool}'
folder: '#{folder}'
size: 1
datacenter: '#{datacenter}'
provider: '#{provider_name}'
EOT
)
}
it 'should run purge_unconfigured_folders' do
expect(subject).to receive(:pool_folders).and_return(configured_folders)
expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist)
subject.purge_vms_and_folders(provider)
end
it 'should raise any errors' do
expect(subject).to receive(:pool_folders).and_return(configured_folders)
expect(provider).to receive(:purge_unconfigured_folders).with(base_folders, configured_folders, whitelist).and_raise('mockerror')
expect{ subject.purge_vms_and_folders(provider) }.to raise_error(RuntimeError, 'mockerror')
end
end
describe '#create_vm_disk' do describe '#create_vm_disk' do
let(:provider) { double('provider') } let(:provider) { double('provider') }
let(:disk_size) { 15 } let(:disk_size) { 15 }
@ -2040,6 +2174,8 @@ EOT
let(:config) { let(:config) {
YAML.load(<<-EOT YAML.load(<<-EOT
--- ---
:providers:
:vsphere: {}
:pools: :pools:
- name: #{pool} - name: #{pool}
- name: 'dummy' - name: 'dummy'

View file

@ -1,4 +1,5 @@
require 'spec_helper' require 'spec_helper'
require 'mock_redis'
RSpec::Matchers.define :relocation_spec_with_host do |value| RSpec::Matchers.define :relocation_spec_with_host do |value|
match { |actual| actual[:spec].host == value } match { |actual| actual[:spec].host == value }
@ -87,6 +88,289 @@ EOT
end end
end end
describe '#folder_configured?' do
let(:folder_title) { 'folder1' }
let(:other_folder) { 'folder2' }
let(:base_folder) { 'dc1/vm/base' }
let(:configured_folders) { { folder_title => base_folder } }
let(:whitelist) { nil }
it 'should return true when configured_folders includes the folder_title' do
expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be true
end
it 'should return false when title is not in configured_folders' do
expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be false
end
context 'with another base folder' do
let(:base_folder) { 'dc2/vm/base' }
let(:configured_folders) { { folder_title => 'dc1/vm/base' } }
it 'should return false' do
expect(subject.folder_configured?(folder_title, base_folder, configured_folders, whitelist)).to be false
end
end
context 'with a whitelist set' do
let(:whitelist) { [ other_folder ] }
it 'should return true' do
expect(subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist)).to be true
end
end
context 'with string whitelist value' do
let(:whitelist) { 'whitelist' }
it 'should raise an error' do
expect(whitelist).to receive(:include?).and_raise('mockerror')
expect{ subject.folder_configured?(other_folder, base_folder, configured_folders, whitelist) }.to raise_error(RuntimeError, 'mockerror')
end
end
end
describe '#destroy_vm_and_log' do
let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({
:name => vmname,
:powerstate => 'poweredOn',
})
}
let(:pool) { 'pool1' }
let(:power_off_task) { mock_RbVmomi_VIM_Task() }
let(:destroy_task) { mock_RbVmomi_VIM_Task() }
let(:data_ttl) { 1 }
let(:redis) { MockRedis.new }
let(:finish) { '0.00' }
let(:now) { Time.now }
context 'when destroying a vm' do
before(:each) do
allow(power_off_task).to receive(:wait_for_completion)
allow(destroy_task).to receive(:wait_for_completion)
allow(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
allow(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
$redis = redis
end
it 'should remove redis data and expire the vm key' do
allow(Time).to receive(:now).and_return(now)
expect(redis).to receive(:srem).with("vmpooler__completed__#{pool}", vmname)
expect(redis).to receive(:hdel).with("vmpooler__active__#{pool}", vmname)
expect(redis).to receive(:hset).with("vmpooler__vm__#{vmname}", 'destroy', now)
expect(redis).to receive(:expire).with("vmpooler__vm__#{vmname}", data_ttl * 60 * 60)
subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
end
it 'should log a message that the vm is destroyed' do
expect(logger).to receive(:log).with('s', "[-] [#{pool}] '#{vmname}' destroyed in #{finish} seconds")
subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
end
it 'should record metrics' do
expect(metrics).to receive(:timing).with("destroy.#{pool}", finish)
subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
end
it 'should power off and destroy the vm' do
allow(destroy_task).to receive(:wait_for_completion)
expect(vm_object).to receive(:PowerOffVM_Task).and_return(power_off_task)
expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
end
end
context 'with a powered off vm' do
before(:each) do
vm_object.runtime.powerState = 'poweredOff'
end
it 'should destroy the vm without attempting to power it off' do
allow(destroy_task).to receive(:wait_for_completion)
expect(vm_object).to_not receive(:PowerOffVM_Task)
expect(vm_object).to receive(:Destroy_Task).and_return(destroy_task)
subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl)
end
end
context 'with a folder object' do
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => vmname }) }
it 'should log that a folder object was received' do
expect(logger).to receive(:log).with('s', "[!] [#{pool}] '#{vmname}' is a folder, bailing on destroying")
expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
end
it 'should raise an error' do
expect{ subject.destroy_vm_and_log(vmname, folder_object, pool, data_ttl) }.to raise_error(RuntimeError, 'Expected VM, but received a folder object')
end
end
context 'with an error that is not a RuntimeError' do
it 'should retry three times' do
expect(vm_object).to receive(:PowerOffVM_Task).and_throw(:powerofffailed, 'failed').exactly(3).times
expect{ subject.destroy_vm_and_log(vmname, vm_object, pool, data_ttl) }.to raise_error(/failed/)
end
end
end
describe '#destroy_folder_and_children' do
let(:data_ttl) { 1 }
let(:config) {
{
redis: {
'data_ttl' => data_ttl
}
}
}
let(:foldername) { 'pool1' }
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
before(:each) do
$config = config
end
context 'with an empty folder' do
it 'should destroy the folder' do
expect(subject).to_not receive(:destroy_vm_and_log)
expect(subject).to receive(:destroy_folder).with(folder_object).and_return(nil)
subject.destroy_folder_and_children(folder_object)
end
end
context 'with a folder containing vms' do
let(:vm_object) { mock_RbVmomi_VIM_VirtualMachine({ :name => vmname }) }
before(:each) do
folder_object.childEntity << vm_object
end
it 'should destroy the vms' do
allow(subject).to receive(:destroy_vm_and_log).and_return(nil)
allow(subject).to receive(:destroy_folder).and_return(nil)
expect(subject).to receive(:destroy_vm_and_log).with(vmname, vm_object, foldername, data_ttl)
subject.destroy_folder_and_children(folder_object)
end
end
it 'should raise any errors' do
expect(subject).to receive(:destroy_folder).and_throw('mockerror')
expect{ subject.destroy_folder_and_children(folder_object) }.to raise_error(/mockerror/)
end
end
describe '#destroy_folder' do
let(:foldername) { 'pool1' }
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
let(:destroy_task) { mock_RbVmomi_VIM_Task() }
before(:each) do
allow(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
allow(destroy_task).to receive(:wait_for_completion)
end
it 'should destroy the folder' do
expect(folder_object).to receive(:Destroy_Task).and_return(destroy_task)
subject.destroy_folder(folder_object)
end
it 'should log that the folder is being destroyed' do
expect(logger).to receive(:log).with('s', "[-] [#{foldername}] removing unconfigured folder")
subject.destroy_folder(folder_object)
end
it 'should retry three times when failing' do
expect(folder_object).to receive(:Destroy_Task).and_throw('mockerror').exactly(3).times
expect{ subject.destroy_folder(folder_object) }.to raise_error(/mockerror/)
end
end
describe '#purge_unconfigured_folders' do
let(:folder_title) { 'folder1' }
let(:base_folder) { 'dc1/vm/base' }
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
let(:child_folder) { mock_RbVmomi_VIM_Folder({ :name => folder_title }) }
let(:whitelist) { nil }
let(:base_folders) { [ base_folder ] }
let(:configured_folders) { { folder_title => base_folder } }
let(:folder_children) { [ folder_title => child_folder ] }
let(:empty_list) { [] }
before(:each) do
allow(subject).to receive(:connect_to_vsphere).and_return(connection)
end
context 'with an empty folder' do
it 'should not attempt to destroy any folders' do
expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(empty_list)
expect(subject).to_not receive(:destroy_folder_and_children)
subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
end
end
it 'should retrieve the folder children' do
expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
allow(subject).to receive(:folder_configured?).and_return(true)
subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
end
context 'with a folder that is not configured' do
before(:each) do
expect(subject).to receive(:get_folder_children).with(base_folder, connection).and_return(folder_children)
allow(subject).to receive(:folder_configured?).and_return(false)
end
it 'should destroy the folder and children' do
expect(subject).to receive(:destroy_folder_and_children).with(child_folder).and_return(nil)
subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
end
end
it 'should raise any errors' do
expect(subject).to receive(:get_folder_children).and_throw('mockerror')
expect{ subject.purge_unconfigured_folders(base_folders, configured_folders, whitelist) }.to raise_error(/mockerror/)
end
end
describe '#get_folder_children' do
let(:base_folder) { 'dc1/vm/base' }
let(:base_folder_object) { mock_RbVmomi_VIM_Folder({ :name => base_folder }) }
let(:foldername) { 'folder1' }
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => foldername }) }
let(:folder_return) { [ { foldername => folder_object } ] }
before(:each) do
base_folder_object.childEntity << folder_object
end
it 'should return an array of configured folder hashes' do
expect(connection.searchIndex).to receive(:FindByInventoryPath).and_return(base_folder_object)
result = subject.get_folder_children(foldername, connection)
expect(result).to eq(folder_return)
end
it 'should raise any errors' do
expect(connection.searchIndex).to receive(:FindByInventoryPath).and_throw('mockerror')
expect{ subject.get_folder_children(foldername, connection) }.to raise_error(/mockerror/)
end
end
describe '#vms_in_pool' do describe '#vms_in_pool' do
let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) } let(:folder_object) { mock_RbVmomi_VIM_Folder({ :name => 'pool1'}) }
let(:pool_config) { config[:pools][0] } let(:pool_config) { config[:pools][0] }

View file

@ -11,6 +11,22 @@
# For multiple providers, specify one of the supported backing services (vsphere or dummy) # For multiple providers, specify one of the supported backing services (vsphere or dummy)
# (optional: will default to it's parent :key: name eg. 'vsphere') # (optional: will default to it's parent :key: name eg. 'vsphere')
# #
# - purge_unconfigured_folders
# Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
# Only a single layer of folders and their child VMs are evaluated from detected base folder paths
# Nested child folders will not be destroyed. An optional whitelist can be provided to exclude folders
# A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
# Setting this on the provider will enable folder purging for the provider
# Expects a boolean value
# (optional; default: false)
#
# - folder_whitelist
# Specify folders that are within the base folder path, not in the configuration, and should not be destroyed
# To exclude 'vmpooler/myfolder' from being destroyed when the base path is 'vmpooler' you would specify 'myfolder' in the whitelist
# This option is only evaluated when 'purge_unconfigured_folders' is enabled
# Expects an array of strings specifying the whitelisted folders by name
# (optional; default: nil)
#
# If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the # If you want to support more than one provider with different parameters (server, username or passwords) you have to specify the
# backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify # backing service in the provider_class configuration parameter for example 'vsphere' or 'dummy'. Each pool can specify
# the provider to use. # the provider to use.
@ -298,6 +314,7 @@
# #
# - base # - base
# The base DN used for LDAP searches. # The base DN used for LDAP searches.
# This can be a string providing a single DN, or an array of DNs to search.
# #
# - user_object # - user_object
# The LDAP object-type used to designate a user object. # The LDAP object-type used to designate a user object.
@ -448,6 +465,14 @@
# Expects a boolean value # Expects a boolean value
# (optional; default: false) # (optional; default: false)
# #
# - purge_unconfigured_folders (vSphere Provider only)
# Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
# Only a single layer of folders and their child VMs are evaluated from detected base folder paths
# A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
# When enabled in the global configuration then purging is enabled for all providers
# Expects a boolean value
# (optional; default: false)
#
# Example: # Example:
:config: :config: