puppetlabs/vmpooler
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler.git synced 2026-01-26 10:08:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Simplify vsphere connection handling in order to make it reasonable to test. Simplify migrate_vm method by breaking out some componenents. Improve error handling around migrate_vm.

Browse source 
Add helpers to support setting up redis for migration at checkout
testing.
This commit is contained in:
kirby@puppetlabs.com 2016-11-16 03:20:59 -08:00
parent 12c2c4a09b
commit a15090e005
2 changed files with 252 additions and 241 deletions
Download patch file Download diff file Expand all files Collapse all files

465
lib/vmpooler/pool_manager.rb
View file

@ -20,9 +20,9 @@ module Vmpooler
end end
# Check the state of a VM # Check the state of a VM
def check_pending_vm(vm, pool, timeout) def check_pending_vm(vm, pool, timeout, vsphere)
Thread.new do Thread.new do
_check_pending_vm(vm, pool, timeout) _check_pending_vm(vm, pool, timeout, vsphere)
end end
end end
@ -34,12 +34,14 @@ module Vmpooler
end end
end end
def _check_pending_vm(vm, pool, timeout) def _check_pending_vm(vm, pool, timeout, vsphere)
host = $vsphere[pool].find_vm(vm) host = vsphere.find_vm(vm)
if host if host
begin begin
open_socket vm, $config[:config]['domain'], timeout Timeout.timeout(5) do
TCPSocket.new vm, 22
end
move_pending_vm_to_ready(vm, pool, host) move_pending_vm_to_ready(vm, pool, host)
rescue rescue
fail_pending_vm(vm, pool, timeout) fail_pending_vm(vm, pool, timeout)
@ -87,7 +89,7 @@ module Vmpooler
end end
end end
def check_ready_vm(vm, pool, ttl) def check_ready_vm(vm, pool, ttl, vsphere)
Thread.new do Thread.new do
if ttl > 0 if ttl > 0
if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl if (((Time.now - host.runtime.bootTime) / 60).to_s[/^\d+\.\d{1}/].to_f) > ttl
@ -105,8 +107,8 @@ module Vmpooler
$redis.hset('vmpooler__vm__' + vm, 'check', Time.now) $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
host = $vsphere[pool].find_vm(vm) || host = vsphere.find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm] vsphere.find_vm_heavy(vm)[vm]
if host if host
if if
@ -147,14 +149,14 @@ module Vmpooler
end end
end end
def check_running_vm(vm, pool, ttl) def check_running_vm(vm, pool, ttl, vsphere)
Thread.new do Thread.new do
_check_running_vm(vm, pool, ttl) _check_running_vm(vm, pool, ttl, vsphere)
end end
end end
def _check_running_vm(vm, pool, ttl) def _check_running_vm(vm, pool, ttl, vsphere)
host = $vsphere[pool].find_vm(vm) host = vsphere.find_vm(vm)
if host if host
queue_from, queue_to = 'running', 'completed' queue_from, queue_to = 'running', 'completed'
@ -178,101 +180,105 @@ module Vmpooler
end end
# Clone a VM # Clone a VM
def clone_vm(template, folder, datastore, target) def clone_vm(template, folder, datastore, target, vsphere)
Thread.new do Thread.new do
vm = {}
if template =~ /\//
templatefolders = template.split('/')
vm['template'] = templatefolders.pop
end
if templatefolders
vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
else
fail 'Please provide a full path to the template'
end
if vm['template'].length == 0
fail "Unable to find template '#{vm['template']}'!"
end
# Generate a randomized hostname
o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
# Add VM to Redis inventory ('pending' pool)
$redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname'])
$redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now)
$redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template'])
# Annotate with creation time, origin template, etc.
# Add extraconfig options that can be queried by vmtools
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
annotation: JSON.pretty_generate(
name: vm['hostname'],
created_by: $config[:vsphere]['username'],
base_template: vm['template'],
creation_timestamp: Time.now.utc
),
extraConfig: [
{ key: 'guestinfo.hostname',
value: vm['hostname']
}
]
)
# Choose a clone target
if target
$clone_target = $vsphere[vm['template']].find_least_used_host(target)
elsif $config[:config]['clone_target']
$clone_target = $vsphere[vm['template']].find_least_used_host($config[:config]['clone_target'])
end
# Put the VM in the specified folder and resource pool
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
datastore: $vsphere[vm['template']].find_datastore(datastore),
host: $clone_target,
diskMoveType: :moveChildMostDiskBacking
)
# Create a clone spec
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
location: relocateSpec,
config: configSpec,
powerOn: true,
template: false
)
# Clone the VM
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
begin begin
start = Time.now vm = {}
vm[vm['template']].CloneVM_Task(
folder: $vsphere[vm['template']].find_folder(folder),
name: vm['hostname'],
spec: spec
).wait_for_completion
finish = '%.2f' % (Time.now - start)
$redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish) if template =~ /\//
$redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish) templatefolders = template.split('/')
vm['template'] = templatefolders.pop
end
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds") if templatefolders
rescue vm[vm['template']] = vsphere.find_folder(templatefolders.join('/')).find(vm['template'])
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed") else
$redis.srem('vmpooler__pending__' + vm['template'], vm['hostname']) fail 'Please provide a full path to the template'
end
if vm['template'].length == 0
fail "Unable to find template '#{vm['template']}'!"
end
# Generate a randomized hostname
o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
vm['hostname'] = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
# Add VM to Redis inventory ('pending' pool)
$redis.sadd('vmpooler__pending__' + vm['template'], vm['hostname'])
$redis.hset('vmpooler__vm__' + vm['hostname'], 'clone', Time.now)
$redis.hset('vmpooler__vm__' + vm['hostname'], 'template', vm['template'])
# Annotate with creation time, origin template, etc.
# Add extraconfig options that can be queried by vmtools
configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
annotation: JSON.pretty_generate(
name: vm['hostname'],
created_by: $config[:vsphere]['username'],
base_template: vm['template'],
creation_timestamp: Time.now.utc
),
extraConfig: [
{ key: 'guestinfo.hostname',
value: vm['hostname']
}
]
)
# Choose a clone target
if target
$clone_target = vsphere.find_least_used_host(target)
elsif $config[:config]['clone_target']
$clone_target = vsphere.find_least_used_host($config[:config]['clone_target'])
end
# Put the VM in the specified folder and resource pool
relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
datastore: vsphere.find_datastore(datastore),
host: $clone_target,
diskMoveType: :moveChildMostDiskBacking
)
# Create a clone spec
spec = RbVmomi::VIM.VirtualMachineCloneSpec(
location: relocateSpec,
config: configSpec,
powerOn: true,
template: false
)
# Clone the VM
$logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
begin
start = Time.now
vm[vm['template']].CloneVM_Task(
folder: vsphere.find_folder(folder),
name: vm['hostname'],
spec: spec
).wait_for_completion
finish = '%.2f' % (Time.now - start)
$redis.hset('vmpooler__clone__' + Date.today.to_s, vm['template'] + ':' + vm['hostname'], finish)
$redis.hset('vmpooler__vm__' + vm['hostname'], 'clone_time', finish)
$logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
rescue => err
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone failed with an error: #{err}")
$redis.srem('vmpooler__pending__' + vm['template'], vm['hostname'])
end
$redis.decr('vmpooler__tasks__clone')
$metrics.timing("clone.#{vm['template']}", finish)
rescue => err
$logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' failed while preparing to clone with an error: #{err}")
end end
$redis.decr('vmpooler__tasks__clone')
$metrics.timing("clone.#{vm['template']}", finish)
end end
end end
# Destroy a VM # Destroy a VM
def destroy_vm(vm, pool) def destroy_vm(vm, pool, vsphere)
Thread.new do Thread.new do
$redis.srem('vmpooler__completed__' + pool, vm) $redis.srem('vmpooler__completed__' + pool, vm)
$redis.hdel('vmpooler__active__' + pool, vm) $redis.hdel('vmpooler__active__' + pool, vm)
@ -281,8 +287,8 @@ module Vmpooler
# Auto-expire metadata key # Auto-expire metadata key
$redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60)) $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
host = $vsphere[pool].find_vm(vm) || host = vsphere.find_vm(vm) ||
$vsphere[pool].find_vm_heavy(vm)[vm] vsphere.find_vm_heavy(vm)[vm]
if host if host
start = Time.now start = Time.now
@ -305,15 +311,15 @@ module Vmpooler
end end
end end
def create_vm_disk(vm, disk_size) def create_vm_disk(vm, disk_size, vsphere)
Thread.new do Thread.new do
_create_vm_disk(vm, disk_size) _create_vm_disk(vm, disk_size, vsphere)
end end
end end
def _create_vm_disk(vm, disk_size) def _create_vm_disk(vm, disk_size, vsphere)
host = $vsphere['disk_manager'].find_vm(vm) || host = vsphere.find_vm(vm) ||
$vsphere['disk_manager'].find_vm_heavy(vm)[vm] vsphere.find_vm_heavy(vm)[vm]
if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0)) if (host) && ((! disk_size.nil?) && (! disk_size.empty?) && (disk_size.to_i > 0))
$logger.log('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk") $logger.log('s', "[ ] [disk_manager] '#{vm}' is attaching a #{disk_size}gb disk")
@ -330,7 +336,7 @@ module Vmpooler
end end
if ((! datastore.nil?) && (! datastore.empty?)) if ((! datastore.nil?) && (! datastore.empty?))
$vsphere['disk_manager'].add_disk(host, disk_size, datastore) vsphere.add_disk(host, disk_size, datastore)
rdisks = $redis.hget('vmpooler__vm__' + vm, 'disk') rdisks = $redis.hget('vmpooler__vm__' + vm, 'disk')
disks = rdisks ? rdisks.split(':') : [] disks = rdisks ? rdisks.split(':') : []
@ -346,15 +352,15 @@ module Vmpooler
end end
end end
def create_vm_snapshot(vm, snapshot_name) def create_vm_snapshot(vm, snapshot_name, vsphere)
Thread.new do Thread.new do
_create_vm_snapshot(vm, snapshot_name) _create_vm_snapshot(vm, snapshot_name, vsphere)
end end
end end
def _create_vm_snapshot(vm, snapshot_name) def _create_vm_snapshot(vm, snapshot_name, vsphere)
host = $vsphere['snapshot_manager'].find_vm(vm) || host = vsphere.find_vm(vm) ||
$vsphere['snapshot_manager'].find_vm_heavy(vm)[vm] vsphere.find_vm_heavy(vm)[vm]
if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?)) if (host) && ((! snapshot_name.nil?) && (! snapshot_name.empty?))
$logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being snapshotted") $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being snapshotted")
@ -376,18 +382,18 @@ module Vmpooler
end end
end end
def revert_vm_snapshot(vm, snapshot_name) def revert_vm_snapshot(vm, snapshot_name, vsphere)
Thread.new do Thread.new do
_revert_vm_snapshot(vm, snapshot_name) _revert_vm_snapshot(vm, snapshot_name, vsphere)
end end
end end
def _revert_vm_snapshot(vm, snapshot_name) def _revert_vm_snapshot(vm, snapshot_name, vsphere)
host = $vsphere['snapshot_manager'].find_vm(vm) || host = vsphere.find_vm(vm) ||
$vsphere['snapshot_manager'].find_vm_heavy(vm)[vm] vsphere.find_vm_heavy(vm)[vm]
if host if host
snapshot = $vsphere['snapshot_manager'].find_snapshot(host, snapshot_name) snapshot = vsphere.find_snapshot(host, snapshot_name)
if snapshot if snapshot
$logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being reverted to snapshot '#{snapshot_name}'") $logger.log('s', "[ ] [snapshot_manager] '#{vm}' is being reverted to snapshot '#{snapshot_name}'")
@ -410,19 +416,19 @@ module Vmpooler
$threads['disk_manager'] = Thread.new do $threads['disk_manager'] = Thread.new do
loop do loop do
_check_disk_queue _check_disk_queue $vsphere['disk_manager']
sleep(5) sleep(5)
end end
end end
end end
def _check_disk_queue def _check_disk_queue(vsphere)
vm = $redis.spop('vmpooler__tasks__disk') vm = $redis.spop('vmpooler__tasks__disk')
unless vm.nil? unless vm.nil?
begin begin
vm_name, disk_size = vm.split(':') vm_name, disk_size = vm.split(':')
create_vm_disk(vm_name, disk_size) create_vm_disk(vm_name, disk_size, vsphere)
rescue rescue
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed") $logger.log('s', "[!] [disk_manager] disk creation appears to have failed")
end end
@ -436,19 +442,19 @@ module Vmpooler
$threads['snapshot_manager'] = Thread.new do $threads['snapshot_manager'] = Thread.new do
loop do loop do
_check_snapshot_queue _check_snapshot_queue $vsphere['snapshot_manager']
sleep(5) sleep(5)
end end
end end
end end
def _check_snapshot_queue def _check_snapshot_queue(vsphere)
vm = $redis.spop('vmpooler__tasks__snapshot') vm = $redis.spop('vmpooler__tasks__snapshot')
unless vm.nil? unless vm.nil?
begin begin
vm_name, snapshot_name = vm.split(':') vm_name, snapshot_name = vm.split(':')
create_vm_snapshot(vm_name, snapshot_name) create_vm_snapshot(vm_name, snapshot_name, vsphere)
rescue rescue
$logger.log('s', "[!] [snapshot_manager] snapshot appears to have failed") $logger.log('s', "[!] [snapshot_manager] snapshot appears to have failed")
end end
@ -459,15 +465,15 @@ module Vmpooler
unless vm.nil? unless vm.nil?
begin begin
vm_name, snapshot_name = vm.split(':') vm_name, snapshot_name = vm.split(':')
revert_vm_snapshot(vm_name, snapshot_name) revert_vm_snapshot(vm_name, snapshot_name, vsphere)
rescue rescue
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed") $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed")
end end
end end
end end
def find_vsphere_pool_vm(pool, vm) def find_vsphere_pool_vm(pool, vm, vsphere)
$vsphere[pool].find_vm(vm) || $vsphere[pool].find_vm_heavy(vm)[vm] vsphere.find_vm(vm) || vsphere.find_vm_heavy(vm)[vm]
end end
def migration_limit(migration_limit) def migration_limit(migration_limit)
@ -476,45 +482,67 @@ module Vmpooler
migration_limit if migration_limit >= 1 migration_limit if migration_limit >= 1
end end
def migrate_vm(vm, pool) def migrate_vm(vm, pool, vsphere)
Thread.new do Thread.new do
_migrate_vm(vm, pool) _migrate_vm(vm, pool, vsphere)
end end
end end
def _migrate_vm(vm, pool) def _migrate_vm(vm, pool, vsphere)
$redis.srem('vmpooler__migrating__' + pool, vm) begin
vm_object = find_vsphere_pool_vm(pool, vm) $redis.srem('vmpooler__migrating__' + pool, vm)
parent_host = vm_object.summary.runtime.host vm_object = find_vsphere_pool_vm(pool, vm, vsphere)
parent_host_name = parent_host.name parent_host, parent_host_name = get_vm_host_info(vm_object)
migration_limit = migration_limit $config[:config]['migration_limit'] migration_limit = migration_limit $config[:config]['migration_limit']
if not migration_limit if not migration_limit
$logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}") $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}")
else
migration_count = $redis.smembers('vmpooler__migration').size
if migration_count >= migration_limit
$logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
else else
$redis.sadd('vmpooler__migration', vm) migration_count = $redis.smembers('vmpooler__migration').size
host = $vsphere[pool].find_least_used_compatible_host(vm_object) if migration_count >= migration_limit
if host == parent_host $logger.log('s', "[ ] [#{pool}] '#{vm}' is running on #{parent_host_name}. No migration will be evaluated since the migration_limit has been reached")
$logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
else else
start = Time.now $redis.sadd('vmpooler__migration', vm)
$vsphere[pool].migrate_vm_host(vm_object, host) host, host_name = vsphere.find_least_used_compatible_host(vm_object)
finish = '%.2f' % (Time.now - start) if host == parent_host
$metrics.timing("migrate.#{vm['template']}", finish) $logger.log('s', "[ ] [#{pool}] No migration required for '#{vm}' running on #{parent_host_name}")
checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget('vmpooler__vm__' + vm, 'checkout'))) else
$redis.hset('vmpooler__vm__' + vm, 'migration_time', finish) finish = migrate_vm_and_record_timing(vm_object, vm, host, vsphere)
$redis.hset('vmpooler__vm__' + vm, 'checkout_to_migration', checkout_to_migration) $logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host_name} in #{finish} seconds")
$logger.log('s', "[>] [#{pool}] '#{vm}' migrated from #{parent_host_name} to #{host.name} in #{finish} seconds") end
remove_vmpooler_migration_vm(pool, vm)
end end
$redis.srem('vmpooler__migration', vm)
end end
rescue => err
$logger.log('s', "[x] [#{pool}] '#{vm}' migration failed with an error: #{err}")
remove_vmpooler_migration_vm(pool, vm)
end end
end end
def get_vm_host_info(vm_object)
parent_host = vm_object.summary.runtime.host
[parent_host, parent_host.name]
end
def remove_vmpooler_migration_vm(pool, vm)
begin
$redis.srem('vmpooler__migration', vm)
rescue => err
$logger.log('s', "[x] [#{pool}] '#{vm}' removal from vmpooler__migration failed with an error: #{err}")
end
end
def migrate_vm_and_record_timing(vm_object, vm_name, host, vsphere)
start = Time.now
vsphere.migrate_vm_host(vm_object, host)
finish = '%.2f' % (Time.now - start)
$metrics.timing("migrate.#{vm_name}", finish)
checkout_to_migration = '%.2f' % (Time.now - Time.parse($redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
$redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)
$redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration)
finish
end
def check_pool(pool) def check_pool(pool)
$logger.log('d', "[*] [#{pool['name']}] starting worker thread") $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
@ -522,17 +550,17 @@ module Vmpooler
$threads[pool['name']] = Thread.new do $threads[pool['name']] = Thread.new do
loop do loop do
_check_pool(pool) _check_pool(pool, $vsphere[pool['name']])
sleep(5) sleep(5)
end end
end end
end end
def _check_pool(pool) def _check_pool(pool, vsphere)
# INVENTORY # INVENTORY
inventory = {} inventory = {}
begin begin
base = $vsphere[pool['name']].find_folder(pool['folder']) base = vsphere.find_folder(pool['folder'])
base.childEntity.each do |vm| base.childEntity.each do |vm|
if if
@ -554,111 +582,93 @@ module Vmpooler
end end
# RUNNING # RUNNING
running = $redis.smembers("vmpooler__running__#{pool['name']}") $redis.smembers("vmpooler__running__#{pool['name']}").each do |vm|
if running if inventory[vm]
running.each do |vm| begin
if inventory[vm] check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12, vsphere)
begin rescue
check_running_vm(vm, pool['name'], $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12)
rescue
end
end end
end end
end end
# READY # READY
ready = $redis.smembers("vmpooler__ready__#{pool['name']}") $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
if ready if inventory[vm]
ready.each do |vm| if ready begin
if inventory[vm] check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0, vsphere)
begin rescue
check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
rescue
end
end end
end end
end end
# PENDING # PENDING
pending = $redis.smembers('vmpooler__pending__' + pool['name']) $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
if pending if inventory[vm]
pending.each do |vm| begin
if inventory[vm] check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15, vsphere)
begin rescue
check_pending_vm(vm, pool['name'], pool['timeout'] || $config[:config]['timeout'] || 15)
rescue
end
end end
end end
end end
# COMPLETED # COMPLETED
completed = $redis.smembers('vmpooler__completed__' + pool['name']) $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm|
if completed if inventory[vm]
completed.each do |vm| begin
if inventory[vm] destroy_vm(vm, pool['name'], vsphere)
begin rescue
destroy_vm(vm, pool['name']) $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
rescue $redis.srem("vmpooler__completed__#{pool['name']}", vm)
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed") $redis.hdel("vmpooler__active__#{pool['name']}", vm)
$redis.srem('vmpooler__completed__' + pool['name'], vm) $redis.del("vmpooler__vm__#{vm}")
$redis.hdel('vmpooler__active__' + pool['name'], vm)
$redis.del('vmpooler__vm__' + vm)
end
else
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
$redis.srem('vmpooler__completed__' + pool['name'], vm)
$redis.hdel('vmpooler__active__' + pool['name'], vm)
$redis.del('vmpooler__vm__' + vm)
end end
else
$logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
$redis.srem("vmpooler__completed__#{pool['name']}", vm)
$redis.hdel("vmpooler__active__#{pool['name']}", vm)
$redis.del("vmpooler__vm__#{vm}")
end end
end end
# DISCOVERED # DISCOVERED
discovered = $redis.smembers("vmpooler__discovered__#{pool['name']}") $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm|
if discovered %w(pending ready running completed).each do |queue|
discovered.each do |vm| if $redis.sismember("vmpooler__#{queue}__#{pool['name']}", vm)
%w(pending ready running completed).each do |queue| $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
if $redis.sismember('vmpooler__' + queue + '__' + pool['name'], vm) $redis.srem("vmpooler__discovered__#{pool['name']}", vm)
$logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
$redis.srem('vmpooler__discovered__' + pool['name'], vm)
end
end end
end
if $redis.sismember('vmpooler__discovered__' + pool['name'], vm) if $redis.sismember("vmpooler__discovered__#{pool['name']}", vm)
$redis.smove('vmpooler__discovered__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm) $redis.smove("vmpooler__discovered__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
end
end end
end end
# MIGRATIONS # MIGRATIONS
migrations = $redis.smembers('vmpooler__migrating__' + pool['name']) $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm|
if migrations if inventory[vm]
migrations.each do |vm| begin
if inventory[vm] migrate_vm(vm, pool['name'], vsphere)
begin rescue => err
migrate_vm(vm, pool['name']) $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
rescue => err
$logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
end
end end
end end
end end
# REPOPULATE # REPOPULATE
ready = $redis.scard('vmpooler__ready__' + pool['name']) ready = $redis.scard("vmpooler__ready__#{pool['name']}")
total = $redis.scard('vmpooler__pending__' + pool['name']) + ready total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
$metrics.gauge('ready.' + pool['name'], $redis.scard('vmpooler__ready__' + pool['name'])) $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
$metrics.gauge('running.' + pool['name'], $redis.scard('vmpooler__running__' + pool['name'])) $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
if $redis.get('vmpooler__empty__' + pool['name']) if $redis.get("vmpooler__empty__#{pool['name']}")
unless ready == 0 unless ready == 0
$redis.del('vmpooler__empty__' + pool['name']) $redis.del("vmpooler__empty__#{pool['name']}")
end end
else else
if ready == 0 if ready == 0
$redis.set('vmpooler__empty__' + pool['name'], 'true') $redis.set("vmpooler__empty__#{pool['name']}", 'true')
$logger.log('s', "[!] [#{pool['name']}] is empty") $logger.log('s', "[!] [#{pool['name']}] is empty")
end end
end end
@ -673,10 +683,11 @@ module Vmpooler
pool['template'], pool['template'],
pool['folder'], pool['folder'],
pool['datastore'], pool['datastore'],
pool['clone_target'] pool['clone_target'],
vsphere
) )
rescue rescue => err
$logger.log('s', "[!] [#{pool['name']}] clone appears to have failed") $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
$redis.decr('vmpooler__tasks__clone') $redis.decr('vmpooler__tasks__clone')
end end
end end

28
lib/vmpooler/vsphere_helper.rb
View file

@ -26,7 +26,7 @@ module Vmpooler
end end
def add_disk(vm, size, datastore) def add_disk(vm, size, datastore)
ensure_connected @connection ensure_connected @connection, $credentials
return false unless size.to_i > 0 return false unless size.to_i > 0
@ -76,14 +76,14 @@ module Vmpooler
end end
def find_datastore(datastorename) def find_datastore(datastorename)
ensure_connected @connection ensure_connected @connection, $credentials
datacenter = @connection.serviceInstance.find_datacenter datacenter = @connection.serviceInstance.find_datacenter
datacenter.find_datastore(datastorename) datacenter.find_datastore(datastorename)
end end
def find_device(vm, deviceName) def find_device(vm, deviceName)
ensure_connected @connection ensure_connected @connection, $credentials
vm.config.hardware.device.each do |device| vm.config.hardware.device.each do |device|
return device if device.deviceInfo.label == deviceName return device if device.deviceInfo.label == deviceName
@ -93,7 +93,7 @@ module Vmpooler
end end
def find_disk_controller(vm) def find_disk_controller(vm)
ensure_connected @connection ensure_connected @connection, $credentials
devices = find_disk_devices(vm) devices = find_disk_devices(vm)
@ -107,7 +107,7 @@ module Vmpooler
end end
def find_disk_devices(vm) def find_disk_devices(vm)
ensure_connected @connection ensure_connected @connection, $credentials
devices = {} devices = {}
@ -135,7 +135,7 @@ module Vmpooler
end end
def find_disk_unit_number(vm, controller) def find_disk_unit_number(vm, controller)
ensure_connected @connection ensure_connected @connection, $credentials
used_unit_numbers = [] used_unit_numbers = []
available_unit_numbers = [] available_unit_numbers = []
@ -160,7 +160,7 @@ module Vmpooler
end end
def find_folder(foldername) def find_folder(foldername)
ensure_connected @connection ensure_connected @connection, $credentials
datacenter = @connection.serviceInstance.find_datacenter datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.vmFolder base = datacenter.vmFolder
@ -221,7 +221,7 @@ module Vmpooler
end end
def find_least_used_host(cluster) def find_least_used_host(cluster)
ensure_connected @connection ensure_connected @connection, $credentials
cluster_object = find_cluster(cluster) cluster_object = find_cluster(cluster)
target_hosts = get_cluster_host_utilization(cluster_object) target_hosts = get_cluster_host_utilization(cluster_object)
@ -244,7 +244,7 @@ module Vmpooler
end end
def find_least_used_compatible_host(vm) def find_least_used_compatible_host(vm)
ensure_connected @connection ensure_connected @connection, $credentials
source_host = vm.summary.runtime.host source_host = vm.summary.runtime.host
model = get_host_cpu_arch_version(source_host) model = get_host_cpu_arch_version(source_host)
@ -259,7 +259,7 @@ module Vmpooler
end end
def find_pool(poolname) def find_pool(poolname)
ensure_connected @connection ensure_connected @connection, $credentials
datacenter = @connection.serviceInstance.find_datacenter datacenter = @connection.serviceInstance.find_datacenter
base = datacenter.hostFolder base = datacenter.hostFolder
@ -288,13 +288,13 @@ module Vmpooler
end end
def find_vm(vmname) def find_vm(vmname)
ensure_connected @connection ensure_connected @connection, $credentials
@connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname) @connection.searchIndex.FindByDnsName(vmSearch: true, dnsName: vmname)
end end
def find_vm_heavy(vmname) def find_vm_heavy(vmname)
ensure_connected @connection ensure_connected @connection, $credentials
vmname = vmname.is_a?(Array) ? vmname : [vmname] vmname = vmname.is_a?(Array) ? vmname : [vmname]
containerView = get_base_vm_container_from @connection containerView = get_base_vm_container_from @connection
@ -344,7 +344,7 @@ module Vmpooler
end end
def find_vmdks(vmname, datastore) def find_vmdks(vmname, datastore)
ensure_connected @connection ensure_connected @connection, $credentials
disks = [] disks = []
@ -363,7 +363,7 @@ module Vmpooler
end end
def get_base_vm_container_from(connection) def get_base_vm_container_from(connection)
ensure_connected @connection ensure_connected @connection, $credentials
viewManager = connection.serviceContent.viewManager viewManager = connection.serviceContent.viewManager
viewManager.CreateContainerView( viewManager.CreateContainerView(