puppetlabs/vmpooler-provider-vsphere
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler-provider-vsphere.git synced 2026-03-26 23:47:43 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Wire circuit breaker into vsphere provider public methods

Browse source 
Without this change, the circuit breaker was initialized in base.rb
but never called in the vsphere provider, so vSphere API failures
(TCP timeouts, connection errors) would not trip the circuit open.

Changes:
- Add with_circuit_breaker helper that calls @circuit_breaker.call
  if circuit_breaker is configured, otherwise yields directly
- Wrap vms_in_pool, get_vm, create_vm, destroy_vm, get_vm_ip_address
  with with_circuit_breaker so vSphere failures trip the circuit
- Replace 'return' with 'next' inside blocks where needed to ensure
  circuit breaker on_success is properly called on partial results

This prevents cascading failures: once the circuit opens after
failure_threshold errors, subsequent calls fail fast (CircuitOpenError)
instead of waiting for the full TCP timeout (~18s per pool).

Resolves: P4DEVOPS-9438
This commit is contained in:
Mahima Singh 2026-03-12 16:49:20 +05:30
parent 3c7bd24fa4
commit 4a57a270f6
1 changed files with 107 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

194
lib/vmpooler/providers/vsphere.rb
View file

@ -186,14 +186,16 @@ module Vmpooler
def vms_in_pool(pool_name) def vms_in_pool(pool_name)
vms = [] vms = []
@connection_pool.with_metrics do |pool_object| with_circuit_breaker do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
folder_object = find_vm_folder(pool_name, connection) connection = ensured_vsphere_connection(pool_object)
folder_object = find_vm_folder(pool_name, connection)
return vms if folder_object.nil? next if folder_object.nil?
folder_object.childEntity.each do |vm| folder_object.childEntity.each do |vm|
vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
end
end end
end end
vms vms
@ -305,12 +307,14 @@ module Vmpooler
def get_vm(pool_name, vm_name) def get_vm(pool_name, vm_name)
vm_hash = nil vm_hash = nil
@connection_pool.with_metrics do |pool_object| with_circuit_breaker do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
vm_object = find_vm(pool_name, vm_name, connection) connection = ensured_vsphere_connection(pool_object)
return vm_hash if vm_object.nil? vm_object = find_vm(pool_name, vm_name, connection)
next if vm_object.nil?
vm_hash = generate_vm_hash(vm_object, pool_name) vm_hash = generate_vm_hash(vm_object, pool_name)
end
end end
vm_hash vm_hash
end end
@ -320,86 +324,92 @@ module Vmpooler
raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
vm_hash = nil vm_hash = nil
@connection_pool.with_metrics do |pool_object| with_circuit_breaker do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
# Assume all pool config is valid i.e. not missing connection = ensured_vsphere_connection(pool_object)
template_path = pool['template'] # Assume all pool config is valid i.e. not missing
target_folder_path = pool['folder'] template_path = pool['template']
target_datastore = pool['datastore'] target_folder_path = pool['folder']
target_datacenter_name = get_target_datacenter_from_config(pool_name) target_datastore = pool['datastore']
target_datacenter_name = get_target_datacenter_from_config(pool_name)
# Get the template VM object # Get the template VM object
raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless valid_template_path? template_path raise("Pool #{pool_name} did not specify a full path for the template for the provider #{name}") unless valid_template_path? template_path
template_vm_object = find_template_vm(pool, connection) template_vm_object = find_template_vm(pool, connection)
extra_config = [ extra_config = [
{ key: 'guestinfo.hostname', value: new_vmname } { key: 'guestinfo.hostname', value: new_vmname }
] ]
if pool.key?('snapshot_mainMem_ioBlockPages') if pool.key?('snapshot_mainMem_ioBlockPages')
ioblockpages = pool['snapshot_mainMem_ioBlockPages'] ioblockpages = pool['snapshot_mainMem_ioBlockPages']
extra_config.push( extra_config.push(
{ key: 'mainMem.ioBlockPages', value: ioblockpages } { key: 'mainMem.ioBlockPages', value: ioblockpages }
) )
end
if pool.key?('snapshot_mainMem_iowait')
iowait = pool['snapshot_mainMem_iowait']
extra_config.push(
{ key: 'mainMem.iowait', value: iowait }
)
end
# Annotate with creation time, origin template, etc.
# Add extraconfig options that can be queried by vmtools
config_spec = create_config_spec(new_vmname, template_path, extra_config)
# Check if alternate network configuration is specified and add configuration
if pool.key?('network')
template_vm_network_device = template_vm_object.config.hardware.device.grep(RbVmomi::VIM::VirtualEthernetCard).first
network_name = pool['network']
network_device = set_network_device(target_datacenter_name, template_vm_network_device, network_name, connection)
config_spec.deviceChange = [{ operation: 'edit', device: network_device }]
end
# Put the VM in the specified folder and resource pool
relocate_spec = create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
# Create a clone spec
clone_spec = create_clone_spec(relocate_spec, config_spec)
begin
vm_target_folder = find_vm_folder(pool_name, connection)
vm_target_folder ||= create_folder(connection, target_folder_path, target_datacenter_name) if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
rescue StandardError
if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
else
raise
end end
if pool.key?('snapshot_mainMem_iowait')
iowait = pool['snapshot_mainMem_iowait']
extra_config.push(
{ key: 'mainMem.iowait', value: iowait }
)
end
# Annotate with creation time, origin template, etc.
# Add extraconfig options that can be queried by vmtools
config_spec = create_config_spec(new_vmname, template_path, extra_config)
# Check if alternate network configuration is specified and add configuration
if pool.key?('network')
template_vm_network_device = template_vm_object.config.hardware.device.grep(RbVmomi::VIM::VirtualEthernetCard).first
network_name = pool['network']
network_device = set_network_device(target_datacenter_name, template_vm_network_device, network_name, connection)
config_spec.deviceChange = [{ operation: 'edit', device: network_device }]
end
# Put the VM in the specified folder and resource pool
relocate_spec = create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
# Create a clone spec
clone_spec = create_clone_spec(relocate_spec, config_spec)
begin
vm_target_folder = find_vm_folder(pool_name, connection)
vm_target_folder ||= create_folder(connection, target_folder_path, target_datacenter_name) if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
rescue StandardError
if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
else
raise
end
end
raise ArgumentError, "Cannot find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
# Create the new VM
new_vm_object = template_vm_object.CloneVM_Task(
folder: vm_target_folder,
name: new_vmname,
spec: clone_spec
).wait_for_completion
vm_hash = generate_vm_hash(new_vm_object, pool_name)
end end
raise ArgumentError, "Cannot find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
# Create the new VM
new_vm_object = template_vm_object.CloneVM_Task(
folder: vm_target_folder,
name: new_vmname,
spec: clone_spec
).wait_for_completion
vm_hash = generate_vm_hash(new_vm_object, pool_name)
end end
vm_hash vm_hash
end end
# The inner method requires vmware tools running in the guest os # The inner method requires vmware tools running in the guest os
def get_vm_ip_address(vm_name, pool_name) def get_vm_ip_address(vm_name, pool_name)
@connection_pool.with_metrics do |pool_object| ip = nil
connection = ensured_vsphere_connection(pool_object) with_circuit_breaker do
vm_object = find_vm(pool_name, vm_name, connection) @connection_pool.with_metrics do |pool_object|
vm_hash = generate_vm_hash(vm_object, pool_name) connection = ensured_vsphere_connection(pool_object)
return vm_hash['ip'] vm_object = find_vm(pool_name, vm_name, connection)
vm_hash = generate_vm_hash(vm_object, pool_name)
ip = vm_hash['ip']
end
end end
ip
end end
def create_config_spec(vm_name, template_name, extra_config) def create_config_spec(vm_name, template_name, extra_config)
@ -540,17 +550,19 @@ module Vmpooler
end end
def destroy_vm(pool_name, vm_name) def destroy_vm(pool_name, vm_name)
@connection_pool.with_metrics do |pool_object| with_circuit_breaker do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
vm_object = find_vm(pool_name, vm_name, connection) connection = ensured_vsphere_connection(pool_object)
# If a VM doesn't exist then it is effectively deleted vm_object = find_vm(pool_name, vm_name, connection)
return true if vm_object.nil? # If a VM doesn't exist then it is effectively deleted
next if vm_object.nil?
# Poweroff the VM if it's running # Poweroff the VM if it's running
vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn' vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn'
# Kill it with fire # Kill it with fire
vm_object.Destroy_Task.wait_for_completion vm_object.Destroy_Task.wait_for_completion
end
end end
true true
end end
@ -631,6 +643,14 @@ module Vmpooler
DISK_TYPE = 'thin' DISK_TYPE = 'thin'
DISK_MODE = 'persistent' DISK_MODE = 'persistent'
def with_circuit_breaker(&block)
if circuit_breaker
circuit_breaker.call(&block)
else
yield
end
end
def ensured_vsphere_connection(connection_pool_object) def ensured_vsphere_connection(connection_pool_object)
connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection]) connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection])
connection_pool_object[:connection] connection_pool_object[:connection]