mirror of
https://github.com/puppetlabs/vmpooler-provider-vsphere.git
synced 2026-03-26 23:47:43 -04:00
Wire circuit breaker into vsphere provider public methods
Without this change, the circuit breaker was initialized in base.rb but never called in the vsphere provider, so vSphere API failures (TCP timeouts, connection errors) would not trip the circuit open. Changes: - Add with_circuit_breaker helper that calls @circuit_breaker.call if circuit_breaker is configured, otherwise yields directly - Wrap vms_in_pool, get_vm, create_vm, destroy_vm, get_vm_ip_address with with_circuit_breaker so vSphere failures trip the circuit - Replace 'return' with 'next' inside blocks where needed to ensure circuit breaker on_success is properly called on partial results This prevents cascading failures: once the circuit opens after failure_threshold errors, subsequent calls fail fast (CircuitOpenError) instead of waiting for the full TCP timeout (~18s per pool). Resolves: P4DEVOPS-9438
This commit is contained in:
parent
3c7bd24fa4
commit
4a57a270f6
1 changed files with 107 additions and 87 deletions
|
|
@ -186,16 +186,18 @@ module Vmpooler
|
|||
|
||||
def vms_in_pool(pool_name)
|
||||
vms = []
|
||||
with_circuit_breaker do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
folder_object = find_vm_folder(pool_name, connection)
|
||||
|
||||
return vms if folder_object.nil?
|
||||
next if folder_object.nil?
|
||||
|
||||
folder_object.childEntity.each do |vm|
|
||||
vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
|
||||
end
|
||||
end
|
||||
end
|
||||
vms
|
||||
end
|
||||
|
||||
|
|
@ -305,13 +307,15 @@ module Vmpooler
|
|||
|
||||
def get_vm(pool_name, vm_name)
|
||||
vm_hash = nil
|
||||
with_circuit_breaker do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
vm_object = find_vm(pool_name, vm_name, connection)
|
||||
return vm_hash if vm_object.nil?
|
||||
next if vm_object.nil?
|
||||
|
||||
vm_hash = generate_vm_hash(vm_object, pool_name)
|
||||
end
|
||||
end
|
||||
vm_hash
|
||||
end
|
||||
|
||||
|
|
@ -320,6 +324,7 @@ module Vmpooler
|
|||
raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
|
||||
|
||||
vm_hash = nil
|
||||
with_circuit_breaker do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
# Assume all pool config is valid i.e. not missing
|
||||
|
|
@ -389,18 +394,23 @@ module Vmpooler
|
|||
|
||||
vm_hash = generate_vm_hash(new_vm_object, pool_name)
|
||||
end
|
||||
end
|
||||
vm_hash
|
||||
end
|
||||
|
||||
# The inner method requires vmware tools running in the guest os
|
||||
def get_vm_ip_address(vm_name, pool_name)
|
||||
ip = nil
|
||||
with_circuit_breaker do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
vm_object = find_vm(pool_name, vm_name, connection)
|
||||
vm_hash = generate_vm_hash(vm_object, pool_name)
|
||||
return vm_hash['ip']
|
||||
ip = vm_hash['ip']
|
||||
end
|
||||
end
|
||||
ip
|
||||
end
|
||||
|
||||
def create_config_spec(vm_name, template_name, extra_config)
|
||||
RbVmomi::VIM.VirtualMachineConfigSpec(
|
||||
|
|
@ -540,11 +550,12 @@ module Vmpooler
|
|||
end
|
||||
|
||||
def destroy_vm(pool_name, vm_name)
|
||||
with_circuit_breaker do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
vm_object = find_vm(pool_name, vm_name, connection)
|
||||
# If a VM doesn't exist then it is effectively deleted
|
||||
return true if vm_object.nil?
|
||||
next if vm_object.nil?
|
||||
|
||||
# Poweroff the VM if it's running
|
||||
vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn'
|
||||
|
|
@ -552,6 +563,7 @@ module Vmpooler
|
|||
# Kill it with fire
|
||||
vm_object.Destroy_Task.wait_for_completion
|
||||
end
|
||||
end
|
||||
true
|
||||
end
|
||||
|
||||
|
|
@ -631,6 +643,14 @@ module Vmpooler
|
|||
DISK_TYPE = 'thin'
|
||||
DISK_MODE = 'persistent'
|
||||
|
||||
def with_circuit_breaker(&block)
|
||||
if circuit_breaker
|
||||
circuit_breaker.call(&block)
|
||||
else
|
||||
yield
|
||||
end
|
||||
end
|
||||
|
||||
def ensured_vsphere_connection(connection_pool_object)
|
||||
connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection])
|
||||
connection_pool_object[:connection]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue