Wire circuit breaker into vsphere provider public methods

Without this change, the circuit breaker was initialized in base.rb
but never called in the vsphere provider, so vSphere API failures
(TCP timeouts, connection errors) would not trip the circuit open.

Changes:
- Add with_circuit_breaker helper that calls @circuit_breaker.call
  if circuit_breaker is configured, otherwise yields directly
- Wrap vms_in_pool, get_vm, create_vm, destroy_vm, get_vm_ip_address
  with with_circuit_breaker so vSphere failures trip the circuit
- Replace 'return' with 'next' inside blocks where needed to ensure
  circuit breaker on_success is properly called on partial results

This prevents cascading failures: once the circuit opens after
failure_threshold errors, subsequent calls fail fast (CircuitOpenError)
instead of waiting for the full TCP timeout (~18s per pool).

Resolves: P4DEVOPS-9438
This commit is contained in:
Mahima Singh 2026-03-12 16:49:20 +05:30
parent 3c7bd24fa4
commit 4a57a270f6

View file

@ -186,16 +186,18 @@ module Vmpooler
def vms_in_pool(pool_name) def vms_in_pool(pool_name)
vms = [] vms = []
with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object) connection = ensured_vsphere_connection(pool_object)
folder_object = find_vm_folder(pool_name, connection) folder_object = find_vm_folder(pool_name, connection)
return vms if folder_object.nil? next if folder_object.nil?
folder_object.childEntity.each do |vm| folder_object.childEntity.each do |vm|
vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
end end
end end
end
vms vms
end end
@ -305,13 +307,15 @@ module Vmpooler
def get_vm(pool_name, vm_name) def get_vm(pool_name, vm_name)
vm_hash = nil vm_hash = nil
with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object) connection = ensured_vsphere_connection(pool_object)
vm_object = find_vm(pool_name, vm_name, connection) vm_object = find_vm(pool_name, vm_name, connection)
return vm_hash if vm_object.nil? next if vm_object.nil?
vm_hash = generate_vm_hash(vm_object, pool_name) vm_hash = generate_vm_hash(vm_object, pool_name)
end end
end
vm_hash vm_hash
end end
@ -320,6 +324,7 @@ module Vmpooler
raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
vm_hash = nil vm_hash = nil
with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object) connection = ensured_vsphere_connection(pool_object)
# Assume all pool config is valid i.e. not missing # Assume all pool config is valid i.e. not missing
@ -389,18 +394,23 @@ module Vmpooler
vm_hash = generate_vm_hash(new_vm_object, pool_name) vm_hash = generate_vm_hash(new_vm_object, pool_name)
end end
end
vm_hash vm_hash
end end
# The inner method requires vmware tools running in the guest os # The inner method requires vmware tools running in the guest os
def get_vm_ip_address(vm_name, pool_name) def get_vm_ip_address(vm_name, pool_name)
ip = nil
with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object) connection = ensured_vsphere_connection(pool_object)
vm_object = find_vm(pool_name, vm_name, connection) vm_object = find_vm(pool_name, vm_name, connection)
vm_hash = generate_vm_hash(vm_object, pool_name) vm_hash = generate_vm_hash(vm_object, pool_name)
return vm_hash['ip'] ip = vm_hash['ip']
end end
end end
ip
end
def create_config_spec(vm_name, template_name, extra_config) def create_config_spec(vm_name, template_name, extra_config)
RbVmomi::VIM.VirtualMachineConfigSpec( RbVmomi::VIM.VirtualMachineConfigSpec(
@ -540,11 +550,12 @@ module Vmpooler
end end
def destroy_vm(pool_name, vm_name) def destroy_vm(pool_name, vm_name)
with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| @connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object) connection = ensured_vsphere_connection(pool_object)
vm_object = find_vm(pool_name, vm_name, connection) vm_object = find_vm(pool_name, vm_name, connection)
# If a VM doesn't exist then it is effectively deleted # If a VM doesn't exist then it is effectively deleted
return true if vm_object.nil? next if vm_object.nil?
# Poweroff the VM if it's running # Poweroff the VM if it's running
vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn' vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn'
@ -552,6 +563,7 @@ module Vmpooler
# Kill it with fire # Kill it with fire
vm_object.Destroy_Task.wait_for_completion vm_object.Destroy_Task.wait_for_completion
end end
end
true true
end end
@ -631,6 +643,14 @@ module Vmpooler
DISK_TYPE = 'thin' DISK_TYPE = 'thin'
DISK_MODE = 'persistent' DISK_MODE = 'persistent'
def with_circuit_breaker(&block)
if circuit_breaker
circuit_breaker.call(&block)
else
yield
end
end
def ensured_vsphere_connection(connection_pool_object) def ensured_vsphere_connection(connection_pool_object)
connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection]) connection_pool_object[:connection] = connect_to_vsphere unless vsphere_connection_ok?(connection_pool_object[:connection])
connection_pool_object[:connection] connection_pool_object[:connection]