Wrap vms_in_pool/get_vm/get_vm_ip_address with Timeout.timeout

The read_timeout on Net::HTTP connections may not reliably interrupt
blocking Java SSL socket reads in JRuby. Add an explicit Timeout.timeout
wrapper around the connection pool block in vms_in_pool, get_vm, and
get_vm_ip_address so that if vSphere hangs mid-operation (after the
health check passes), the Timeout::Error (a StandardError in JRuby 9.x)
propagates to the circuit breaker which counts it as a failure.

After 5 consecutive timeouts the circuit opens and subsequent check_pool
cycles fail immediately rather than blocking all pool threads.
This commit is contained in:
Mahima Singh 2026-03-13 13:29:18 +05:30
parent d9aee6baee
commit f2390311df

View file

@ -188,6 +188,7 @@ module Vmpooler
def vms_in_pool(pool_name)
vms = []
with_circuit_breaker do
Timeout.timeout(vsphere_connection_timeout) do
@connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object)
folder_object = find_vm_folder(pool_name, connection)
@ -199,6 +200,7 @@ module Vmpooler
end
end
end
end
vms
end
@ -309,6 +311,7 @@ module Vmpooler
def get_vm(pool_name, vm_name)
vm_hash = nil
with_circuit_breaker do
Timeout.timeout(vsphere_connection_timeout) do
@connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object)
vm_object = find_vm(pool_name, vm_name, connection)
@ -317,6 +320,7 @@ module Vmpooler
vm_hash = generate_vm_hash(vm_object, pool_name)
end
end
end
vm_hash
end
@ -403,6 +407,7 @@ module Vmpooler
def get_vm_ip_address(vm_name, pool_name)
ip = nil
with_circuit_breaker do
Timeout.timeout(vsphere_connection_timeout) do
@connection_pool.with_metrics do |pool_object|
connection = ensured_vsphere_connection(pool_object)
vm_object = find_vm(pool_name, vm_name, connection)
@ -410,6 +415,7 @@ module Vmpooler
ip = vm_hash['ip']
end
end
end
ip
end
@ -658,9 +664,7 @@ module Vmpooler
end
def vsphere_connection_ok?(connection)
Timeout.timeout(vsphere_connection_timeout) do
_result = connection.serviceInstance.CurrentTime
end
true
rescue StandardError
false
@ -675,7 +679,8 @@ module Vmpooler
user: provider_config['username'],
password: provider_config['password'],
insecure: provider_config['insecure'] || false,
timeout: vsphere_connection_timeout
read_timeout: vsphere_connection_timeout,
open_timeout: vsphere_connection_timeout
metrics.increment('connect.open')
connection
rescue StandardError => e