puppetlabs/vmpooler-provider-vsphere
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmpooler-provider-vsphere.git synced 2026-03-26 23:47:43 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Wrap vms_in_pool/get_vm/get_vm_ip_address with Timeout.timeout

Browse source 
The read_timeout on Net::HTTP connections may not reliably interrupt
blocking Java SSL socket reads in JRuby. Add an explicit Timeout.timeout
wrapper around the connection pool block in vms_in_pool, get_vm, and
get_vm_ip_address so that if vSphere hangs mid-operation (after the
health check passes), the Timeout::Error (a StandardError in JRuby 9.x)
propagates to the circuit breaker which counts it as a failure.

After 5 consecutive timeouts the circuit opens and subsequent check_pool
cycles fail immediately rather than blocking all pool threads.
This commit is contained in:
Mahima Singh 2026-03-13 13:29:18 +05:30
parent d9aee6baee
commit f2390311df
1 changed files with 25 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

45
lib/vmpooler/providers/vsphere.rb
View file

@ -188,14 +188,16 @@ module Vmpooler
def vms_in_pool(pool_name) def vms_in_pool(pool_name)
vms = [] vms = []
with_circuit_breaker do with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| Timeout.timeout(vsphere_connection_timeout) do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
folder_object = find_vm_folder(pool_name, connection) connection = ensured_vsphere_connection(pool_object)
folder_object = find_vm_folder(pool_name, connection)
next if folder_object.nil? next if folder_object.nil?
folder_object.childEntity.each do |vm| folder_object.childEntity.each do |vm|
vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine vms << { 'name' => vm.name } if vm.is_a? RbVmomi::VIM::VirtualMachine
end
end end
end end
end end
@ -309,12 +311,14 @@ module Vmpooler
def get_vm(pool_name, vm_name) def get_vm(pool_name, vm_name)
vm_hash = nil vm_hash = nil
with_circuit_breaker do with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| Timeout.timeout(vsphere_connection_timeout) do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
vm_object = find_vm(pool_name, vm_name, connection) connection = ensured_vsphere_connection(pool_object)
next if vm_object.nil? vm_object = find_vm(pool_name, vm_name, connection)
next if vm_object.nil?
vm_hash = generate_vm_hash(vm_object, pool_name) vm_hash = generate_vm_hash(vm_object, pool_name)
end
end end
end end
vm_hash vm_hash
@ -403,11 +407,13 @@ module Vmpooler
def get_vm_ip_address(vm_name, pool_name) def get_vm_ip_address(vm_name, pool_name)
ip = nil ip = nil
with_circuit_breaker do with_circuit_breaker do
@connection_pool.with_metrics do |pool_object| Timeout.timeout(vsphere_connection_timeout) do
connection = ensured_vsphere_connection(pool_object) @connection_pool.with_metrics do |pool_object|
vm_object = find_vm(pool_name, vm_name, connection) connection = ensured_vsphere_connection(pool_object)
vm_hash = generate_vm_hash(vm_object, pool_name) vm_object = find_vm(pool_name, vm_name, connection)
ip = vm_hash['ip'] vm_hash = generate_vm_hash(vm_object, pool_name)
ip = vm_hash['ip']
end
end end
end end
ip ip
@ -658,9 +664,7 @@ module Vmpooler
end end
def vsphere_connection_ok?(connection) def vsphere_connection_ok?(connection)
Timeout.timeout(vsphere_connection_timeout) do _result = connection.serviceInstance.CurrentTime
_result = connection.serviceInstance.CurrentTime
end
true true
rescue StandardError rescue StandardError
false false
@ -675,7 +679,8 @@ module Vmpooler
user: provider_config['username'], user: provider_config['username'],
password: provider_config['password'], password: provider_config['password'],
insecure: provider_config['insecure'] || false, insecure: provider_config['insecure'] || false,
timeout: vsphere_connection_timeout read_timeout: vsphere_connection_timeout,
open_timeout: vsphere_connection_timeout
metrics.increment('connect.open') metrics.increment('connect.open')
connection connection
rescue StandardError => e rescue StandardError => e