mirror of
https://github.com/puppetlabs/vmpooler-provider-vsphere.git
synced 2026-03-26 23:47:43 -04:00
Wrap vms_in_pool/get_vm/get_vm_ip_address with Timeout.timeout
The read_timeout on Net::HTTP connections may not reliably interrupt blocking Java SSL socket reads in JRuby. Add an explicit Timeout.timeout wrapper around the connection pool block in vms_in_pool, get_vm, and get_vm_ip_address so that if vSphere hangs mid-operation (after the health check passes), the Timeout::Error (a StandardError in JRuby 9.x) propagates to the circuit breaker which counts it as a failure. After 5 consecutive timeouts the circuit opens and subsequent check_pool cycles fail immediately rather than blocking all pool threads.
This commit is contained in:
parent
d9aee6baee
commit
f2390311df
1 changed files with 25 additions and 20 deletions
|
|
@ -188,6 +188,7 @@ module Vmpooler
|
|||
def vms_in_pool(pool_name)
|
||||
vms = []
|
||||
with_circuit_breaker do
|
||||
Timeout.timeout(vsphere_connection_timeout) do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
folder_object = find_vm_folder(pool_name, connection)
|
||||
|
|
@ -199,6 +200,7 @@ module Vmpooler
|
|||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
vms
|
||||
end
|
||||
|
||||
|
|
@ -309,6 +311,7 @@ module Vmpooler
|
|||
def get_vm(pool_name, vm_name)
|
||||
vm_hash = nil
|
||||
with_circuit_breaker do
|
||||
Timeout.timeout(vsphere_connection_timeout) do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
vm_object = find_vm(pool_name, vm_name, connection)
|
||||
|
|
@ -317,6 +320,7 @@ module Vmpooler
|
|||
vm_hash = generate_vm_hash(vm_object, pool_name)
|
||||
end
|
||||
end
|
||||
end
|
||||
vm_hash
|
||||
end
|
||||
|
||||
|
|
@ -403,6 +407,7 @@ module Vmpooler
|
|||
def get_vm_ip_address(vm_name, pool_name)
|
||||
ip = nil
|
||||
with_circuit_breaker do
|
||||
Timeout.timeout(vsphere_connection_timeout) do
|
||||
@connection_pool.with_metrics do |pool_object|
|
||||
connection = ensured_vsphere_connection(pool_object)
|
||||
vm_object = find_vm(pool_name, vm_name, connection)
|
||||
|
|
@ -410,6 +415,7 @@ module Vmpooler
|
|||
ip = vm_hash['ip']
|
||||
end
|
||||
end
|
||||
end
|
||||
ip
|
||||
end
|
||||
|
||||
|
|
@ -658,9 +664,7 @@ module Vmpooler
|
|||
end
|
||||
|
||||
def vsphere_connection_ok?(connection)
|
||||
Timeout.timeout(vsphere_connection_timeout) do
|
||||
_result = connection.serviceInstance.CurrentTime
|
||||
end
|
||||
true
|
||||
rescue StandardError
|
||||
false
|
||||
|
|
@ -675,7 +679,8 @@ module Vmpooler
|
|||
user: provider_config['username'],
|
||||
password: provider_config['password'],
|
||||
insecure: provider_config['insecure'] || false,
|
||||
timeout: vsphere_connection_timeout
|
||||
read_timeout: vsphere_connection_timeout,
|
||||
open_timeout: vsphere_connection_timeout
|
||||
metrics.increment('connect.open')
|
||||
connection
|
||||
rescue StandardError => e
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue