puppetlabs/vmfloaty
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmfloaty.git synced 2026-01-26 05:28:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

The main goal for this PR is to enable using all of the vmfloaty scenarios when

Browse source 
the service is set to ABS. Since ABS does not implement all the services, it
fallsback on vmpooler when needed (example increasing the lifetime value)

- Validating JSON can be parsed before parsing, as it used to throw uncaught errors
- self.list_active is returning hosts for both nspooler and vmpooler but was returning
job_ids for abs. Now standardized the returned values for use in the "modify" cli,
and created a separate list_active_job_ids for the cases where a job_id list is expected

- added a way to change the service in place for methods that do not make sense
for ABS, and instead fallback on using vmpooler in those cases:
1. summary
2. modify
3. snapshot
4. revert
5. disk
For those methods in the class itself, raising a NoMethodError, in case it is used directly

- query now returns the queue info. For information on VMs, users should use --service vmpooler
- pretty_print_hosts (used in list, and delete scenarios) will now print the job_id
ABS information and indent each VM within and print it's metadata. Useful for
knowing the running time and extending it.

- added a new utility method to get the config for the vmpooler service, used for the fallback
- added a passthrough for the vmpooler token to use when running ABS. This enables
vmpooler to track the VMs used by each token (user). Also aligns the list between
both ABS and vmpooler. Fixes the bit-bar issue where the VMs do not appear when created
via ABS
This commit is contained in:
Samuel Beaulieu 2020-09-10 13:49:46 -05:00
parent 5a0640c515
commit dbf6c54173
4 changed files with 166 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

133
lib/vmfloaty/abs.rb
View file

@ -2,6 +2,7 @@
require 'vmfloaty/errors'
require 'vmfloaty/http'
require 'vmfloaty/utils'
require 'faraday'
require 'json'
@ -36,39 +37,59 @@ class ABS
# }
# }
#
@active_hostnames = {}
def self.list_active(verbose, url, _token, user)
all_jobs = []
def self.list_active_job_ids(verbose, url, user)
all_job_ids = []
@active_hostnames = {}
get_active_requests(verbose, url, user).each do |req_hash|
all_jobs.push(req_hash['request']['job']['id'])
@active_hostnames[req_hash['request']['job']['id']] = req_hash
@active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
all_job_ids.push(req_hash['request']['job']['id'])
end
all_jobs
all_job_ids
end
def self.list_active(verbose, url, _token, user)
hosts = []
get_active_requests(verbose, url, user).each do |req_hash|
if req_hash.key?('allocated_resources')
req_hash['allocated_resources'].each do |onehost|
hosts.push(onehost['hostname'])
end
end
end
hosts
end
def self.get_active_requests(verbose, url, user)
conn = Http.get_conn(verbose, url)
res = conn.get 'status/queue'
requests = JSON.parse(res.body)
if valid_json?(res.body)
requests = JSON.parse(res.body)
else
FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
end
ret_val = []
requests.each do |req|
next if req == 'null'
req_hash = JSON.parse(req)
if valid_json?(req)
req_hash = JSON.parse(req)
else
FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
next
end
begin
next unless user == req_hash['request']['job']['user']
ret_val.push(req_hash)
rescue NoMethodError
FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: "
FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
end
end
@ -145,30 +166,43 @@ class ABS
os_list = []
res = conn.get 'status/platforms/vmpooler'
res_body = JSON.parse(res.body)
os_list << '*** VMPOOLER Pools ***'
os_list += JSON.parse(res_body['vmpooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('vmpooler_platforms')
os_list << '*** VMPOOLER Pools ***'
os_list += JSON.parse(res_body['vmpooler_platforms'])
end
end
res = conn.get 'status/platforms/ondemand_vmpooler'
res_body = JSON.parse(res.body)
unless res_body['ondemand_vmpooler_platforms'] == '[]'
os_list << ''
os_list << '*** VMPOOLER ONDEMAND Pools ***'
os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
os_list << ''
os_list << '*** VMPOOLER ONDEMAND Pools ***'
os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
end
end
res = conn.get 'status/platforms/nspooler'
res_body = JSON.parse(res.body)
os_list << ''
os_list << '*** NSPOOLER Pools ***'
os_list += JSON.parse(res_body['nspooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('nspooler_platforms')
os_list << ''
os_list << '*** NSPOOLER Pools ***'
os_list += JSON.parse(res_body['nspooler_platforms'])
end
end
res = conn.get 'status/platforms/aws'
res_body = JSON.parse(res.body)
os_list << ''
os_list << '*** AWS Pools ***'
os_list += JSON.parse(res_body['aws_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('aws_platforms')
os_list << ''
os_list << '*** AWS Pools ***'
os_list += JSON.parse(res_body['aws_platforms'])
end
end
os_list.delete 'ok'
@ -197,7 +231,7 @@ class ABS
conn.headers['X-AUTH-TOKEN'] = token if token
saved_job_id = DateTime.now.strftime('%Q')
vmpooler_config = Utils.get_vmpooler_service_config
req_obj = {
:resources => os_types,
:job => {
@ -206,6 +240,7 @@ class ABS
:user => user,
},
},
:vm_token => vmpooler_config['token'] # request with this token, on behalf of this user
}
if options['priority']
@ -264,12 +299,17 @@ class ABS
def self.check_queue(conn, job_id, req_obj, verbose)
queue_info_res = conn.get "status/queue/info/#{job_id}"
queue_info = JSON.parse(queue_info_res.body)
if valid_json?(queue_info_res.body)
queue_info = JSON.parse(queue_info_res.body)
else
FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
return [nil, nil]
end
res = conn.post 'request', req_obj.to_json
validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
unless res.body.empty?
unless res.body.empty? || !valid_json?(res.body)
res_body = JSON.parse(res.body)
return queue_info['queue_place'], res_body
end
@ -277,7 +317,7 @@ class ABS
end
def self.snapshot(_verbose, _url, _hostname, _token)
FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
end
def self.status(verbose, url)
@ -289,20 +329,24 @@ class ABS
end
def self.summary(verbose, url)
conn = Http.get_conn(verbose, url)
res = conn.get 'summary'
JSON.parse(res.body)
raise NoMethodError, 'summary is not defined for ABS'
end
def self.query(verbose, url, hostname)
return @active_hostnames if @active_hostnames
def self.query(verbose, url, job_id)
# return saved hostnames from the last time list_active was run
# preventing having to query the API again.
# This works as long as query is called after list_active
return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
# If using the cli query job_id
conn = Http.get_conn(verbose, url)
res = conn.get "host/#{hostname}"
JSON.parse(res.body)
queue_info_res = conn.get "status/queue/info/#{job_id}"
if valid_json?(queue_info_res.body)
queue_info = JSON.parse(queue_info_res.body)
else
FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
end
queue_info
end
def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@ -333,4 +377,11 @@ class ABS
raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
end
end
def self.valid_json?(json)
JSON.parse(json)
return true
rescue TypeError, JSON::ParserError => e
return false
end
end

20
lib/vmfloaty/service.rb
View file

@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
class Service
attr_reader :config
attr_accessor :silent
def initialize(options, config_hash = {})
options ||= Commander::Command::Options.new
@config = Utils.get_service_config config_hash, options
@service_object = Utils.get_service_object @config['type']
@silent = false
end
def method_missing(method_name, *args, &block)
@ -103,6 +105,7 @@ class Service
end
def modify(verbose, hostname, modify_hash)
maybe_use_vmpooler
@service_object.modify verbose, url, hostname, token, modify_hash
end
@ -115,18 +118,35 @@ class Service
end
def summary(verbose)
maybe_use_vmpooler
@service_object.summary verbose, url
end
def snapshot(verbose, hostname)
maybe_use_vmpooler
@service_object.snapshot verbose, url, hostname, token
end
def revert(verbose, hostname, snapshot_sha)
maybe_use_vmpooler
@service_object.revert verbose, url, hostname, token, snapshot_sha
end
def disk(verbose, hostname, disk)
maybe_use_vmpooler
@service_object.disk(verbose, url, hostname, token, disk)
end
# some methods do not exist for ABS, and if possible should target the Pooler service
def maybe_use_vmpooler
if @service_object.is_a?(ABS.class)
if !self.silent
FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using vmpooler config from ~/.vmfloaty.yml"
self.silent = true
end
@config = Utils.get_vmpooler_service_config
@service_object = Pooler
end
end
end

38
lib/vmfloaty/utils.rb
View file

@ -3,6 +3,7 @@
require 'vmfloaty/abs'
require 'vmfloaty/nonstandard_pooler'
require 'vmfloaty/pooler'
require 'vmfloaty/conf'
class Utils
# TODO: Takes the json response body from an HTTP GET
@ -78,21 +79,28 @@ class Utils
os_types
end
def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false)
def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0)
fetched_data = self.get_host_data(verbose, service, hostnames)
fetched_data.each do |hostname, host_data|
case service.type
when 'ABS'
# For ABS, 'hostname' variable is the jobID
# For ABS, 'hostname' variable is the jobID
#
# Create a vmpooler service to query each hostname there so as to get the metadata too
vmpooler_service = service.clone
vmpooler_service.silent = true
vmpooler_service.maybe_use_vmpooler
puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>"
host_data['allocated_resources'].each do |vm_name, _i|
puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>"
self.pretty_print_hosts(verbose, vmpooler_service, vm_name['hostname'].split('.')[0], print_to_stderr, indent+2)
end
when 'Pooler'
tag_pairs = []
tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
metadata = [host_data['template'], duration, *tag_pairs]
puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})"
puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent)
when 'NonstandardPooler'
line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
line += ", #{host_data['hours_left_on_reservation']}h remaining"
@ -241,4 +249,26 @@ class Utils
service_config
end
# This method gets the vmpooler service configured in ~/.vmfloaty
def self.get_vmpooler_service_config
config = Conf.read_config
# The top-level url, user, and token values in the config file are treated as defaults
service_config = {
'url' => config['url'],
'user' => config['user'],
'token' => config['token'],
'type' => 'vmpooler',
}
# at a minimum, the url needs to be configured
if config['services'] && config['services']['vmpooler'] && config['services']['vmpooler']['url']
# If the service is configured but some values are missing, use the top-level defaults to fill them in
service_config.merge! config['services']['vmpooler']
else
raise ArgumentError, "Could not find a configured service named 'vmpooler' in ~/.vmfloaty.yml use this format:\nservices:\n vmpooler:\n url: 'http://vmpooler.com'\n user: 'superman'\n token: 'kryptonite'"
end
service_config
end
end