diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index b2fb65a..129cd16 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -99,7 +99,12 @@ class Vmfloaty if options.active # list active vms - running_vms = service.list_active(verbose) + if service.type == "ABS" + # this is actually job_ids + running_vms = service.list_active_job_ids(verbose, service.url, service.user) + else + running_vms = service.list_active(verbose) + end host = URI.parse(service.url).host if running_vms.empty? if options.json @@ -126,7 +131,7 @@ class Vmfloaty command :query do |c| c.syntax = 'floaty query hostname [options]' c.summary = 'Get information about a given vm' - c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm.' + c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm. If using ABS, you can query a job_id' c.example 'Get information about a sample host', 'floaty query hostname --url http://vmpooler.example.com' c.option '--verbose', 'Enables verbose output' c.option '--service STRING', String, 'Configured pooler service name' @@ -165,7 +170,12 @@ class Vmfloaty FloatyLogger.error 'ERROR: Provide a hostname or specify --all.' exit 1 end - running_vms = modify_all ? service.list_active(verbose) : hostname.split(',') + running_vms = + if modify_all + service.list_active(verbose) + else + hostname.split(',') + end tags = options.tags ? JSON.parse(options.tags) : nil modify_hash = { @@ -189,7 +199,7 @@ class Vmfloaty end if ok if modify_all - puts 'Successfully modified all VMs.' + puts "Successfully modified all #{running_vms.count} VMs." else puts "Successfully modified VM #{hostname}." end @@ -225,7 +235,12 @@ class Vmfloaty successes = [] if delete_all - running_vms = service.list_active(verbose) + if service.type == "ABS" + # this is actually job_ids + running_vms = service.list_active_job_ids(verbose, service.url, service.user) + else + running_vms = service.list_active(verbose) + end if running_vms.empty? if options.json puts {}.to_json @@ -274,7 +289,6 @@ class Vmfloaty end unless successes.empty? - FloatyLogger.info unless failures.empty? if options.json puts successes.to_json else diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index 8c37072..43734cd 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -2,6 +2,7 @@ require 'vmfloaty/errors' require 'vmfloaty/http' +require 'vmfloaty/utils' require 'faraday' require 'json' @@ -36,39 +37,59 @@ class ABS # } # } # - @active_hostnames = {} - def self.list_active(verbose, url, _token, user) - all_jobs = [] + def self.list_active_job_ids(verbose, url, user) + all_job_ids = [] @active_hostnames = {} - get_active_requests(verbose, url, user).each do |req_hash| - all_jobs.push(req_hash['request']['job']['id']) - @active_hostnames[req_hash['request']['job']['id']] = req_hash + @active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval + all_job_ids.push(req_hash['request']['job']['id']) end - all_jobs + all_job_ids + end + + def self.list_active(verbose, url, _token, user) + hosts = [] + get_active_requests(verbose, url, user).each do |req_hash| + if req_hash.key?('allocated_resources') + req_hash['allocated_resources'].each do |onehost| + hosts.push(onehost['hostname']) + end + end + end + + hosts end def self.get_active_requests(verbose, url, user) conn = Http.get_conn(verbose, url) res = conn.get 'status/queue' - requests = JSON.parse(res.body) + if valid_json?(res.body) + requests = JSON.parse(res.body) + else + FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue" + end ret_val = [] requests.each do |req| next if req == 'null' - req_hash = JSON.parse(req) + if valid_json?(req) + req_hash = JSON.parse(req) + else + FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue" + next + end begin next unless user == req_hash['request']['job']['user'] ret_val.push(req_hash) rescue NoMethodError - FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: " + FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: " end end @@ -145,30 +166,43 @@ class ABS os_list = [] res = conn.get 'status/platforms/vmpooler' - - res_body = JSON.parse(res.body) - os_list << '*** VMPOOLER Pools ***' - os_list += JSON.parse(res_body['vmpooler_platforms']) + if valid_json?(res.body) + res_body = JSON.parse(res.body) + if res_body.key?('vmpooler_platforms') + os_list << '*** VMPOOLER Pools ***' + os_list += JSON.parse(res_body['vmpooler_platforms']) + end + end res = conn.get 'status/platforms/ondemand_vmpooler' - res_body = JSON.parse(res.body) - unless res_body['ondemand_vmpooler_platforms'] == '[]' - os_list << '' - os_list << '*** VMPOOLER ONDEMAND Pools ***' - os_list += JSON.parse(res_body['ondemand_vmpooler_platforms']) + if valid_json?(res.body) + res_body = JSON.parse(res.body) + if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]' + os_list << '' + os_list << '*** VMPOOLER ONDEMAND Pools ***' + os_list += JSON.parse(res_body['ondemand_vmpooler_platforms']) + end end res = conn.get 'status/platforms/nspooler' - res_body = JSON.parse(res.body) - os_list << '' - os_list << '*** NSPOOLER Pools ***' - os_list += JSON.parse(res_body['nspooler_platforms']) + if valid_json?(res.body) + res_body = JSON.parse(res.body) + if res_body.key?('nspooler_platforms') + os_list << '' + os_list << '*** NSPOOLER Pools ***' + os_list += JSON.parse(res_body['nspooler_platforms']) + end + end res = conn.get 'status/platforms/aws' - res_body = JSON.parse(res.body) - os_list << '' - os_list << '*** AWS Pools ***' - os_list += JSON.parse(res_body['aws_platforms']) + if valid_json?(res.body) + res_body = JSON.parse(res.body) + if res_body.key?('aws_platforms') + os_list << '' + os_list << '*** AWS Pools ***' + os_list += JSON.parse(res_body['aws_platforms']) + end + end os_list.delete 'ok' @@ -197,7 +231,7 @@ class ABS conn.headers['X-AUTH-TOKEN'] = token if token saved_job_id = DateTime.now.strftime('%Q') - + vmpooler_config = Utils.get_vmpooler_service_config req_obj = { :resources => os_types, :job => { @@ -206,6 +240,7 @@ class ABS :user => user, }, }, + :vm_token => vmpooler_config['token'] # request with this token, on behalf of this user } if options['priority'] @@ -264,12 +299,17 @@ class ABS def self.check_queue(conn, job_id, req_obj, verbose) queue_info_res = conn.get "status/queue/info/#{job_id}" - queue_info = JSON.parse(queue_info_res.body) + if valid_json?(queue_info_res.body) + queue_info = JSON.parse(queue_info_res.body) + else + FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}" + return [nil, nil] + end res = conn.post 'request', req_obj.to_json validate_queue_status_response(res.status, res.body, "Check queue request", verbose) - unless res.body.empty? + unless res.body.empty? || !valid_json?(res.body) res_body = JSON.parse(res.body) return queue_info['queue_place'], res_body end @@ -277,7 +317,7 @@ class ABS end def self.snapshot(_verbose, _url, _hostname, _token) - FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)" + raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)" end def self.status(verbose, url) @@ -289,20 +329,24 @@ class ABS end def self.summary(verbose, url) - conn = Http.get_conn(verbose, url) - - res = conn.get 'summary' - JSON.parse(res.body) + raise NoMethodError, 'summary is not defined for ABS' end - def self.query(verbose, url, hostname) - return @active_hostnames if @active_hostnames + def self.query(verbose, url, job_id) + # return saved hostnames from the last time list_active was run + # preventing having to query the API again. + # This works as long as query is called after list_active + return @active_hostnames if @active_hostnames && !@active_hostnames.empty? - FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)" + # If using the cli query job_id conn = Http.get_conn(verbose, url) - - res = conn.get "host/#{hostname}" - JSON.parse(res.body) + queue_info_res = conn.get "status/queue/info/#{job_id}" + if valid_json?(queue_info_res.body) + queue_info = JSON.parse(queue_info_res.body) + else + FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}" + end + queue_info end def self.modify(_verbose, _url, _hostname, _token, _modify_hash) @@ -333,4 +377,11 @@ class ABS raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}" end end + + def self.valid_json?(json) + JSON.parse(json) + return true + rescue TypeError, JSON::ParserError => e + return false + end end diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 11ddc29..d512c4d 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -7,11 +7,13 @@ require 'vmfloaty/ssh' class Service attr_reader :config + attr_accessor :silent def initialize(options, config_hash = {}) options ||= Commander::Command::Options.new @config = Utils.get_service_config config_hash, options @service_object = Utils.get_service_object @config['type'] + @silent = false end def method_missing(method_name, *args, &block) @@ -103,6 +105,7 @@ class Service end def modify(verbose, hostname, modify_hash) + maybe_use_vmpooler @service_object.modify verbose, url, hostname, token, modify_hash end @@ -115,18 +118,35 @@ class Service end def summary(verbose) + maybe_use_vmpooler @service_object.summary verbose, url end def snapshot(verbose, hostname) + maybe_use_vmpooler @service_object.snapshot verbose, url, hostname, token end def revert(verbose, hostname, snapshot_sha) + maybe_use_vmpooler @service_object.revert verbose, url, hostname, token, snapshot_sha end def disk(verbose, hostname, disk) + maybe_use_vmpooler @service_object.disk(verbose, url, hostname, token, disk) end + + # some methods do not exist for ABS, and if possible should target the Pooler service + def maybe_use_vmpooler + if @service_object.is_a?(ABS.class) + if !self.silent + FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using vmpooler config from ~/.vmfloaty.yml" + self.silent = true + end + + @config = Utils.get_vmpooler_service_config + @service_object = Pooler + end + end end diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index 2a4cbce..ed01ac6 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -3,6 +3,7 @@ require 'vmfloaty/abs' require 'vmfloaty/nonstandard_pooler' require 'vmfloaty/pooler' +require 'vmfloaty/conf' class Utils # TODO: Takes the json response body from an HTTP GET @@ -78,21 +79,28 @@ class Utils os_types end - def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false) + def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0) fetched_data = self.get_host_data(verbose, service, hostnames) fetched_data.each do |hostname, host_data| case service.type when 'ABS' - # For ABS, 'hostname' variable is the jobID + # For ABS, 'hostname' variable is the jobID + # + # Create a vmpooler service to query each hostname there so as to get the metadata too + + vmpooler_service = service.clone + vmpooler_service.silent = true + vmpooler_service.maybe_use_vmpooler + puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>" host_data['allocated_resources'].each do |vm_name, _i| - puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>" + self.pretty_print_hosts(verbose, vmpooler_service, vm_name['hostname'].split('.')[0], print_to_stderr, indent+2) end when 'Pooler' tag_pairs = [] tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil? duration = "#{host_data['running']}/#{host_data['lifetime']} hours" metadata = [host_data['template'], duration, *tag_pairs] - puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})" + puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent) when 'NonstandardPooler' line = "- #{host_data['fqdn']} (#{host_data['os_triple']}" line += ", #{host_data['hours_left_on_reservation']}h remaining" @@ -241,4 +249,26 @@ class Utils service_config end + + # This method gets the vmpooler service configured in ~/.vmfloaty + def self.get_vmpooler_service_config + config = Conf.read_config + # The top-level url, user, and token values in the config file are treated as defaults + service_config = { + 'url' => config['url'], + 'user' => config['user'], + 'token' => config['token'], + 'type' => 'vmpooler', + } + + # at a minimum, the url needs to be configured + if config['services'] && config['services']['vmpooler'] && config['services']['vmpooler']['url'] + # If the service is configured but some values are missing, use the top-level defaults to fill them in + service_config.merge! config['services']['vmpooler'] + else + raise ArgumentError, "Could not find a configured service named 'vmpooler' in ~/.vmfloaty.yml use this format:\nservices:\n vmpooler:\n url: 'http://vmpooler.com'\n user: 'superman'\n token: 'kryptonite'" + end + + service_config + end end