From d963e357d37244d167d0f56a931b7c4caaa91ae8 Mon Sep 17 00:00:00 2001 From: Mikker Gimenez-Peterson Date: Mon, 4 Nov 2019 13:30:58 -0800 Subject: [PATCH] Adding delete and get active requests --- lib/vmfloaty.rb | 1 + lib/vmfloaty/abs.rb | 133 ++++++++++++++--------- lib/vmfloaty/nonstandard_pooler.rb | 2 +- lib/vmfloaty/pooler.rb | 2 +- lib/vmfloaty/service.rb | 2 +- lib/vmfloaty/utils.rb | 8 +- spec/vmfloaty/abs_spec.rb | 16 +-- spec/vmfloaty/nonstandard_pooler_spec.rb | 14 +-- spec/vmfloaty/pooler_spec.rb | 6 +- 9 files changed, 109 insertions(+), 75 deletions(-) diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index caf6a7a..285958c 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -33,6 +33,7 @@ class Vmfloaty c.option '--user STRING', String, 'User to authenticate with' c.option '--url STRING', String, 'URL of pooler service' c.option '--token STRING', String, 'Token for pooler service' + c.option '--priority STRING', 'Priority for supported backends(ABS) (High(1), Medium(2), Low(3))' c.option '--notoken', 'Makes a request without a token' c.option '--force', 'Forces vmfloaty to get requested vms' c.option '--json', 'Prints retrieved vms in JSON format' diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index 381d95b..29468c6 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -37,38 +37,39 @@ class ABS # } # - @@active_hostnames = Hash.new + @active_hostnames = {} def self.list_active(verbose, url, _token, user) - all_jobs = Array.new() - @@active_hostnames = Hash.new + all_jobs = [] + @active_hostnames = {} - self.get_active_requests(verbose, url, user).each do |reqHash| - all_jobs.push(reqHash['request']['job']['id']) - @@active_hostnames[reqHash['request']['job']['id']] = reqHash + get_active_requests(verbose, url, user).each do |req_hash| + all_jobs.push(req_hash['request']['job']['id']) + @active_hostnames[req_hash['request']['job']['id']] = req_hash end all_jobs end - def self.get_active_requests verbose, url, user + def self.get_active_requests(verbose, url, user) conn = Http.get_conn(verbose, url) res = conn.get 'status/queue' requests = JSON.parse(res.body) - retVal = [] + ret_val = [] requests.each do |req| - reqHash = JSON.parse(req) - next unless user == reqHash['request']['job']['user'] - retVal.push(reqHash) + req_hash = JSON.parse(req) + next unless user == req_hash['request']['job']['user'] + + ret_val.push(req_hash) end - retVal + ret_val end def self.all_job_resources_accounted_for(allocated_resources, hosts) - allocated_host_list = allocated_resources.map {|ar| ar["hostname"] } - return (allocated_host_list-hosts).empty? + allocated_host_list = allocated_resources.map { |ar| ar['hostname'] } + (allocated_host_list - hosts).empty? end def self.delete(verbose, url, hosts, token, user) @@ -77,19 +78,29 @@ class ABS conn.headers['X-AUTH-TOKEN'] = token if token puts "Trying to delete hosts #{hosts}" if verbose - requests = self.get_active_requests(verbose, url, user) + requests = get_active_requests(verbose, url, user) jobs_to_delete = [] - requests.each do |reqHash| - if reqHash['state'] == 'allocated' || reqHash['state'] == 'filled' - reqHash['allocated_resources'].each do |vm_name, i| - if hosts.include? vm_name["hostname"] - if (all_job_resources_accounted_for(job['allocated_resources'], hosts)) - jobs_to_delete.push(reqHash) - else - puts "Can't delete #{job_id}: #{hosts} does not include all of #{job['allocated_resources']}" - end + retStatus = {} + hosts.each do |host| + retStatus[host] = { + 'ok' => false + } + end + + requests.each do |req_hash| + next unless req_hash['state'] == 'allocated' || req_hash['state'] == 'filled' + + req_hash['allocated_resources'].each do |vm_name, _i| + if hosts.include? vm_name['hostname'] + if all_job_resources_accounted_for(req_hash['allocated_resources'], hosts) + retStatus[vm_name['hostname']] = { + 'ok' => true + } + jobs_to_delete.push(req_hash) + else + puts "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}" end end end @@ -98,21 +109,22 @@ class ABS response_body = {} jobs_to_delete.each do |job| - reqObj = { - 'job_id': job['request']['job']['id'], - 'hosts': job['allocated_resources'], + req_obj = { + 'job_id' => job['request']['job']['id'], + 'hosts' => job['allocated_resources'], } - puts "Deleting #{reqObj}" if verbose + puts "Deleting #{req_obj}" if verbose - res = conn.post 'api/v2/return', reqObj.to_json - response_body[job_id] = res_body + return_result = conn.post 'return', req_obj.to_json + req_obj['hosts'].each do |host| + response_body[host["hostname"]] = { 'ok' => true } if return_result.body == "OK" + end end - return response_body + response_body end - # List available VMs in ABS def self.list(verbose, url, os_filter = nil) conn = Http.get_conn(verbose, url) @@ -143,7 +155,7 @@ class ABS end # Retrieve an OS from ABS. - def self.retrieve(verbose, os_types, token, url, user) + def self.retrieve(verbose, os_types, token, url, user, options) # # Contents of post must be like: # @@ -155,8 +167,7 @@ class ABS # "job": { # "id": "12345", # "tags": { - # "user": "jenkins", - # "jenkins_build_url": "https://jenkins/job/platform_puppet_intn-van-sys_master" + # "user": "username", # } # } # } @@ -164,35 +175,51 @@ class ABS conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token - saved_job_id = Time.now.to_i + saved_job_id = DateTime.now.strftime('%Q') - reqObj = { + req_obj = { :resources => os_types, :job => { :id => saved_job_id, :tags => { - :user => user, - :url_string => "floaty://#{user}/#{saved_job_id}", + :user => user, }, }, } + if options["priority"] + if options["priority"] == "high" + req_obj[:priority] = 1 + elsif options["priority"] == "medium" + req_obj[:priority] = 2 + elsif options["priority"] == "low" + req_obj[:priority] = 3 + else + req_obj[:priority] = options["priority"].to_i + end + end + + puts "Posting to ABS #{req_obj.to_json}" if verbose + # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? puts "Requesting VMs with job_id: #{saved_job_id}. Will retry for up to an hour." - res = conn.post 'api/v2/request', reqObj.to_json + res = conn.post 'request', req_obj.to_json - i = 0 retries = 360 raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401 (1..retries).each do |i| - queue_place, res_body = check_queue(conn, saved_job_id, reqObj) + queue_place, res_body = check_queue(conn, saved_job_id, req_obj) return translated(res_body) if res_body - puts "Waiting 10 seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" - sleep(10) + + sleepSeconds = 10 if i >= 10 + sleepSeconds = i if i < 10 + puts "Waiting #{sleepSeconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" + + sleep(sleepSeconds) end nil end @@ -215,11 +242,11 @@ class ABS vmpooler_formatted_body end - def self.check_queue(conn, job_id, reqObj) - queue_info_res = conn.get "/status/queue/info/#{job_id}" + def self.check_queue(conn, job_id, req_obj) + queue_info_res = conn.get "status/queue/info/#{job_id}" queue_info = JSON.parse(queue_info_res.body) - res = conn.post 'api/v2/request', reqObj.to_json + res = conn.post 'request', req_obj.to_json unless res.body.empty? res_body = JSON.parse(res.body) @@ -228,26 +255,28 @@ class ABS [queue_info['queue_place'], nil] end - def self.snapshot(verbose, url, hostname, token) + def self.snapshot(_verbose, _url, _hostname, _token) puts "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)" end def self.status(verbose, url) conn = Http.get_conn(verbose, url) - res = conn.get '/status' - JSON.parse(res.body) + res = conn.get 'status' + + return res.body == "OK" end def self.summary(verbose, url) conn = Http.get_conn(verbose, url) - res = conn.get '/summary' + res = conn.get 'summary' JSON.parse(res.body) end def self.query(verbose, url, hostname) - return @@active_hostnames if @@active_hostnames + return @active_hostnames if @active_hostnames + puts "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)" conn = Http.get_conn(verbose, url) diff --git a/lib/vmfloaty/nonstandard_pooler.rb b/lib/vmfloaty/nonstandard_pooler.rb index d094288..89f6c64 100644 --- a/lib/vmfloaty/nonstandard_pooler.rb +++ b/lib/vmfloaty/nonstandard_pooler.rb @@ -22,7 +22,7 @@ class NonstandardPooler status['reserved_hosts'] || [] end - def self.retrieve(verbose, os_type, token, url, _user) + def self.retrieve(verbose, os_type, token, url, _user, _options) conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token diff --git a/lib/vmfloaty/pooler.rb b/lib/vmfloaty/pooler.rb index 354b8a9..a735c3b 100644 --- a/lib/vmfloaty/pooler.rb +++ b/lib/vmfloaty/pooler.rb @@ -28,7 +28,7 @@ class Pooler vms end - def self.retrieve(verbose, os_type, token, url, user) + def self.retrieve(verbose, os_type, token, url, _user, _options) # NOTE: # Developers can use `Utils.generate_os_hash` to # generate the os_type param. diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 951db9c..28fc90c 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -78,7 +78,7 @@ class Service def retrieve(verbose, os_types, use_token = true) puts 'Requesting a vm without a token...' unless use_token token_value = use_token ? token : nil - @service_object.retrieve verbose, os_types, token_value, url, user + @service_object.retrieve verbose, os_types, token_value, url, user, @config end def ssh(verbose, host_os, use_token = true) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index eb3464a..f70eb20 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -88,8 +88,8 @@ class Utils when 'ABS' # For ABS, 'hostname' variable is the jobID if host_data['state'] == 'allocated' || host_data['state'] == 'filled' - host_data['allocated_resources'].each do |vm_name, i| - puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name["hostname"]} (#{vm_name["type"]}) <#{host_data['state']}>" + host_data['allocated_resources'].each do |vm_name, _i| + puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>" end end when 'Pooler' @@ -155,6 +155,9 @@ class Utils puts "#{name.ljust(width)} #{e.red}" end end + when 'ABS' + puts "ABS Not OK".red unless status_response + puts "ABS is OK".green if status_response else raise "Invalid service type #{service.type}" end @@ -205,6 +208,7 @@ class Utils end # Prioritize an explicitly specified url, user, or token if the user provided one + service_config['priority'] = options.priority unless options.priority.nil? service_config['url'] = options.url unless options.url.nil? service_config['token'] = options.token unless options.token.nil? service_config['user'] = options.user unless options.user.nil? diff --git a/spec/vmfloaty/abs_spec.rb b/spec/vmfloaty/abs_spec.rb index c59e14c..5a94b72 100644 --- a/spec/vmfloaty/abs_spec.rb +++ b/spec/vmfloaty/abs_spec.rb @@ -36,25 +36,25 @@ describe ABS do end it 'won\'t delete a job if not all vms are listed' do - hosts = ["host1"] + hosts = ['host1'] allocated_resources = [ { - "hostname" => "host1" + 'hostname' => 'host1', }, { - "hostname" => "host2" - } + 'hostname' => 'host2', + }, ] expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(false) - hosts = ["host1", "host2"] + hosts = ['host1', 'host2'] allocated_resources = [ { - "hostname" => "host1" + 'hostname' => 'host1', }, { - "hostname" => "host2" - } + 'hostname' => 'host2', + }, ] expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(true) end diff --git a/spec/vmfloaty/nonstandard_pooler_spec.rb b/spec/vmfloaty/nonstandard_pooler_spec.rb index 6863078..911c96c 100644 --- a/spec/vmfloaty/nonstandard_pooler_spec.rb +++ b/spec/vmfloaty/nonstandard_pooler_spec.rb @@ -66,7 +66,7 @@ describe NonstandardPooler do @token_status_body_active = <<~BODY { "ok": true, - "user": 'first.last', + "user": "first.last", "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": ["sol10-9", "sol10-11"] @@ -75,7 +75,7 @@ describe NonstandardPooler do @token_status_body_empty = <<~BODY { "ok": true, - "user": 'first.last', + "user": "first.last", "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": [] @@ -88,7 +88,7 @@ describe NonstandardPooler do .with(false, @nspooler_url, 'token-value') .and_return(JSON.parse(@token_status_body_active)) - list = NonstandardPooler.list_active(false, @nspooler_url, 'token-value') + list = NonstandardPooler.list_active(false, @nspooler_url, 'token-value', 'user') expect(list).to eql ['sol10-9', 'sol10-11'] end end @@ -125,7 +125,7 @@ describe NonstandardPooler do .to_return(:status => 401, :body => '{"ok":false,"reason": "token: token-value does not exist"}', :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') }.to raise_error(AuthError) + expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) }.to raise_error(AuthError) end it 'retrieves a single vm with a token' do @@ -134,7 +134,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_single, :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-11-sparc']['hostname']).to eq 'sol11-4.delivery.puppetlabs.net' @@ -146,7 +146,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_many, :headers => {}) vm_hash = { 'aix-7.1-power' => 1, 'solaris-10-sparc' => 2 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-10-sparc']['hostname']).to be_an_instance_of Array @@ -246,7 +246,7 @@ describe NonstandardPooler do "sol10-11": { "fqdn": "sol10-11.delivery.puppetlabs.net", "os_triple": "solaris-10-sparc", - "reserved_by_user": 'first.last', + "reserved_by_user": "first.last", "reserved_for_reason": "testing", "hours_left_on_reservation": 29.12 } diff --git a/spec/vmfloaty/pooler_spec.rb b/spec/vmfloaty/pooler_spec.rb index 62035e4..dffaf2f 100644 --- a/spec/vmfloaty/pooler_spec.rb +++ b/spec/vmfloaty/pooler_spec.rb @@ -53,7 +53,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 1 - expect { Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) }.to raise_error(AuthError) + expect { Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) }.to raise_error(AuthError) end it 'retrieves a single vm with a token' do @@ -63,7 +63,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 1 - vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) + vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['debian-7-i386']['hostname']).to eq 'fq6qlpjlsskycq6' @@ -77,7 +77,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 2 vm_hash['centos-7-x86_64'] = 1 - vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) + vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['debian-7-i386']['hostname']).to be_an_instance_of Array