From de7d0fdeaba12c5375c38ed7bc9edbb3536869e4 Mon Sep 17 00:00:00 2001 From: Mikker Gimenez-Peterson Date: Thu, 17 Oct 2019 16:09:08 -0700 Subject: [PATCH 1/6] Adding command to list pools out of ABS --- README.md | 4 + lib/vmfloaty/abs.rb | 139 +++++++++++++++++++++++++++++++++ lib/vmfloaty/service.rb | 2 +- lib/vmfloaty/utils.rb | 6 +- spec/vmfloaty/abs/auth_spec.rb | 88 +++++++++++++++++++++ 5 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 lib/vmfloaty/abs.rb create mode 100644 spec/vmfloaty/abs/auth_spec.rb diff --git a/README.md b/README.md index 5526e42..ca7b8f8 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,10 @@ services: url: 'https://nspooler.example.net/api/v1' token: 'nspooler-tokenstring' type: 'nonstandard' # <-- 'type' is necessary for any non-vmpooler service + abs: + url: 'https://abs.example.net/api/v2' + token: 'abs-tokenstring' + type: 'abs' # <-- 'type' is necessary for any non-vmpooler service ``` With this configuration, you could list available OS types from nspooler like this: diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb new file mode 100644 index 0000000..d35a776 --- /dev/null +++ b/lib/vmfloaty/abs.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require 'vmfloaty/errors' +require 'vmfloaty/http' +require 'faraday' +require 'json' + +class ABS + # List available VMs in ABS + def self.list(verbose, url, os_filter = nil) + conn = Http.get_conn(verbose, url) + + os_list = [] + + response = conn.get 'status/platforms/vmpooler' + response_body = JSON.parse(response.body) + os_list << "*** VMPOOLER Pools ***" + os_list = os_list + JSON.parse(response_body["vmpooler_platforms"]) + + response = conn.get 'status/platforms/nspooler' + response_body = JSON.parse(response.body) + os_list << "" + os_list << "*** NSPOOLER Pools ***" + os_list = os_list + JSON.parse(response_body["nspooler_platforms"]) + + response = conn.get 'status/platforms/aws' + response_body = JSON.parse(response.body) + os_list << "" + os_list << "*** AWS Pools ***" + os_list = os_list + JSON.parse(response_body["aws_platforms"]) + + os_list.delete 'ok' + + puts os_list + + os_filter ? os_list.select { |i| i[/#{os_filter}/] } : os_list + end + + # List active VMs from ABS + def self.list_active(verbose, url, token) + status = Auth.token_status(verbose, url, token) + status['reserved_hosts'] || [] + end + + def self.retrieve(verbose, os_type, token, url) + conn = Http.get_conn(verbose, url) + conn.headers['X-AUTH-TOKEN'] = token if token + + os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') + raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? + + response = conn.post "host/#{os_string}" + + res_body = JSON.parse(response.body) + + if res_body['ok'] + res_body + elsif response.status == 401 + raise AuthError, "HTTP #{response.status}: The token provided could not authenticate to the pooler.\n#{res_body}" + else + raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/host/#{os_string}. #{res_body}" + end + end + + def self.modify(verbose, url, hostname, token, modify_hash) + raise TokenError, 'Token provided was nil; Request cannot be made to modify VM' if token.nil? + + modify_hash.each do |key, _value| + raise ModifyError, "Configured service type does not support modification of #{key}" unless %i[reason reserved_for_reason].include? key + end + + if modify_hash[:reason] + # "reason" is easier to type than "reserved_for_reason", but nspooler needs the latter + modify_hash[:reserved_for_reason] = modify_hash.delete :reason + end + + conn = Http.get_conn(verbose, url) + conn.headers['X-AUTH-TOKEN'] = token + + response = conn.put do |req| + req.url "host/#{hostname}" + req.body = modify_hash.to_json + end + + response.body.empty? ? {} : JSON.parse(response.body) + end + + def self.disk(_verbose, _url, _hostname, _token, _disk) + raise ModifyError, 'Configured service type does not support modification of disk space' + end + + def self.snapshot(_verbose, _url, _hostname, _token) + raise ModifyError, 'Configured service type does not support snapshots' + end + + def self.revert(_verbose, _url, _hostname, _token, _snapshot_sha) + raise ModifyError, 'Configured service type does not support snapshots' + end + + def self.delete(verbose, url, hosts, token) + raise TokenError, 'Token provided was nil; Request cannot be made to delete VM' if token.nil? + + conn = Http.get_conn(verbose, url) + + conn.headers['X-AUTH-TOKEN'] = token if token + + response_body = {} + + hosts = hosts.split(',') unless hosts.is_a? Array + hosts.each do |host| + response = conn.delete "host/#{host}" + res_body = JSON.parse(response.body) + response_body[host] = res_body + end + + response_body + end + + def self.status(verbose, url) + conn = Http.get_conn(verbose, url) + + response = conn.get '/status' + JSON.parse(response.body) + end + + def self.summary(verbose, url) + conn = Http.get_conn(verbose, url) + + response = conn.get '/summary' + JSON.parse(response.body) + end + + def self.query(verbose, url, hostname) + conn = Http.get_conn(verbose, url) + + response = conn.get "host/#{hostname}" + JSON.parse(response.body) + end +end diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 648e71b..732cf59 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -52,7 +52,7 @@ class Service def get_new_token(verbose) username = user - pass = Commander::UI.password 'Enter your pooler service password:', '*' + pass = Commander::UI.password "Enter your #{@config["url"]} service password:", '*' Auth.get_token(verbose, url, username, pass) end diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index b9f5a9c..22cc0c3 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true -require 'vmfloaty/pooler' +require 'vmfloaty/abs' require 'vmfloaty/nonstandard_pooler' +require 'vmfloaty/pooler' class Utils # TODO: Takes the json response body from an HTTP GET @@ -155,8 +156,11 @@ class Utils def self.get_service_object(type = '') nspooler_strings = %w[ns nspooler nonstandard nonstandard_pooler] + abs_strings = %w[abs alwaysbescheduling always_be_scheduling] if nspooler_strings.include? type.downcase NonstandardPooler + elsif abs_strings.include? type.downcase + ABS else Pooler end diff --git a/spec/vmfloaty/abs/auth_spec.rb b/spec/vmfloaty/abs/auth_spec.rb new file mode 100644 index 0000000..d6b0d26 --- /dev/null +++ b/spec/vmfloaty/abs/auth_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../lib/vmfloaty/auth' + +describe Pooler do + before :each do + @abs_url = 'https://abs.example.com' + end + + describe '#get_token' do + before :each do + @get_token_response = '{"ok": true,"token":"utpg2i2xswor6h8ttjhu3d47z53yy47y"}' + @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y' + end + + it 'returns a token from vmpooler' do + stub_request(:post, 'https://first.last:password@abs.example.com/api/v2/token') + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Content-Length' => '0', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 200, :body => @get_token_response, :headers => {}) + + token = Auth.get_token(false, @abs_url, 'first.last', 'password') + expect(token).to eq @token + end + + it 'raises a token error if something goes wrong' do + stub_request(:post, 'https://first.last:password@abs.example.com/api/v2/token') + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Content-Length' => '0', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) + + expect { Auth.get_token(false, @abs_url, 'first.last', 'password') }.to raise_error(TokenError) + end + end + + describe '#delete_token' do + before :each do + @delete_token_response = '{"ok":true}' + @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y' + end + + it 'deletes the specified token' do + stub_request(:delete, 'https://first.last:password@abs.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y') + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 200, :body => @delete_token_response, :headers => {}) + + expect(Auth.delete_token(false, @abs_url, 'first.last', 'password', @token)).to eq JSON.parse(@delete_token_response) + end + + it 'raises a token error if something goes wrong' do + stub_request(:delete, 'https://first.last:password@abs.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y') + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) + + expect { Auth.delete_token(false, @abs_url, 'first.last', 'password', @token) }.to raise_error(TokenError) + end + + it 'raises a token error if no token provided' do + expect { Auth.delete_token(false, @abs_url, 'first.last', 'password', nil) }.to raise_error(TokenError) + end + end + + describe '#token_status' do + before :each do + @token_status_response = '{"ok":true,"utpg2i2xswor6h8ttjhu3d47z53yy47y":{"created":"2015-04-28 19:17:47 -0700"}}' + @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y' + end + + it 'checks the status of a token' do + stub_request(:get, "#{@abs_url}/token/utpg2i2xswor6h8ttjhu3d47z53yy47y") + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 200, :body => @token_status_response, :headers => {}) + + expect(Auth.token_status(false, @abs_url, @token)).to eq JSON.parse(@token_status_response) + end + + it 'raises a token error if something goes wrong' do + stub_request(:get, "#{@abs_url}/token/utpg2i2xswor6h8ttjhu3d47z53yy47y") + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) + + expect { Auth.token_status(false, @abs_url, @token) }.to raise_error(TokenError) + end + + it 'raises a token error if no token provided' do + expect { Auth.token_status(false, @abs_url, nil) }.to raise_error(TokenError) + end + end +end From a77ea84092b36eb863c734b71a17ba48924e729c Mon Sep 17 00:00:00 2001 From: Mikker Gimenez-Peterson Date: Thu, 31 Oct 2019 11:41:51 -0700 Subject: [PATCH 2/6] Rebasing fixed tests --- README.md | 5 +- lib/vmfloaty.rb | 1 + lib/vmfloaty/abs.rb | 227 +++++++++++++++-------- lib/vmfloaty/nonstandard_pooler.rb | 4 +- lib/vmfloaty/pooler.rb | 2 +- lib/vmfloaty/service.rb | 10 +- lib/vmfloaty/utils.rb | 7 + spec/vmfloaty/abs/auth_spec.rb | 12 +- spec/vmfloaty/abs_spec.rb | 38 ++++ spec/vmfloaty/nonstandard_pooler_spec.rb | 12 +- spec/vmfloaty/service_spec.rb | 10 +- 11 files changed, 217 insertions(+), 111 deletions(-) create mode 100644 spec/vmfloaty/abs_spec.rb diff --git a/README.md b/README.md index ca7b8f8..9e278fc 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ floaty get centos-7-x86_64=2 debian-7-x86_64 windows-10=3 --token mytokenstring ### vmfloaty dotfile -If you do not wish to continuely specify various config options with the cli, you can have a dotfile in your home directory for some defaults. For example: +If you do not wish to continually specify various config options with the cli, you can have a dotfile in your home directory for some defaults. For example: #### Basic configuration @@ -132,9 +132,10 @@ services: token: 'nspooler-tokenstring' type: 'nonstandard' # <-- 'type' is necessary for any non-vmpooler service abs: - url: 'https://abs.example.net/api/v2' + url: 'https://abs.example.net/' token: 'abs-tokenstring' type: 'abs' # <-- 'type' is necessary for any non-vmpooler service + ``` With this configuration, you could list available OS types from nspooler like this: diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index dfe390a..caf6a7a 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -84,6 +84,7 @@ class Vmfloaty c.option '--url STRING', String, 'URL of pooler service' c.action do |args, options| verbose = options.verbose || config['verbose'] + service = Service.new(options, config) filter = args[0] diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index d35a776..a826ed6 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -6,134 +6,197 @@ require 'faraday' require 'json' class ABS + # List active VMs in ABS + # + # + # { + # "state":"filled", + # "last_processed":"2019-10-31 20:59:33 +0000", + # "allocated_resources": [ + # { + # "hostname":"h3oyntawjm7xdch.delivery.puppetlabs.net", + # "type":"centos-7.2-tmpfs-x86_64", + # "engine":"vmpooler"} + # ], + # "audit_log":{ + # "2019-10-30 20:33:12 +0000":"Allocated h3oyntawjm7xdch.delivery.puppetlabs.net for job 1572467589" + # }, + # "request":{ + # "resources":{ + # "centos-7.2-tmpfs-x86_64":1 + # }, + # "job": { + # "id":1572467589, + # "tags": { + # "user":"mikker", + # "url_string":"floaty://mikker/1572467589" + # }, + # "user":"mikker", + # "time-received":1572467589 + # } + # } + # } + # + def self.list_active(verbose, url, _token, user) + conn = Http.get_conn(verbose, url) + res = conn.get 'status/queue' + requests = JSON.parse(res.body) + + requests.each do |req| + reqHash = JSON.parse(req) + next unless user == reqHash['request']['job']['user'] + + puts '------------------------------------' + puts "State: #{reqHash['state']}" + puts "Job ID: #{reqHash['request']['job']['id']}" + reqHash['request']['resources'].each do |vm_template, i| + puts "--VMRequest: #{vm_template}: #{i}" + end + if reqHash['state'] == 'allocated' || reqHash['state'] == 'filled' + reqHash['allocated_resources'].each do |vm_name, i| + puts "----VM: #{vm_name}: #{i}" + end + end + puts "User: #{reqHash['request']['job']['user']}" + puts '' + end + + sleep(100) + end + # List available VMs in ABS def self.list(verbose, url, os_filter = nil) conn = Http.get_conn(verbose, url) os_list = [] - response = conn.get 'status/platforms/vmpooler' - response_body = JSON.parse(response.body) - os_list << "*** VMPOOLER Pools ***" - os_list = os_list + JSON.parse(response_body["vmpooler_platforms"]) + res = conn.get 'status/platforms/vmpooler' - response = conn.get 'status/platforms/nspooler' - response_body = JSON.parse(response.body) - os_list << "" - os_list << "*** NSPOOLER Pools ***" - os_list = os_list + JSON.parse(response_body["nspooler_platforms"]) + res_body = JSON.parse(res.body) + os_list << '*** VMPOOLER Pools ***' + os_list += JSON.parse(res_body['vmpooler_platforms']) - response = conn.get 'status/platforms/aws' - response_body = JSON.parse(response.body) - os_list << "" - os_list << "*** AWS Pools ***" - os_list = os_list + JSON.parse(response_body["aws_platforms"]) + res = conn.get 'status/platforms/nspooler' + res_body = JSON.parse(res.body) + os_list << '' + os_list << '*** NSPOOLER Pools ***' + os_list += JSON.parse(res_body['nspooler_platforms']) + + res = conn.get 'status/platforms/aws' + res_body = JSON.parse(res.body) + os_list << '' + os_list << '*** AWS Pools ***' + os_list += JSON.parse(res_body['aws_platforms']) os_list.delete 'ok' - puts os_list - os_filter ? os_list.select { |i| i[/#{os_filter}/] } : os_list end - # List active VMs from ABS - def self.list_active(verbose, url, token) - status = Auth.token_status(verbose, url, token) - status['reserved_hosts'] || [] - end + # Retrieve an OS from ABS. + def self.retrieve(verbose, os_types, token, url, user) + # + # Contents of post must be:j + # + # { + # "resources": { + # "centos-7-i386": 1, + # "ubuntu-1404-x86_64": 2 + # }, + # "job": { + # "id": "12345", + # "tags": { + # "user": "jenkins", + # "jenkins_build_url": "https://jenkins/job/platform_puppet_intn-van-sys_master" + # } + # } + # } - def self.retrieve(verbose, os_type, token, url) conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token - os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') - raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? + saved_job_id = Time.now.to_i - response = conn.post "host/#{os_string}" + reqObj = { + :resources => os_types, + :job => { + :id => saved_job_id, + :tags => { + :user => user, + :url_string => "floaty://#{user}/#{saved_job_id}", + }, + }, + } - res_body = JSON.parse(response.body) + # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') + # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? + puts "Requesting VMs with job_id: #{saved_job_id}. Will retry for up to an hour." + res = conn.post 'api/v2/request', reqObj.to_json - if res_body['ok'] - res_body - elsif response.status == 401 - raise AuthError, "HTTP #{response.status}: The token provided could not authenticate to the pooler.\n#{res_body}" - else - raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/host/#{os_string}. #{res_body}" + i = 0 + retries = 360 + + raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401 + + (1..retries).each do |i| + queue_place, res_body = check_queue(conn, saved_job_id, reqObj) + return translated(res_body) if res_body + + puts "Waiting 10 seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" + sleep(10) end + nil end - def self.modify(verbose, url, hostname, token, modify_hash) - raise TokenError, 'Token provided was nil; Request cannot be made to modify VM' if token.nil? + # + # We should fix the ABS API to be more like the vmpooler or nspooler api, but for now + # + def self.translated(res_body) + vmpooler_formatted_body = {} - modify_hash.each do |key, _value| - raise ModifyError, "Configured service type does not support modification of #{key}" unless %i[reason reserved_for_reason].include? key + res_body.each do |host| + if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].class == Array + vmpooler_formatted_body[host['type']]['hostname'] << host['hostname'] + else + vmpooler_formatted_body[host['type']] = { 'hostname' => [host['hostname']] } + end end + vmpooler_formatted_body['ok'] = true - if modify_hash[:reason] - # "reason" is easier to type than "reserved_for_reason", but nspooler needs the latter - modify_hash[:reserved_for_reason] = modify_hash.delete :reason - end - - conn = Http.get_conn(verbose, url) - conn.headers['X-AUTH-TOKEN'] = token - - response = conn.put do |req| - req.url "host/#{hostname}" - req.body = modify_hash.to_json - end - - response.body.empty? ? {} : JSON.parse(response.body) + vmpooler_formatted_body end - def self.disk(_verbose, _url, _hostname, _token, _disk) - raise ModifyError, 'Configured service type does not support modification of disk space' - end + def self.check_queue(conn, job_id, reqObj) + queue_info_res = conn.get "/status/queue/info/#{job_id}" + queue_info = JSON.parse(queue_info_res.body) - def self.snapshot(_verbose, _url, _hostname, _token) - raise ModifyError, 'Configured service type does not support snapshots' - end + res = conn.post 'api/v2/request', reqObj.to_json - def self.revert(_verbose, _url, _hostname, _token, _snapshot_sha) - raise ModifyError, 'Configured service type does not support snapshots' - end - - def self.delete(verbose, url, hosts, token) - raise TokenError, 'Token provided was nil; Request cannot be made to delete VM' if token.nil? - - conn = Http.get_conn(verbose, url) - - conn.headers['X-AUTH-TOKEN'] = token if token - - response_body = {} - - hosts = hosts.split(',') unless hosts.is_a? Array - hosts.each do |host| - response = conn.delete "host/#{host}" - res_body = JSON.parse(response.body) - response_body[host] = res_body + unless res.body.empty? + res_body = JSON.parse(res.body) + return queue_info['queue_place'], res_body end - - response_body + [queue_info['queue_place'], nil] end def self.status(verbose, url) conn = Http.get_conn(verbose, url) - response = conn.get '/status' - JSON.parse(response.body) + res = conn.get '/status' + JSON.parse(res.body) end def self.summary(verbose, url) conn = Http.get_conn(verbose, url) - response = conn.get '/summary' - JSON.parse(response.body) + res = conn.get '/summary' + JSON.parse(res.body) end def self.query(verbose, url, hostname) conn = Http.get_conn(verbose, url) - response = conn.get "host/#{hostname}" - JSON.parse(response.body) + res = conn.get "host/#{hostname}" + JSON.parse(res.body) end end diff --git a/lib/vmfloaty/nonstandard_pooler.rb b/lib/vmfloaty/nonstandard_pooler.rb index f5451b7..d094288 100644 --- a/lib/vmfloaty/nonstandard_pooler.rb +++ b/lib/vmfloaty/nonstandard_pooler.rb @@ -17,12 +17,12 @@ class NonstandardPooler os_filter ? os_list.select { |i| i[/#{os_filter}/] } : os_list end - def self.list_active(verbose, url, token) + def self.list_active(verbose, url, token, _user) status = Auth.token_status(verbose, url, token) status['reserved_hosts'] || [] end - def self.retrieve(verbose, os_type, token, url) + def self.retrieve(verbose, os_type, token, url, _user) conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token diff --git a/lib/vmfloaty/pooler.rb b/lib/vmfloaty/pooler.rb index f8d8633..69de7da 100644 --- a/lib/vmfloaty/pooler.rb +++ b/lib/vmfloaty/pooler.rb @@ -21,7 +21,7 @@ class Pooler hosts end - def self.list_active(verbose, url, token) + def self.list_active(verbose, url, token, _user) status = Auth.token_status(verbose, url, token) vms = [] vms = status[token]['vms']['running'] if status[token] && status[token]['vms'] diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 732cf59..6d606f1 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -36,7 +36,7 @@ class Service def user unless @config['user'] - puts 'Enter your pooler service username:' + puts "Enter your #{@config['url']} service username:" @config['user'] = STDIN.gets.chomp end @config['user'] @@ -52,13 +52,13 @@ class Service def get_new_token(verbose) username = user - pass = Commander::UI.password "Enter your #{@config["url"]} service password:", '*' + pass = Commander::UI.password "Enter your #{@config['url']} service password:", '*' Auth.get_token(verbose, url, username, pass) end def delete_token(verbose, token_value = @config['token']) username = user - pass = Commander::UI.password 'Enter your pooler service password:', '*' + pass = Commander::UI.password "Enter your #{@config['url']} service password:", '*' Auth.delete_token(verbose, url, username, pass, token_value) end @@ -72,13 +72,13 @@ class Service end def list_active(verbose) - @service_object.list_active verbose, url, token + @service_object.list_active verbose, url, token, user end def retrieve(verbose, os_types, use_token = true) puts 'Requesting a vm without a token...' unless use_token token_value = use_token ? token : nil - @service_object.retrieve verbose, os_types, token_value, url + @service_object.retrieve verbose, os_types, token_value, url, user end def ssh(verbose, host_os, use_token = true) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index 22cc0c3..bd83f7e 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -31,6 +31,13 @@ class Utils # } # } + # abs pooler response body example when `floaty get` arguments are : + # { + # "hostname"=>"thin-soutane.delivery.puppetlabs.net", + # "type"=>"centos-7.2-tmpfs-x86_64", + # "engine"=>"vmpooler" + # } + raise ArgumentError, "Bad GET response passed to format_hosts: #{response_body.to_json}" unless response_body.delete('ok') # vmpooler reports the domain separately from the hostname diff --git a/spec/vmfloaty/abs/auth_spec.rb b/spec/vmfloaty/abs/auth_spec.rb index d6b0d26..555d6c5 100644 --- a/spec/vmfloaty/abs/auth_spec.rb +++ b/spec/vmfloaty/abs/auth_spec.rb @@ -5,7 +5,7 @@ require_relative '../../../lib/vmfloaty/auth' describe Pooler do before :each do - @abs_url = 'https://abs.example.com' + @abs_url = 'https://abs.example.com/api/v2' end describe '#get_token' do @@ -14,9 +14,8 @@ describe Pooler do @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y' end - it 'returns a token from vmpooler' do + it 'returns a token from abs' do stub_request(:post, 'https://first.last:password@abs.example.com/api/v2/token') - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Content-Length' => '0', 'User-Agent' => 'Faraday v0.9.2' }) .to_return(:status => 200, :body => @get_token_response, :headers => {}) token = Auth.get_token(false, @abs_url, 'first.last', 'password') @@ -25,7 +24,6 @@ describe Pooler do it 'raises a token error if something goes wrong' do stub_request(:post, 'https://first.last:password@abs.example.com/api/v2/token') - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Content-Length' => '0', 'User-Agent' => 'Faraday v0.9.2' }) .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) expect { Auth.get_token(false, @abs_url, 'first.last', 'password') }.to raise_error(TokenError) @@ -40,7 +38,6 @@ describe Pooler do it 'deletes the specified token' do stub_request(:delete, 'https://first.last:password@abs.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y') - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) .to_return(:status => 200, :body => @delete_token_response, :headers => {}) expect(Auth.delete_token(false, @abs_url, 'first.last', 'password', @token)).to eq JSON.parse(@delete_token_response) @@ -48,7 +45,6 @@ describe Pooler do it 'raises a token error if something goes wrong' do stub_request(:delete, 'https://first.last:password@abs.example.com/api/v2/token/utpg2i2xswor6h8ttjhu3d47z53yy47y') - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) expect { Auth.delete_token(false, @abs_url, 'first.last', 'password', @token) }.to raise_error(TokenError) @@ -67,7 +63,7 @@ describe Pooler do it 'checks the status of a token' do stub_request(:get, "#{@abs_url}/token/utpg2i2xswor6h8ttjhu3d47z53yy47y") - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3' }) .to_return(:status => 200, :body => @token_status_response, :headers => {}) expect(Auth.token_status(false, @abs_url, @token)).to eq JSON.parse(@token_status_response) @@ -75,7 +71,7 @@ describe Pooler do it 'raises a token error if something goes wrong' do stub_request(:get, "#{@abs_url}/token/utpg2i2xswor6h8ttjhu3d47z53yy47y") - .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent' => 'Faraday v0.9.2' }) + .with(:headers => { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3' }) .to_return(:status => 500, :body => '{"ok":false}', :headers => {}) expect { Auth.token_status(false, @abs_url, @token) }.to raise_error(TokenError) diff --git a/spec/vmfloaty/abs_spec.rb b/spec/vmfloaty/abs_spec.rb new file mode 100644 index 0000000..3939e61 --- /dev/null +++ b/spec/vmfloaty/abs_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'vmfloaty/utils' +require 'vmfloaty/errors' +require 'vmfloaty/abs' + +describe ABS do + before :each do + end + + describe '#format' do + it 'returns an hash formatted like a vmpooler return' do + abs_formatted_response = [ + { 'hostname' => 'aaaaaaaaaaaaaaa.delivery.puppetlabs.net', 'type' => 'centos-7.2-x86_64', 'engine' => 'vmpooler' }, + { 'hostname' => 'aaaaaaaaaaaaaab.delivery.puppetlabs.net', 'type' => 'centos-7.2-x86_64', 'engine' => 'vmpooler' }, + { 'hostname' => 'aaaaaaaaaaaaaac.delivery.puppetlabs.net', 'type' => 'ubuntu-7.2-x86_64', 'engine' => 'vmpooler' }, + ] + + vmpooler_formatted_response = ABS.translated(abs_formatted_response) + + vmpooler_formatted_compare = { + 'centos-7.2-x86_64' => {}, + 'ubuntu-7.2-x86_64' => {}, + } + + vmpooler_formatted_compare['centos-7.2-x86_64']['hostname'] = ['aaaaaaaaaaaaaaa.delivery.puppetlabs.net', 'aaaaaaaaaaaaaab.delivery.puppetlabs.net'] + vmpooler_formatted_compare['ubuntu-7.2-x86_64']['hostname'] = ['aaaaaaaaaaaaaac.delivery.puppetlabs.net'] + + vmpooler_formatted_compare['ok'] = true + + expect(vmpooler_formatted_response).to eq(vmpooler_formatted_compare) + vmpooler_formatted_response.delete('ok') + vmpooler_formatted_compare.delete('ok') + expect(vmpooler_formatted_response).to eq(vmpooler_formatted_compare) + end + end +end diff --git a/spec/vmfloaty/nonstandard_pooler_spec.rb b/spec/vmfloaty/nonstandard_pooler_spec.rb index 4d6f7ff..6863078 100644 --- a/spec/vmfloaty/nonstandard_pooler_spec.rb +++ b/spec/vmfloaty/nonstandard_pooler_spec.rb @@ -66,7 +66,7 @@ describe NonstandardPooler do @token_status_body_active = <<~BODY { "ok": true, - "user": "first.last", + "user": 'first.last', "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": ["sol10-9", "sol10-11"] @@ -75,7 +75,7 @@ describe NonstandardPooler do @token_status_body_empty = <<~BODY { "ok": true, - "user": "first.last", + "user": 'first.last', "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": [] @@ -125,7 +125,7 @@ describe NonstandardPooler do .to_return(:status => 401, :body => '{"ok":false,"reason": "token: token-value does not exist"}', :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url) }.to raise_error(AuthError) + expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') }.to raise_error(AuthError) end it 'retrieves a single vm with a token' do @@ -134,7 +134,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_single, :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url) + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-11-sparc']['hostname']).to eq 'sol11-4.delivery.puppetlabs.net' @@ -146,7 +146,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_many, :headers => {}) vm_hash = { 'aix-7.1-power' => 1, 'solaris-10-sparc' => 2 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url) + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-10-sparc']['hostname']).to be_an_instance_of Array @@ -246,7 +246,7 @@ describe NonstandardPooler do "sol10-11": { "fqdn": "sol10-11.delivery.puppetlabs.net", "os_triple": "solaris-10-sparc", - "reserved_by_user": "first.last", + "reserved_by_user": 'first.last', "reserved_for_reason": "testing", "hours_left_on_reservation": 29.12 } diff --git a/spec/vmfloaty/service_spec.rb b/spec/vmfloaty/service_spec.rb index e622c15..13426b3 100644 --- a/spec/vmfloaty/service_spec.rb +++ b/spec/vmfloaty/service_spec.rb @@ -16,9 +16,9 @@ describe Service do it 'prompts the user for their password and retrieves a token' do config = { 'user' => 'first.last', 'url' => 'http://default.url' } service = Service.new(MockOptions.new, config) - allow(STDOUT).to receive(:puts).with('Enter your pooler service password:') + allow(STDOUT).to receive(:puts).with('Enter your http://default.url service password:') allow(Commander::UI).to(receive(:password) - .with('Enter your pooler service password:', '*') + .with('Enter your http://default.url service password:', '*') .and_return('hunter2')) allow(Auth).to(receive(:get_token) .with(nil, config['url'], config['user'], 'hunter2') @@ -29,11 +29,11 @@ describe Service do it 'prompts the user for their username and password if the username is unknown' do config = { 'url' => 'http://default.url' } service = Service.new(MockOptions.new({}), config) - allow(STDOUT).to receive(:puts).with 'Enter your pooler service username:' + allow(STDOUT).to receive(:puts).with 'Enter your http://default.url service username:' allow(STDOUT).to receive(:puts).with "\n" allow(STDIN).to receive(:gets).and_return('first.last') allow(Commander::UI).to(receive(:password) - .with('Enter your pooler service password:', '*') + .with('Enter your http://default.url service password:', '*') .and_return('hunter2')) allow(Auth).to(receive(:get_token) .with(nil, config['url'], 'first.last', 'hunter2') @@ -46,7 +46,7 @@ describe Service do it 'deletes a token' do service = Service.new(MockOptions.new, 'user' => 'first.last', 'url' => 'http://default.url') allow(Commander::UI).to(receive(:password) - .with('Enter your pooler service password:', '*') + .with('Enter your http://default.url service password:', '*') .and_return('hunter2')) allow(Auth).to(receive(:delete_token) .with(nil, 'http://default.url', 'first.last', 'hunter2', 'token-value') From 7e275426705545e3f36f7b46feb1cbb661aeb618 Mon Sep 17 00:00:00 2001 From: Mikker Gimenez-Peterson Date: Mon, 4 Nov 2019 11:41:49 -0800 Subject: [PATCH 3/6] Adding delete and get active requests --- lib/vmfloaty/abs.rb | 91 +++++++++++++++++++++++++++++++-------- lib/vmfloaty/pooler.rb | 2 +- lib/vmfloaty/service.rb | 2 +- lib/vmfloaty/utils.rb | 7 +++ spec/vmfloaty/abs_spec.rb | 24 +++++++++++ 5 files changed, 106 insertions(+), 20 deletions(-) diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index a826ed6..381d95b 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -7,8 +7,7 @@ require 'json' class ABS # List active VMs in ABS - # - # + # This is what a job request looks like: # { # "state":"filled", # "last_processed":"2019-10-31 20:59:33 +0000", @@ -37,33 +36,83 @@ class ABS # } # } # + + @@active_hostnames = Hash.new + def self.list_active(verbose, url, _token, user) + all_jobs = Array.new() + @@active_hostnames = Hash.new + + self.get_active_requests(verbose, url, user).each do |reqHash| + all_jobs.push(reqHash['request']['job']['id']) + @@active_hostnames[reqHash['request']['job']['id']] = reqHash + end + + all_jobs + end + + def self.get_active_requests verbose, url, user conn = Http.get_conn(verbose, url) res = conn.get 'status/queue' requests = JSON.parse(res.body) + retVal = [] requests.each do |req| reqHash = JSON.parse(req) next unless user == reqHash['request']['job']['user'] - - puts '------------------------------------' - puts "State: #{reqHash['state']}" - puts "Job ID: #{reqHash['request']['job']['id']}" - reqHash['request']['resources'].each do |vm_template, i| - puts "--VMRequest: #{vm_template}: #{i}" - end - if reqHash['state'] == 'allocated' || reqHash['state'] == 'filled' - reqHash['allocated_resources'].each do |vm_name, i| - puts "----VM: #{vm_name}: #{i}" - end - end - puts "User: #{reqHash['request']['job']['user']}" - puts '' + retVal.push(reqHash) end - sleep(100) + retVal end + def self.all_job_resources_accounted_for(allocated_resources, hosts) + allocated_host_list = allocated_resources.map {|ar| ar["hostname"] } + return (allocated_host_list-hosts).empty? + end + + def self.delete(verbose, url, hosts, token, user) + # In ABS terms, this is a "returned" host. + conn = Http.get_conn(verbose, url) + conn.headers['X-AUTH-TOKEN'] = token if token + + puts "Trying to delete hosts #{hosts}" if verbose + requests = self.get_active_requests(verbose, url, user) + + jobs_to_delete = [] + + requests.each do |reqHash| + if reqHash['state'] == 'allocated' || reqHash['state'] == 'filled' + reqHash['allocated_resources'].each do |vm_name, i| + if hosts.include? vm_name["hostname"] + if (all_job_resources_accounted_for(job['allocated_resources'], hosts)) + jobs_to_delete.push(reqHash) + else + puts "Can't delete #{job_id}: #{hosts} does not include all of #{job['allocated_resources']}" + end + end + end + end + end + + response_body = {} + + jobs_to_delete.each do |job| + reqObj = { + 'job_id': job['request']['job']['id'], + 'hosts': job['allocated_resources'], + } + + puts "Deleting #{reqObj}" if verbose + + res = conn.post 'api/v2/return', reqObj.to_json + response_body[job_id] = res_body + end + + return response_body + end + + # List available VMs in ABS def self.list(verbose, url, os_filter = nil) conn = Http.get_conn(verbose, url) @@ -96,7 +145,7 @@ class ABS # Retrieve an OS from ABS. def self.retrieve(verbose, os_types, token, url, user) # - # Contents of post must be:j + # Contents of post must be like: # # { # "resources": { @@ -179,6 +228,10 @@ class ABS [queue_info['queue_place'], nil] end + def self.snapshot(verbose, url, hostname, token) + puts "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)" + end + def self.status(verbose, url) conn = Http.get_conn(verbose, url) @@ -194,6 +247,8 @@ class ABS end def self.query(verbose, url, hostname) + return @@active_hostnames if @@active_hostnames + puts "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)" conn = Http.get_conn(verbose, url) res = conn.get "host/#{hostname}" diff --git a/lib/vmfloaty/pooler.rb b/lib/vmfloaty/pooler.rb index 69de7da..354b8a9 100644 --- a/lib/vmfloaty/pooler.rb +++ b/lib/vmfloaty/pooler.rb @@ -28,7 +28,7 @@ class Pooler vms end - def self.retrieve(verbose, os_type, token, url) + def self.retrieve(verbose, os_type, token, url, user) # NOTE: # Developers can use `Utils.generate_os_hash` to # generate the os_type param. diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 6d606f1..951db9c 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -112,7 +112,7 @@ class Service end def delete(verbose, hosts) - @service_object.delete verbose, url, hosts, token + @service_object.delete verbose, url, hosts, token, user end def status(verbose) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index bd83f7e..eb3464a 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -85,6 +85,13 @@ class Utils host_data = response[hostname] case service.type + when 'ABS' + # For ABS, 'hostname' variable is the jobID + if host_data['state'] == 'allocated' || host_data['state'] == 'filled' + host_data['allocated_resources'].each do |vm_name, i| + puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name["hostname"]} (#{vm_name["type"]}) <#{host_data['state']}>" + end + end when 'Pooler' tag_pairs = [] tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil? diff --git a/spec/vmfloaty/abs_spec.rb b/spec/vmfloaty/abs_spec.rb index 3939e61..c59e14c 100644 --- a/spec/vmfloaty/abs_spec.rb +++ b/spec/vmfloaty/abs_spec.rb @@ -34,5 +34,29 @@ describe ABS do vmpooler_formatted_compare.delete('ok') expect(vmpooler_formatted_response).to eq(vmpooler_formatted_compare) end + + it 'won\'t delete a job if not all vms are listed' do + hosts = ["host1"] + allocated_resources = [ + { + "hostname" => "host1" + }, + { + "hostname" => "host2" + } + ] + expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(false) + + hosts = ["host1", "host2"] + allocated_resources = [ + { + "hostname" => "host1" + }, + { + "hostname" => "host2" + } + ] + expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(true) + end end end From d963e357d37244d167d0f56a931b7c4caaa91ae8 Mon Sep 17 00:00:00 2001 From: Mikker Gimenez-Peterson Date: Mon, 4 Nov 2019 13:30:58 -0800 Subject: [PATCH 4/6] Adding delete and get active requests --- lib/vmfloaty.rb | 1 + lib/vmfloaty/abs.rb | 133 ++++++++++++++--------- lib/vmfloaty/nonstandard_pooler.rb | 2 +- lib/vmfloaty/pooler.rb | 2 +- lib/vmfloaty/service.rb | 2 +- lib/vmfloaty/utils.rb | 8 +- spec/vmfloaty/abs_spec.rb | 16 +-- spec/vmfloaty/nonstandard_pooler_spec.rb | 14 +-- spec/vmfloaty/pooler_spec.rb | 6 +- 9 files changed, 109 insertions(+), 75 deletions(-) diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index caf6a7a..285958c 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -33,6 +33,7 @@ class Vmfloaty c.option '--user STRING', String, 'User to authenticate with' c.option '--url STRING', String, 'URL of pooler service' c.option '--token STRING', String, 'Token for pooler service' + c.option '--priority STRING', 'Priority for supported backends(ABS) (High(1), Medium(2), Low(3))' c.option '--notoken', 'Makes a request without a token' c.option '--force', 'Forces vmfloaty to get requested vms' c.option '--json', 'Prints retrieved vms in JSON format' diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index 381d95b..29468c6 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -37,38 +37,39 @@ class ABS # } # - @@active_hostnames = Hash.new + @active_hostnames = {} def self.list_active(verbose, url, _token, user) - all_jobs = Array.new() - @@active_hostnames = Hash.new + all_jobs = [] + @active_hostnames = {} - self.get_active_requests(verbose, url, user).each do |reqHash| - all_jobs.push(reqHash['request']['job']['id']) - @@active_hostnames[reqHash['request']['job']['id']] = reqHash + get_active_requests(verbose, url, user).each do |req_hash| + all_jobs.push(req_hash['request']['job']['id']) + @active_hostnames[req_hash['request']['job']['id']] = req_hash end all_jobs end - def self.get_active_requests verbose, url, user + def self.get_active_requests(verbose, url, user) conn = Http.get_conn(verbose, url) res = conn.get 'status/queue' requests = JSON.parse(res.body) - retVal = [] + ret_val = [] requests.each do |req| - reqHash = JSON.parse(req) - next unless user == reqHash['request']['job']['user'] - retVal.push(reqHash) + req_hash = JSON.parse(req) + next unless user == req_hash['request']['job']['user'] + + ret_val.push(req_hash) end - retVal + ret_val end def self.all_job_resources_accounted_for(allocated_resources, hosts) - allocated_host_list = allocated_resources.map {|ar| ar["hostname"] } - return (allocated_host_list-hosts).empty? + allocated_host_list = allocated_resources.map { |ar| ar['hostname'] } + (allocated_host_list - hosts).empty? end def self.delete(verbose, url, hosts, token, user) @@ -77,19 +78,29 @@ class ABS conn.headers['X-AUTH-TOKEN'] = token if token puts "Trying to delete hosts #{hosts}" if verbose - requests = self.get_active_requests(verbose, url, user) + requests = get_active_requests(verbose, url, user) jobs_to_delete = [] - requests.each do |reqHash| - if reqHash['state'] == 'allocated' || reqHash['state'] == 'filled' - reqHash['allocated_resources'].each do |vm_name, i| - if hosts.include? vm_name["hostname"] - if (all_job_resources_accounted_for(job['allocated_resources'], hosts)) - jobs_to_delete.push(reqHash) - else - puts "Can't delete #{job_id}: #{hosts} does not include all of #{job['allocated_resources']}" - end + retStatus = {} + hosts.each do |host| + retStatus[host] = { + 'ok' => false + } + end + + requests.each do |req_hash| + next unless req_hash['state'] == 'allocated' || req_hash['state'] == 'filled' + + req_hash['allocated_resources'].each do |vm_name, _i| + if hosts.include? vm_name['hostname'] + if all_job_resources_accounted_for(req_hash['allocated_resources'], hosts) + retStatus[vm_name['hostname']] = { + 'ok' => true + } + jobs_to_delete.push(req_hash) + else + puts "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}" end end end @@ -98,21 +109,22 @@ class ABS response_body = {} jobs_to_delete.each do |job| - reqObj = { - 'job_id': job['request']['job']['id'], - 'hosts': job['allocated_resources'], + req_obj = { + 'job_id' => job['request']['job']['id'], + 'hosts' => job['allocated_resources'], } - puts "Deleting #{reqObj}" if verbose + puts "Deleting #{req_obj}" if verbose - res = conn.post 'api/v2/return', reqObj.to_json - response_body[job_id] = res_body + return_result = conn.post 'return', req_obj.to_json + req_obj['hosts'].each do |host| + response_body[host["hostname"]] = { 'ok' => true } if return_result.body == "OK" + end end - return response_body + response_body end - # List available VMs in ABS def self.list(verbose, url, os_filter = nil) conn = Http.get_conn(verbose, url) @@ -143,7 +155,7 @@ class ABS end # Retrieve an OS from ABS. - def self.retrieve(verbose, os_types, token, url, user) + def self.retrieve(verbose, os_types, token, url, user, options) # # Contents of post must be like: # @@ -155,8 +167,7 @@ class ABS # "job": { # "id": "12345", # "tags": { - # "user": "jenkins", - # "jenkins_build_url": "https://jenkins/job/platform_puppet_intn-van-sys_master" + # "user": "username", # } # } # } @@ -164,35 +175,51 @@ class ABS conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token - saved_job_id = Time.now.to_i + saved_job_id = DateTime.now.strftime('%Q') - reqObj = { + req_obj = { :resources => os_types, :job => { :id => saved_job_id, :tags => { - :user => user, - :url_string => "floaty://#{user}/#{saved_job_id}", + :user => user, }, }, } + if options["priority"] + if options["priority"] == "high" + req_obj[:priority] = 1 + elsif options["priority"] == "medium" + req_obj[:priority] = 2 + elsif options["priority"] == "low" + req_obj[:priority] = 3 + else + req_obj[:priority] = options["priority"].to_i + end + end + + puts "Posting to ABS #{req_obj.to_json}" if verbose + # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? puts "Requesting VMs with job_id: #{saved_job_id}. Will retry for up to an hour." - res = conn.post 'api/v2/request', reqObj.to_json + res = conn.post 'request', req_obj.to_json - i = 0 retries = 360 raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401 (1..retries).each do |i| - queue_place, res_body = check_queue(conn, saved_job_id, reqObj) + queue_place, res_body = check_queue(conn, saved_job_id, req_obj) return translated(res_body) if res_body - puts "Waiting 10 seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" - sleep(10) + + sleepSeconds = 10 if i >= 10 + sleepSeconds = i if i < 10 + puts "Waiting #{sleepSeconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" + + sleep(sleepSeconds) end nil end @@ -215,11 +242,11 @@ class ABS vmpooler_formatted_body end - def self.check_queue(conn, job_id, reqObj) - queue_info_res = conn.get "/status/queue/info/#{job_id}" + def self.check_queue(conn, job_id, req_obj) + queue_info_res = conn.get "status/queue/info/#{job_id}" queue_info = JSON.parse(queue_info_res.body) - res = conn.post 'api/v2/request', reqObj.to_json + res = conn.post 'request', req_obj.to_json unless res.body.empty? res_body = JSON.parse(res.body) @@ -228,26 +255,28 @@ class ABS [queue_info['queue_place'], nil] end - def self.snapshot(verbose, url, hostname, token) + def self.snapshot(_verbose, _url, _hostname, _token) puts "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)" end def self.status(verbose, url) conn = Http.get_conn(verbose, url) - res = conn.get '/status' - JSON.parse(res.body) + res = conn.get 'status' + + return res.body == "OK" end def self.summary(verbose, url) conn = Http.get_conn(verbose, url) - res = conn.get '/summary' + res = conn.get 'summary' JSON.parse(res.body) end def self.query(verbose, url, hostname) - return @@active_hostnames if @@active_hostnames + return @active_hostnames if @active_hostnames + puts "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)" conn = Http.get_conn(verbose, url) diff --git a/lib/vmfloaty/nonstandard_pooler.rb b/lib/vmfloaty/nonstandard_pooler.rb index d094288..89f6c64 100644 --- a/lib/vmfloaty/nonstandard_pooler.rb +++ b/lib/vmfloaty/nonstandard_pooler.rb @@ -22,7 +22,7 @@ class NonstandardPooler status['reserved_hosts'] || [] end - def self.retrieve(verbose, os_type, token, url, _user) + def self.retrieve(verbose, os_type, token, url, _user, _options) conn = Http.get_conn(verbose, url) conn.headers['X-AUTH-TOKEN'] = token if token diff --git a/lib/vmfloaty/pooler.rb b/lib/vmfloaty/pooler.rb index 354b8a9..a735c3b 100644 --- a/lib/vmfloaty/pooler.rb +++ b/lib/vmfloaty/pooler.rb @@ -28,7 +28,7 @@ class Pooler vms end - def self.retrieve(verbose, os_type, token, url, user) + def self.retrieve(verbose, os_type, token, url, _user, _options) # NOTE: # Developers can use `Utils.generate_os_hash` to # generate the os_type param. diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 951db9c..28fc90c 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -78,7 +78,7 @@ class Service def retrieve(verbose, os_types, use_token = true) puts 'Requesting a vm without a token...' unless use_token token_value = use_token ? token : nil - @service_object.retrieve verbose, os_types, token_value, url, user + @service_object.retrieve verbose, os_types, token_value, url, user, @config end def ssh(verbose, host_os, use_token = true) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index eb3464a..f70eb20 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -88,8 +88,8 @@ class Utils when 'ABS' # For ABS, 'hostname' variable is the jobID if host_data['state'] == 'allocated' || host_data['state'] == 'filled' - host_data['allocated_resources'].each do |vm_name, i| - puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name["hostname"]} (#{vm_name["type"]}) <#{host_data['state']}>" + host_data['allocated_resources'].each do |vm_name, _i| + puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>" end end when 'Pooler' @@ -155,6 +155,9 @@ class Utils puts "#{name.ljust(width)} #{e.red}" end end + when 'ABS' + puts "ABS Not OK".red unless status_response + puts "ABS is OK".green if status_response else raise "Invalid service type #{service.type}" end @@ -205,6 +208,7 @@ class Utils end # Prioritize an explicitly specified url, user, or token if the user provided one + service_config['priority'] = options.priority unless options.priority.nil? service_config['url'] = options.url unless options.url.nil? service_config['token'] = options.token unless options.token.nil? service_config['user'] = options.user unless options.user.nil? diff --git a/spec/vmfloaty/abs_spec.rb b/spec/vmfloaty/abs_spec.rb index c59e14c..5a94b72 100644 --- a/spec/vmfloaty/abs_spec.rb +++ b/spec/vmfloaty/abs_spec.rb @@ -36,25 +36,25 @@ describe ABS do end it 'won\'t delete a job if not all vms are listed' do - hosts = ["host1"] + hosts = ['host1'] allocated_resources = [ { - "hostname" => "host1" + 'hostname' => 'host1', }, { - "hostname" => "host2" - } + 'hostname' => 'host2', + }, ] expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(false) - hosts = ["host1", "host2"] + hosts = ['host1', 'host2'] allocated_resources = [ { - "hostname" => "host1" + 'hostname' => 'host1', }, { - "hostname" => "host2" - } + 'hostname' => 'host2', + }, ] expect(ABS.all_job_resources_accounted_for(allocated_resources, hosts)).to eq(true) end diff --git a/spec/vmfloaty/nonstandard_pooler_spec.rb b/spec/vmfloaty/nonstandard_pooler_spec.rb index 6863078..911c96c 100644 --- a/spec/vmfloaty/nonstandard_pooler_spec.rb +++ b/spec/vmfloaty/nonstandard_pooler_spec.rb @@ -66,7 +66,7 @@ describe NonstandardPooler do @token_status_body_active = <<~BODY { "ok": true, - "user": 'first.last', + "user": "first.last", "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": ["sol10-9", "sol10-11"] @@ -75,7 +75,7 @@ describe NonstandardPooler do @token_status_body_empty = <<~BODY { "ok": true, - "user": 'first.last', + "user": "first.last", "created": "2017-09-18 01:25:41 +0000", "last_accessed": "2017-09-21 19:46:25 +0000", "reserved_hosts": [] @@ -88,7 +88,7 @@ describe NonstandardPooler do .with(false, @nspooler_url, 'token-value') .and_return(JSON.parse(@token_status_body_active)) - list = NonstandardPooler.list_active(false, @nspooler_url, 'token-value') + list = NonstandardPooler.list_active(false, @nspooler_url, 'token-value', 'user') expect(list).to eql ['sol10-9', 'sol10-11'] end end @@ -125,7 +125,7 @@ describe NonstandardPooler do .to_return(:status => 401, :body => '{"ok":false,"reason": "token: token-value does not exist"}', :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') }.to raise_error(AuthError) + expect { NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) }.to raise_error(AuthError) end it 'retrieves a single vm with a token' do @@ -134,7 +134,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_single, :headers => {}) vm_hash = { 'solaris-11-sparc' => 1 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-11-sparc']['hostname']).to eq 'sol11-4.delivery.puppetlabs.net' @@ -146,7 +146,7 @@ describe NonstandardPooler do .to_return(:status => 200, :body => @retrieve_response_body_many, :headers => {}) vm_hash = { 'aix-7.1-power' => 1, 'solaris-10-sparc' => 2 } - vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last') + vm_req = NonstandardPooler.retrieve(false, vm_hash, 'token-value', @nspooler_url, 'first.last', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['solaris-10-sparc']['hostname']).to be_an_instance_of Array @@ -246,7 +246,7 @@ describe NonstandardPooler do "sol10-11": { "fqdn": "sol10-11.delivery.puppetlabs.net", "os_triple": "solaris-10-sparc", - "reserved_by_user": 'first.last', + "reserved_by_user": "first.last", "reserved_for_reason": "testing", "hours_left_on_reservation": 29.12 } diff --git a/spec/vmfloaty/pooler_spec.rb b/spec/vmfloaty/pooler_spec.rb index 62035e4..dffaf2f 100644 --- a/spec/vmfloaty/pooler_spec.rb +++ b/spec/vmfloaty/pooler_spec.rb @@ -53,7 +53,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 1 - expect { Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) }.to raise_error(AuthError) + expect { Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) }.to raise_error(AuthError) end it 'retrieves a single vm with a token' do @@ -63,7 +63,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 1 - vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) + vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['debian-7-i386']['hostname']).to eq 'fq6qlpjlsskycq6' @@ -77,7 +77,7 @@ describe Pooler do vm_hash = {} vm_hash['debian-7-i386'] = 2 vm_hash['centos-7-x86_64'] = 1 - vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url) + vm_req = Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}) expect(vm_req).to be_an_instance_of Hash expect(vm_req['ok']).to equal true expect(vm_req['debian-7-i386']['hostname']).to be_an_instance_of Array From 9a2cd816df6ce5f4b5434ca5b728014a17ea5ac1 Mon Sep 17 00:00:00 2001 From: Brandon High Date: Tue, 3 Dec 2019 17:06:08 -0800 Subject: [PATCH 5/6] Update Rubocop to not use old cop name --- .rubocop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index fd3b135..e80ae47 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,9 +14,9 @@ Style/TrailingCommaInArrayLiteral: Style/TrailingCommaInArguments: EnforcedStyleForMultiline: comma -Layout/AlignHash: +Layout/HashAlignment: EnforcedHashRocketStyle: table -Layout/IndentFirstHashElement: +Layout/FirstHashElementIndentation: EnforcedStyle: consistent Style/StderrPuts: From 1e67715e2c019f7ae597889053c24a34591f0abc Mon Sep 17 00:00:00 2001 From: Brandon High Date: Wed, 4 Dec 2019 16:40:57 -0800 Subject: [PATCH 6/6] Fix rubocop issues --- .rubocop.yml | 2 ++ lib/vmfloaty.rb | 2 +- lib/vmfloaty/abs.rb | 45 +++++++++++++++++++++---------------------- lib/vmfloaty/utils.rb | 4 ++-- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index e80ae47..1bbc983 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -18,6 +18,8 @@ Layout/HashAlignment: EnforcedHashRocketStyle: table Layout/FirstHashElementIndentation: EnforcedStyle: consistent +Metrics/ParameterLists: + Enabled: False Style/StderrPuts: Enabled: false diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index 285958c..3672c8b 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -17,7 +17,7 @@ require 'vmfloaty/ssh' class Vmfloaty include Commander::Methods - def run + def run # rubocop:disable Metrics/AbcSize program :version, Vmfloaty::VERSION program :description, 'A CLI helper tool for Puppet Labs VM poolers to help you stay afloat' diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index 29468c6..18347ed 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -82,10 +82,10 @@ class ABS jobs_to_delete = [] - retStatus = {} + ret_status = {} hosts.each do |host| - retStatus[host] = { - 'ok' => false + ret_status[host] = { + 'ok' => false, } end @@ -95,11 +95,11 @@ class ABS req_hash['allocated_resources'].each do |vm_name, _i| if hosts.include? vm_name['hostname'] if all_job_resources_accounted_for(req_hash['allocated_resources'], hosts) - retStatus[vm_name['hostname']] = { - 'ok' => true + ret_status[vm_name['hostname']] = { + 'ok' => true, } jobs_to_delete.push(req_hash) - else + else puts "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}" end end @@ -118,7 +118,7 @@ class ABS return_result = conn.post 'return', req_obj.to_json req_obj['hosts'].each do |host| - response_body[host["hostname"]] = { 'ok' => true } if return_result.body == "OK" + response_body[host['hostname']] = { 'ok' => true } if return_result.body == 'OK' end end @@ -187,16 +187,16 @@ class ABS }, } - if options["priority"] - if options["priority"] == "high" - req_obj[:priority] = 1 - elsif options["priority"] == "medium" - req_obj[:priority] = 2 - elsif options["priority"] == "low" - req_obj[:priority] = 3 - else - req_obj[:priority] = options["priority"].to_i - end + if options['priority'] + req_obj[:priority] = if options['priority'] == 'high' + 1 + elsif options['priority'] == 'medium' + 2 + elsif options['priority'] == 'low' + 3 + else + options['priority'].to_i + end end puts "Posting to ABS #{req_obj.to_json}" if verbose @@ -214,12 +214,11 @@ class ABS queue_place, res_body = check_queue(conn, saved_job_id, req_obj) return translated(res_body) if res_body - - sleepSeconds = 10 if i >= 10 - sleepSeconds = i if i < 10 - puts "Waiting #{sleepSeconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" + sleep_seconds = 10 if i >= 10 + sleep_seconds = i if i < 10 + puts "Waiting #{sleep_seconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})" - sleep(sleepSeconds) + sleep(sleep_seconds) end nil end @@ -264,7 +263,7 @@ class ABS res = conn.get 'status' - return res.body == "OK" + res.body == 'OK' end def self.summary(verbose, url) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index f70eb20..1e2e678 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -156,8 +156,8 @@ class Utils end end when 'ABS' - puts "ABS Not OK".red unless status_response - puts "ABS is OK".green if status_response + puts 'ABS Not OK'.red unless status_response + puts 'ABS is OK'.green if status_response else raise "Invalid service type #{service.type}" end