From 05a133fc7215e6347b2d6f7974be0753f848c898 Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Fri, 22 May 2020 15:36:21 -0700 Subject: [PATCH 1/2] Add support for ondemand provisioning to vmfloaty This commit adds support for provisioning instances on demand with vmpooler. Additionally, this change adds a capability to detect on demand pools available in ABS. Without this change vmfloaty does not support provisioning instances via the vmpooler ondemand endpoints. --- lib/vmfloaty.rb | 9 +++++++-- lib/vmfloaty/abs.rb | 10 +++++++++- lib/vmfloaty/pooler.rb | 36 +++++++++++++++++++++++++++++++++--- lib/vmfloaty/service.rb | 8 ++++++-- lib/vmfloaty/utils.rb | 4 +++- spec/vmfloaty/pooler_spec.rb | 20 ++++++++++++++++++++ 6 files changed, 78 insertions(+), 9 deletions(-) diff --git a/lib/vmfloaty.rb b/lib/vmfloaty.rb index 9eec698..9ba0bd0 100644 --- a/lib/vmfloaty.rb +++ b/lib/vmfloaty.rb @@ -37,6 +37,7 @@ class Vmfloaty c.option '--notoken', 'Makes a request without a token' c.option '--force', 'Forces vmfloaty to get requested vms' c.option '--json', 'Prints retrieved vms in JSON format' + c.option '--ondemand', 'Requested vms are provisioned upon receival of the request, tracked by a request ID' c.action do |args, options| verbose = options.verbose || config['verbose'] service = Service.new(options, config) @@ -63,9 +64,13 @@ class Vmfloaty exit 1 end - response = service.retrieve(verbose, os_types, use_token) + response = service.retrieve(verbose, os_types, use_token, options.ondemand) + request_id = response['request_id'] if options.ondemand + response = service.wait_for_request(verbose, request_id) if options.ondemand + hosts = Utils.standardize_hostnames(response) - if options.json + + if options.json || options.ondemand puts JSON.pretty_generate(hosts) else puts Utils.format_host_output(hosts) diff --git a/lib/vmfloaty/abs.rb b/lib/vmfloaty/abs.rb index 259f411..d644472 100644 --- a/lib/vmfloaty/abs.rb +++ b/lib/vmfloaty/abs.rb @@ -150,6 +150,14 @@ class ABS os_list << '*** VMPOOLER Pools ***' os_list += JSON.parse(res_body['vmpooler_platforms']) + res = conn.get 'status/platforms/ondemand_vmpooler' + res_body = JSON.parse(res.body) + unless res_body['ondemand_vmpooler_platforms'] == '[]' + os_list << '' + os_list << '*** VMPOOLER ONDEMAND Pools ***' + os_list += JSON.parse(res_body['ondemand_vmpooler_platforms']) + end + res = conn.get 'status/platforms/nspooler' res_body = JSON.parse(res.body) os_list << '' @@ -168,7 +176,7 @@ class ABS end # Retrieve an OS from ABS. - def self.retrieve(verbose, os_types, token, url, user, options) + def self.retrieve(verbose, os_types, token, url, user, options, _ondemand = nil) # # Contents of post must be like: # diff --git a/lib/vmfloaty/pooler.rb b/lib/vmfloaty/pooler.rb index c85daf3..8cd11e1 100644 --- a/lib/vmfloaty/pooler.rb +++ b/lib/vmfloaty/pooler.rb @@ -28,7 +28,7 @@ class Pooler vms end - def self.retrieve(verbose, os_type, token, url, _user, _options) + def self.retrieve(verbose, os_type, token, url, _user, _options, ondemand = nil) # NOTE: # Developers can use `Utils.generate_os_hash` to # generate the os_type param. @@ -38,7 +38,8 @@ class Pooler os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+') raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty? - response = conn.post "vm/#{os_string}" + response = conn.post "vm/#{os_string}" unless ondemand + response ||= conn.post "ondemandvm/#{os_string}" res_body = JSON.parse(response.body) @@ -46,11 +47,40 @@ class Pooler res_body elsif response.status == 401 raise AuthError, "HTTP #{response.status}: The token provided could not authenticate to the pooler.\n#{res_body}" + elsif response.status == 403 + raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/vm/#{os_string}. Request exceeds the configured per pool maximum. #{res_body}" else - raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/vm/#{os_string}. #{res_body}" + raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/vm/#{os_string}. #{res_body}" unless ondemand + raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/ondemandvm/#{os_string}. #{res_body}" end end + def self.wait_for_request(verbose, request_id, url, timeout = 300) + start_time = Time.now + while check_ondemandvm(verbose, request_id, url) == false + return false if (Time.now - start_time).to_i > timeout + + STDOUT.puts "waiting for request #{request_id} to be fulfilled" + sleep 5 + end + STDOUT.puts "The request has been fulfilled" + check_ondemandvm(verbose, request_id, url) + end + + def self.check_ondemandvm(verbose, request_id, url) + conn = Http.get_conn(verbose, url) + + response = conn.get "ondemandvm/#{request_id}" + res_body = JSON.parse(response.body) + return res_body if response.status == 200 + + return false if response.status == 202 + + raise "HTTP #{response.status}: The request cannot be found, or an unknown error occurred" if response.status == 404 + + false + end + def self.modify(verbose, url, hostname, token, modify_hash) raise TokenError, 'Token provided was nil. Request cannot be made to modify vm' if token.nil? diff --git a/lib/vmfloaty/service.rb b/lib/vmfloaty/service.rb index 6a89b3d..87428f7 100644 --- a/lib/vmfloaty/service.rb +++ b/lib/vmfloaty/service.rb @@ -75,10 +75,14 @@ class Service @service_object.list_active verbose, url, token, user end - def retrieve(verbose, os_types, use_token = true) + def retrieve(verbose, os_types, use_token = true, ondemand = nil) puts 'Requesting a vm without a token...' unless use_token token_value = use_token ? token : nil - @service_object.retrieve verbose, os_types, token_value, url, user, @config + @service_object.retrieve verbose, os_types, token_value, url, user, @config, ondemand + end + + def wait_for_request(verbose, requestid) + @service_object.wait_for_request verbose, requestid, url end def ssh(verbose, host_os, use_token = true) diff --git a/lib/vmfloaty/utils.rb b/lib/vmfloaty/utils.rb index 1e2e678..105c374 100644 --- a/lib/vmfloaty/utils.rb +++ b/lib/vmfloaty/utils.rb @@ -45,7 +45,9 @@ class Utils result = {} - response_body.each do |os, value| + STDOUT.puts "response body is #{response_body}" + filtered_response_body = response_body.reject { |key, _| key == 'request_id' || key == 'ready' } + filtered_response_body.each do |os, value| hostnames = Array(value['hostname']) hostnames.map! { |host| "#{host}.#{domain}" } if domain result[os] = hostnames diff --git a/spec/vmfloaty/pooler_spec.rb b/spec/vmfloaty/pooler_spec.rb index 6d7def0..7d83c79 100644 --- a/spec/vmfloaty/pooler_spec.rb +++ b/spec/vmfloaty/pooler_spec.rb @@ -84,6 +84,26 @@ describe Pooler do expect(vm_req['debian-7-i386']['hostname']).to eq %w[sc0o4xqtodlul5w 4m4dkhqiufnjmxy] expect(vm_req['centos-7-x86_64']['hostname']).to eq 'zb91y9qbrbf6d3q' end + + context 'with ondemand provisioning' do + let(:ondemand_response) { '{"ok":true,"request_id":"1234"}' } + it 'retreives the vm with a token' do + stub_request(:post, "#{@vmpooler_url}/ondemandvm/debian-7-i386") + .with(:headers => { 'X-Auth-Token' => 'mytokenfile' }) + .to_return(:status => 200, :body => ondemand_response, :headers => {}) + + stub_request(:get, "#{@vmpooler_url}/ondemandvm/1234") + .to_return(:status => 200, :body => @retrieve_response_body_single, :headers => {}) + + vm_hash = {} + vm_hash['debian-7-i386'] = 1 + Pooler.retrieve(false, vm_hash, 'mytokenfile', @vmpooler_url, 'user', {}, true) + vm_req = Pooler.check_ondemandvm(false, '1234', @vmpooler_url) + expect(vm_req).to be_an_instance_of Hash + expect(vm_req['ok']).to equal true + expect(vm_req['debian-7-i386']['hostname']).to eq 'fq6qlpjlsskycq6' + end + end end describe '#modify' do From 7b0a7679daed55106a25a53e6c858927fab9490d Mon Sep 17 00:00:00 2001 From: "kirby@puppetlabs.com" Date: Fri, 22 May 2020 15:36:49 -0700 Subject: [PATCH 2/2] Add Dockerfile to make it easier to test changes to vmfloaty by using Docker --- .dockerignore | 12 ++++++++++++ Dockerfile | 6 ++++++ 2 files changed, 18 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6174523 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +**/*.yml +**/*.yaml +**/*.md +**/*example +**/Dockerfile* +Gemfile.lock +Rakefile +coverage +spec +examples +scripts +vendor diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c564fa8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM ruby:2.7 + +COPY ./ ./ + +RUN apt-get update && apt-get install -y less +RUN gem install bundler && bundle install && gem build vmfloaty.gemspec && gem install vmfloaty*.gem