puppetlabs/vmfloaty
2
0
Fork 0
mirror of https://github.com/puppetlabs/vmfloaty.git synced 2026-01-26 05:28:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #94 from puppetlabs/fix-abs-vmpooler

Browse source 
ABS enables fallback to vmpooler for some scenarios
This commit is contained in:
mattkirby 2020-09-11 13:27:02 -07:00 • committed by GitHub
commit 512adb4af1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 178 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

16
README.md
View file

@ -105,12 +105,20 @@ Now vmfloaty will use those config files if no flag was specified.
#### Default to Puppet's ABS instead of vmpooler
When the --service is not specified on the command line, the first one is selected, so put ABS first.
Also provide a "vmpooler" service that ABS can use as fallback for operations targeting vmpooler directly
```yaml
# file at ~/.vmfloaty.yml
url: 'https://abs.example.net'
user: 'brian'
token: 'tokenstring'
type: 'abs'
services:
abs:
url: 'https://abs/api/v2'
type: 'abs'
user: 'samuel'
token: 'foo'
vmpooler:
url: 'http://vmpooler'
user: 'samuel'
token: 'bar'
```
#### Configuring multiple services

26
lib/vmfloaty.rb
View file

@ -99,7 +99,12 @@ class Vmfloaty
if options.active
# list active vms
running_vms = service.list_active(verbose)
if service.type == "ABS"
# this is actually job_ids
running_vms = service.list_active_job_ids(verbose, service.url, service.user)
else
running_vms = service.list_active(verbose)
end
host = URI.parse(service.url).host
if running_vms.empty?
if options.json
@ -126,7 +131,7 @@ class Vmfloaty
command :query do |c|
c.syntax = 'floaty query hostname [options]'
c.summary = 'Get information about a given vm'
c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm.'
c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm. If using ABS, you can query a job_id'
c.example 'Get information about a sample host', 'floaty query hostname --url http://vmpooler.example.com'
c.option '--verbose', 'Enables verbose output'
c.option '--service STRING', String, 'Configured pooler service name'
@ -165,7 +170,12 @@ class Vmfloaty
FloatyLogger.error 'ERROR: Provide a hostname or specify --all.'
exit 1
end
running_vms = modify_all ? service.list_active(verbose) : hostname.split(',')
running_vms =
if modify_all
service.list_active(verbose)
else
hostname.split(',')
end
tags = options.tags ? JSON.parse(options.tags) : nil
modify_hash = {
@ -189,7 +199,7 @@ class Vmfloaty
end
if ok
if modify_all
puts 'Successfully modified all VMs.'
puts "Successfully modified all #{running_vms.count} VMs."
else
puts "Successfully modified VM #{hostname}."
end
@ -225,7 +235,12 @@ class Vmfloaty
successes = []
if delete_all
running_vms = service.list_active(verbose)
if service.type == "ABS"
# this is actually job_ids
running_vms = service.list_active_job_ids(verbose, service.url, service.user)
else
running_vms = service.list_active(verbose)
end
if running_vms.empty?
if options.json
puts {}.to_json
@ -274,7 +289,6 @@ class Vmfloaty
end
unless successes.empty?
FloatyLogger.info unless failures.empty?
if options.json
puts successes.to_json
else

133
lib/vmfloaty/abs.rb
View file

@ -2,6 +2,7 @@
require 'vmfloaty/errors'
require 'vmfloaty/http'
require 'vmfloaty/utils'
require 'faraday'
require 'json'
@ -36,39 +37,59 @@ class ABS
# }
# }
#
@active_hostnames = {}
def self.list_active(verbose, url, _token, user)
all_jobs = []
def self.list_active_job_ids(verbose, url, user)
all_job_ids = []
@active_hostnames = {}
get_active_requests(verbose, url, user).each do |req_hash|
all_jobs.push(req_hash['request']['job']['id'])
@active_hostnames[req_hash['request']['job']['id']] = req_hash
@active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
all_job_ids.push(req_hash['request']['job']['id'])
end
all_jobs
all_job_ids
end
def self.list_active(verbose, url, _token, user)
hosts = []
get_active_requests(verbose, url, user).each do |req_hash|
if req_hash.key?('allocated_resources')
req_hash['allocated_resources'].each do |onehost|
hosts.push(onehost['hostname'])
end
end
end
hosts
end
def self.get_active_requests(verbose, url, user)
conn = Http.get_conn(verbose, url)
res = conn.get 'status/queue'
requests = JSON.parse(res.body)
if valid_json?(res.body)
requests = JSON.parse(res.body)
else
FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
end
ret_val = []
requests.each do |req|
next if req == 'null'
req_hash = JSON.parse(req)
if valid_json?(req)
req_hash = JSON.parse(req)
else
FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
next
end
begin
next unless user == req_hash['request']['job']['user']
ret_val.push(req_hash)
rescue NoMethodError
FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: "
FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
end
end
@ -145,30 +166,43 @@ class ABS
os_list = []
res = conn.get 'status/platforms/vmpooler'
res_body = JSON.parse(res.body)
os_list << '*** VMPOOLER Pools ***'
os_list += JSON.parse(res_body['vmpooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('vmpooler_platforms')
os_list << '*** VMPOOLER Pools ***'
os_list += JSON.parse(res_body['vmpooler_platforms'])
end
end
res = conn.get 'status/platforms/ondemand_vmpooler'
res_body = JSON.parse(res.body)
unless res_body['ondemand_vmpooler_platforms'] == '[]'
os_list << ''
os_list << '*** VMPOOLER ONDEMAND Pools ***'
os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
os_list << ''
os_list << '*** VMPOOLER ONDEMAND Pools ***'
os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
end
end
res = conn.get 'status/platforms/nspooler'
res_body = JSON.parse(res.body)
os_list << ''
os_list << '*** NSPOOLER Pools ***'
os_list += JSON.parse(res_body['nspooler_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('nspooler_platforms')
os_list << ''
os_list << '*** NSPOOLER Pools ***'
os_list += JSON.parse(res_body['nspooler_platforms'])
end
end
res = conn.get 'status/platforms/aws'
res_body = JSON.parse(res.body)
os_list << ''
os_list << '*** AWS Pools ***'
os_list += JSON.parse(res_body['aws_platforms'])
if valid_json?(res.body)
res_body = JSON.parse(res.body)
if res_body.key?('aws_platforms')
os_list << ''
os_list << '*** AWS Pools ***'
os_list += JSON.parse(res_body['aws_platforms'])
end
end
os_list.delete 'ok'
@ -197,7 +231,7 @@ class ABS
conn.headers['X-AUTH-TOKEN'] = token if token
saved_job_id = DateTime.now.strftime('%Q')
vmpooler_config = Utils.get_vmpooler_service_config
req_obj = {
:resources => os_types,
:job => {
@ -206,6 +240,7 @@ class ABS
:user => user,
},
},
:vm_token => vmpooler_config['token'] # request with this token, on behalf of this user
}
if options['priority']
@ -264,12 +299,17 @@ class ABS
def self.check_queue(conn, job_id, req_obj, verbose)
queue_info_res = conn.get "status/queue/info/#{job_id}"
queue_info = JSON.parse(queue_info_res.body)
if valid_json?(queue_info_res.body)
queue_info = JSON.parse(queue_info_res.body)
else
FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
return [nil, nil]
end
res = conn.post 'request', req_obj.to_json
validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
unless res.body.empty?
unless res.body.empty? || !valid_json?(res.body)
res_body = JSON.parse(res.body)
return queue_info['queue_place'], res_body
end
@ -277,7 +317,7 @@ class ABS
end
def self.snapshot(_verbose, _url, _hostname, _token)
FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
end
def self.status(verbose, url)
@ -289,20 +329,24 @@ class ABS
end
def self.summary(verbose, url)
conn = Http.get_conn(verbose, url)
res = conn.get 'summary'
JSON.parse(res.body)
raise NoMethodError, 'summary is not defined for ABS'
end
def self.query(verbose, url, hostname)
return @active_hostnames if @active_hostnames
def self.query(verbose, url, job_id)
# return saved hostnames from the last time list_active was run
# preventing having to query the API again.
# This works as long as query is called after list_active
return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
# If using the cli query job_id
conn = Http.get_conn(verbose, url)
res = conn.get "host/#{hostname}"
JSON.parse(res.body)
queue_info_res = conn.get "status/queue/info/#{job_id}"
if valid_json?(queue_info_res.body)
queue_info = JSON.parse(queue_info_res.body)
else
FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
end
queue_info
end
def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@ -333,4 +377,11 @@ class ABS
raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
end
end
def self.valid_json?(json)
JSON.parse(json)
return true
rescue TypeError, JSON::ParserError => e
return false
end
end

20
lib/vmfloaty/service.rb
View file

@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
class Service
attr_reader :config
attr_accessor :silent
def initialize(options, config_hash = {})
options ||= Commander::Command::Options.new
@config = Utils.get_service_config config_hash, options
@service_object = Utils.get_service_object @config['type']
@silent = false
end
def method_missing(method_name, *args, &block)
@ -103,6 +105,7 @@ class Service
end
def modify(verbose, hostname, modify_hash)
maybe_use_vmpooler
@service_object.modify verbose, url, hostname, token, modify_hash
end
@ -115,18 +118,35 @@ class Service
end
def summary(verbose)
maybe_use_vmpooler
@service_object.summary verbose, url
end
def snapshot(verbose, hostname)
maybe_use_vmpooler
@service_object.snapshot verbose, url, hostname, token
end
def revert(verbose, hostname, snapshot_sha)
maybe_use_vmpooler
@service_object.revert verbose, url, hostname, token, snapshot_sha
end
def disk(verbose, hostname, disk)
maybe_use_vmpooler
@service_object.disk(verbose, url, hostname, token, disk)
end
# some methods do not exist for ABS, and if possible should target the Pooler service
def maybe_use_vmpooler
if @service_object.is_a?(ABS.class)
if !self.silent
FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using vmpooler config from ~/.vmfloaty.yml"
self.silent = true
end
@config = Utils.get_vmpooler_service_config
@service_object = Pooler
end
end
end

38
lib/vmfloaty/utils.rb
View file

@ -3,6 +3,7 @@
require 'vmfloaty/abs'
require 'vmfloaty/nonstandard_pooler'
require 'vmfloaty/pooler'
require 'vmfloaty/conf'
class Utils
# TODO: Takes the json response body from an HTTP GET
@ -78,21 +79,28 @@ class Utils
os_types
end
def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false)
def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0)
fetched_data = self.get_host_data(verbose, service, hostnames)
fetched_data.each do |hostname, host_data|
case service.type
when 'ABS'
# For ABS, 'hostname' variable is the jobID
# For ABS, 'hostname' variable is the jobID
#
# Create a vmpooler service to query each hostname there so as to get the metadata too
vmpooler_service = service.clone
vmpooler_service.silent = true
vmpooler_service.maybe_use_vmpooler
puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>"
host_data['allocated_resources'].each do |vm_name, _i|
puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>"
self.pretty_print_hosts(verbose, vmpooler_service, vm_name['hostname'].split('.')[0], print_to_stderr, indent+2)
end
when 'Pooler'
tag_pairs = []
tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
metadata = [host_data['template'], duration, *tag_pairs]
puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})"
puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent)
when 'NonstandardPooler'
line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
line += ", #{host_data['hours_left_on_reservation']}h remaining"
@ -241,4 +249,26 @@ class Utils
service_config
end
# This method gets the vmpooler service configured in ~/.vmfloaty
def self.get_vmpooler_service_config
config = Conf.read_config
# The top-level url, user, and token values in the config file are treated as defaults
service_config = {
'url' => config['url'],
'user' => config['user'],
'token' => config['token'],
'type' => 'vmpooler',
}
# at a minimum, the url needs to be configured
if config['services'] && config['services']['vmpooler'] && config['services']['vmpooler']['url']
# If the service is configured but some values are missing, use the top-level defaults to fill them in
service_config.merge! config['services']['vmpooler']
else
raise ArgumentError, "Could not find a configured service named 'vmpooler' in ~/.vmfloaty.yml use this format:\nservices:\n vmpooler:\n url: 'http://vmpooler.com'\n user: 'superman'\n token: 'kryptonite'"
end
service_config
end
end