The main goal for this PR is to enable using all of the vmfloaty scenarios when

the service is set to ABS. Since ABS does not implement all the services, it
fallsback on vmpooler when needed (example increasing the lifetime value)

- Validating JSON can be parsed before parsing, as it used to throw uncaught errors
- self.list_active is returning hosts for both nspooler and vmpooler but was returning
job_ids for abs. Now standardized the returned values for use in the "modify" cli,
and created a separate list_active_job_ids for the cases where a job_id list is expected

- added a way to change the service in place for methods that do not make sense
for ABS, and instead fallback on using vmpooler in those cases:
1. summary
2. modify
3. snapshot
4. revert
5. disk
For those methods in the class itself, raising a NoMethodError, in case it is used directly

- query now returns the queue info. For information on VMs, users should use --service vmpooler
- pretty_print_hosts (used in list, and delete scenarios) will now print the job_id
ABS information and indent each VM within and print it's metadata. Useful for
knowing the running time and extending it.

- added a new utility method to get the config for the vmpooler service, used for the fallback
- added a passthrough for the vmpooler token to use when running ABS. This enables
vmpooler to track the VMs used by each token (user). Also aligns the list between
both ABS and vmpooler. Fixes the bit-bar issue where the VMs do not appear when created
via ABS

lib/vmfloaty.rb

@@ -99,7 +99,12 @@ class Vmfloaty
 
         if options.active
           # list active vms
+          if service.type == "ABS"
+            # this is actually job_ids
+            running_vms = service.list_active_job_ids(verbose, service.url, service.user)
+          else
             running_vms = service.list_active(verbose)
+          end
           host = URI.parse(service.url).host
           if running_vms.empty?
             if options.json
@@ -126,7 +131,7 @@ class Vmfloaty
     command :query do |c|
       c.syntax = 'floaty query hostname [options]'
       c.summary = 'Get information about a given vm'
-      c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm.'
+      c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm. If using ABS, you can query a job_id'
       c.example 'Get information about a sample host', 'floaty query hostname --url http://vmpooler.example.com'
       c.option '--verbose', 'Enables verbose output'
       c.option '--service STRING', String, 'Configured pooler service name'
@@ -165,7 +170,12 @@ class Vmfloaty
           FloatyLogger.error 'ERROR: Provide a hostname or specify --all.'
           exit 1
         end
-        running_vms = modify_all ? service.list_active(verbose) : hostname.split(',')
+        running_vms =
+            if modify_all
+              service.list_active(verbose)
+            else
+              hostname.split(',')
+            end
 
         tags = options.tags ? JSON.parse(options.tags) : nil
         modify_hash = {
@@ -189,7 +199,7 @@ class Vmfloaty
           end
           if ok
             if modify_all
-              puts 'Successfully modified all VMs.'
+              puts "Successfully modified all #{running_vms.count} VMs."
             else
               puts "Successfully modified VM #{hostname}."
             end
@@ -225,7 +235,12 @@ class Vmfloaty
         successes = []
 
         if delete_all
+          if service.type == "ABS"
+            # this is actually job_ids
+            running_vms = service.list_active_job_ids(verbose, service.url, service.user)
+          else
             running_vms = service.list_active(verbose)
+          end
           if running_vms.empty?
             if options.json
               puts {}.to_json
@@ -274,7 +289,6 @@ class Vmfloaty
         end
 
         unless successes.empty?
-          FloatyLogger.info unless failures.empty?
           if options.json
             puts successes.to_json
           else

lib/vmfloaty/abs.rb

@@ -2,6 +2,7 @@
 
 require 'vmfloaty/errors'
 require 'vmfloaty/http'
+require 'vmfloaty/utils'
 require 'faraday'
 require 'json'
 
@@ -36,39 +37,59 @@ class ABS
   #       }
   #     }
   #
-
   @active_hostnames = {}
 
-  def self.list_active(verbose, url, _token, user)
+  def self.list_active_job_ids(verbose, url, user)
-    all_jobs = []
+    all_job_ids = []
     @active_hostnames = {}
-
     get_active_requests(verbose, url, user).each do |req_hash|
-      all_jobs.push(req_hash['request']['job']['id'])
+      @active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
-      @active_hostnames[req_hash['request']['job']['id']] = req_hash
+      all_job_ids.push(req_hash['request']['job']['id'])
     end
 
-    all_jobs
+    all_job_ids
+  end
+
+  def self.list_active(verbose, url, _token, user)
+    hosts = []
+    get_active_requests(verbose, url, user).each do |req_hash|
+      if req_hash.key?('allocated_resources')
+        req_hash['allocated_resources'].each do |onehost|
+          hosts.push(onehost['hostname'])
+        end
+      end
+    end
+
+    hosts
   end
 
   def self.get_active_requests(verbose, url, user)
     conn = Http.get_conn(verbose, url)
     res = conn.get 'status/queue'
+    if valid_json?(res.body)
       requests = JSON.parse(res.body)
+    else
+      FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
+    end
 
     ret_val = []
 
     requests.each do |req|
       next if req == 'null'
 
+      if valid_json?(req)
         req_hash = JSON.parse(req)
+      else
+        FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
+        next
+      end
 
       begin
         next unless user == req_hash['request']['job']['user']
 
         ret_val.push(req_hash)
       rescue NoMethodError
-        FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: "
+        FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
       end
     end
 
@@ -145,30 +166,43 @@ class ABS
     os_list = []
 
     res = conn.get 'status/platforms/vmpooler'
-
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('vmpooler_platforms')
         os_list << '*** VMPOOLER Pools ***'
         os_list += JSON.parse(res_body['vmpooler_platforms'])
+      end
+    end
 
     res = conn.get 'status/platforms/ondemand_vmpooler'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
-    unless res_body['ondemand_vmpooler_platforms'] == '[]'
+      if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
         os_list << ''
         os_list << '*** VMPOOLER ONDEMAND Pools ***'
         os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
       end
+    end
 
     res = conn.get 'status/platforms/nspooler'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('nspooler_platforms')
         os_list << ''
         os_list << '*** NSPOOLER Pools ***'
         os_list += JSON.parse(res_body['nspooler_platforms'])
+      end
+    end
 
     res = conn.get 'status/platforms/aws'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('aws_platforms')
         os_list << ''
         os_list << '*** AWS Pools ***'
         os_list += JSON.parse(res_body['aws_platforms'])
+      end
+    end
 
     os_list.delete 'ok'
 
@@ -197,7 +231,7 @@ class ABS
     conn.headers['X-AUTH-TOKEN'] = token if token
 
     saved_job_id = DateTime.now.strftime('%Q')
-
+    vmpooler_config = Utils.get_vmpooler_service_config
     req_obj = {
       :resources => os_types,
       :job       => {
@@ -206,6 +240,7 @@ class ABS
           :user => user,
         },
       },
+      :vm_token => vmpooler_config['token'] # request with this token, on behalf of this user
     }
 
     if options['priority']
@@ -264,12 +299,17 @@ class ABS
 
   def self.check_queue(conn, job_id, req_obj, verbose)
     queue_info_res = conn.get "status/queue/info/#{job_id}"
+    if valid_json?(queue_info_res.body)
       queue_info = JSON.parse(queue_info_res.body)
+    else
+      FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
+      return [nil, nil]
+    end
 
     res = conn.post 'request', req_obj.to_json
     validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
 
-    unless res.body.empty?
+    unless res.body.empty? || !valid_json?(res.body)
       res_body = JSON.parse(res.body)
       return queue_info['queue_place'], res_body
     end
@@ -277,7 +317,7 @@ class ABS
   end
 
   def self.snapshot(_verbose, _url, _hostname, _token)
-    FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
+    raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
   end
 
   def self.status(verbose, url)
@@ -289,20 +329,24 @@ class ABS
   end
 
   def self.summary(verbose, url)
-    conn = Http.get_conn(verbose, url)
+    raise NoMethodError, 'summary is not defined for ABS'
-
-    res = conn.get 'summary'
-    JSON.parse(res.body)
   end
 
-  def self.query(verbose, url, hostname)
+  def self.query(verbose, url, job_id)
-    return @active_hostnames if @active_hostnames
+    # return saved hostnames from the last time list_active was run
+    # preventing having to query the API again.
+    # This works as long as query is called after list_active
+    return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
 
-    FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
+    # If using the cli query job_id
     conn = Http.get_conn(verbose, url)
-
+    queue_info_res = conn.get "status/queue/info/#{job_id}"
-    res = conn.get "host/#{hostname}"
+    if valid_json?(queue_info_res.body)
-    JSON.parse(res.body)
+      queue_info = JSON.parse(queue_info_res.body)
+    else
+      FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
+    end
+    queue_info
   end
 
   def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@@ -333,4 +377,11 @@ class ABS
       raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
     end
   end
+
+  def self.valid_json?(json)
+    JSON.parse(json)
+    return true
+  rescue TypeError, JSON::ParserError => e
+    return false
+  end
 end

lib/vmfloaty/service.rb

@@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
 
 class Service
   attr_reader :config
+  attr_accessor :silent
 
   def initialize(options, config_hash = {})
     options ||= Commander::Command::Options.new
     @config = Utils.get_service_config config_hash, options
     @service_object = Utils.get_service_object @config['type']
+    @silent = false
   end
 
   def method_missing(method_name, *args, &block)
@@ -103,6 +105,7 @@ class Service
   end
 
   def modify(verbose, hostname, modify_hash)
+    maybe_use_vmpooler
     @service_object.modify verbose, url, hostname, token, modify_hash
   end
 
@@ -115,18 +118,35 @@ class Service
   end
 
   def summary(verbose)
+    maybe_use_vmpooler
     @service_object.summary verbose, url
   end
 
   def snapshot(verbose, hostname)
+    maybe_use_vmpooler
     @service_object.snapshot verbose, url, hostname, token
   end
 
   def revert(verbose, hostname, snapshot_sha)
+    maybe_use_vmpooler
     @service_object.revert verbose, url, hostname, token, snapshot_sha
   end
 
   def disk(verbose, hostname, disk)
+    maybe_use_vmpooler
     @service_object.disk(verbose, url, hostname, token, disk)
   end
+
+  # some methods do not exist for ABS, and if possible should target the Pooler service
+  def maybe_use_vmpooler
+    if @service_object.is_a?(ABS.class)
+      if !self.silent
+        FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using vmpooler config from ~/.vmfloaty.yml"
+        self.silent = true
+      end
+
+      @config = Utils.get_vmpooler_service_config
+      @service_object = Pooler
+    end
+  end
 end

lib/vmfloaty/utils.rb

@@ -3,6 +3,7 @@
 require 'vmfloaty/abs'
 require 'vmfloaty/nonstandard_pooler'
 require 'vmfloaty/pooler'
+require 'vmfloaty/conf'
 
 class Utils
   # TODO: Takes the json response body from an HTTP GET
@@ -78,21 +79,28 @@ class Utils
     os_types
   end
 
-  def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false)
+  def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0)
     fetched_data = self.get_host_data(verbose, service, hostnames)
     fetched_data.each do |hostname, host_data|
       case service.type
       when 'ABS'
         # For ABS, 'hostname' variable is the jobID
+        #
+        # Create a vmpooler service to query each hostname there so as to get the metadata too
+
+        vmpooler_service = service.clone
+        vmpooler_service.silent = true
+        vmpooler_service.maybe_use_vmpooler
+        puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>"
         host_data['allocated_resources'].each do |vm_name, _i|
-          puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>"
+          self.pretty_print_hosts(verbose, vmpooler_service, vm_name['hostname'].split('.')[0], print_to_stderr, indent+2)
         end
       when 'Pooler'
         tag_pairs = []
         tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
         duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
         metadata = [host_data['template'], duration, *tag_pairs]
-        puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})"
+        puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent)
       when 'NonstandardPooler'
         line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
         line += ", #{host_data['hours_left_on_reservation']}h remaining"
@@ -241,4 +249,26 @@ class Utils
 
     service_config
   end
+
+  # This method gets the vmpooler service configured in ~/.vmfloaty
+  def self.get_vmpooler_service_config
+    config = Conf.read_config
+    # The top-level url, user, and token values in the config file are treated as defaults
+    service_config = {
+        'url'   => config['url'],
+        'user'  => config['user'],
+        'token' => config['token'],
+        'type'  => 'vmpooler',
+    }
+
+    # at a minimum, the url needs to be configured
+    if config['services'] && config['services']['vmpooler'] && config['services']['vmpooler']['url']
+      # If the service is configured but some values are missing, use the top-level defaults to fill them in
+      service_config.merge! config['services']['vmpooler']
+    else
+      raise ArgumentError, "Could not find a configured service named 'vmpooler' in ~/.vmfloaty.yml use this format:\nservices:\n  vmpooler:\n    url: 'http://vmpooler.com'\n    user: 'superman'\n    token: 'kryptonite'"
+    end
+
+    service_config
+  end
 end