Merge pull request #94 from puppetlabs/fix-abs-vmpooler

ABS enables fallback to vmpooler for some scenarios

README.md

@@ -105,12 +105,20 @@ Now vmfloaty will use those config files if no flag was specified.
 
 #### Default to Puppet's ABS instead of vmpooler
 
+When the --service is not specified on the command line, the first one is selected, so put ABS first.
+Also provide a "vmpooler" service that ABS can use as fallback for operations targeting vmpooler directly
 ```yaml
 # file at ~/.vmfloaty.yml
-url: 'https://abs.example.net'
+services:
-user: 'brian'
+  abs:
-token: 'tokenstring'
+    url: 'https://abs/api/v2'
     type: 'abs'
+    user: 'samuel'
+    token: 'foo'
+  vmpooler:
+    url: 'http://vmpooler'
+    user: 'samuel'
+    token: 'bar'
 ```
 
 #### Configuring multiple services

lib/vmfloaty.rb

@@ -99,7 +99,12 @@ class Vmfloaty
 
         if options.active
           # list active vms
+          if service.type == "ABS"
+            # this is actually job_ids
+            running_vms = service.list_active_job_ids(verbose, service.url, service.user)
+          else
             running_vms = service.list_active(verbose)
+          end
           host = URI.parse(service.url).host
           if running_vms.empty?
             if options.json
@@ -126,7 +131,7 @@ class Vmfloaty
     command :query do |c|
       c.syntax = 'floaty query hostname [options]'
       c.summary = 'Get information about a given vm'
-      c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm.'
+      c.description = 'Given a hostname from the pooler service, vmfloaty with query the service to get various details about the vm. If using ABS, you can query a job_id'
       c.example 'Get information about a sample host', 'floaty query hostname --url http://vmpooler.example.com'
       c.option '--verbose', 'Enables verbose output'
       c.option '--service STRING', String, 'Configured pooler service name'
@@ -165,7 +170,12 @@ class Vmfloaty
           FloatyLogger.error 'ERROR: Provide a hostname or specify --all.'
           exit 1
         end
-        running_vms = modify_all ? service.list_active(verbose) : hostname.split(',')
+        running_vms =
+            if modify_all
+              service.list_active(verbose)
+            else
+              hostname.split(',')
+            end
 
         tags = options.tags ? JSON.parse(options.tags) : nil
         modify_hash = {
@@ -189,7 +199,7 @@ class Vmfloaty
           end
           if ok
             if modify_all
-              puts 'Successfully modified all VMs.'
+              puts "Successfully modified all #{running_vms.count} VMs."
             else
               puts "Successfully modified VM #{hostname}."
             end
@@ -225,7 +235,12 @@ class Vmfloaty
         successes = []
 
         if delete_all
+          if service.type == "ABS"
+            # this is actually job_ids
+            running_vms = service.list_active_job_ids(verbose, service.url, service.user)
+          else
             running_vms = service.list_active(verbose)
+          end
           if running_vms.empty?
             if options.json
               puts {}.to_json
@@ -274,7 +289,6 @@ class Vmfloaty
         end
 
         unless successes.empty?
-          FloatyLogger.info unless failures.empty?
           if options.json
             puts successes.to_json
           else

lib/vmfloaty/abs.rb

@@ -2,6 +2,7 @@
 
 require 'vmfloaty/errors'
 require 'vmfloaty/http'
+require 'vmfloaty/utils'
 require 'faraday'
 require 'json'
 
@@ -36,39 +37,59 @@ class ABS
   #       }
   #     }
   #
-
   @active_hostnames = {}
 
-  def self.list_active(verbose, url, _token, user)
+  def self.list_active_job_ids(verbose, url, user)
-    all_jobs = []
+    all_job_ids = []
     @active_hostnames = {}
-
     get_active_requests(verbose, url, user).each do |req_hash|
-      all_jobs.push(req_hash['request']['job']['id'])
+      @active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
-      @active_hostnames[req_hash['request']['job']['id']] = req_hash
+      all_job_ids.push(req_hash['request']['job']['id'])
     end
 
-    all_jobs
+    all_job_ids
+  end
+
+  def self.list_active(verbose, url, _token, user)
+    hosts = []
+    get_active_requests(verbose, url, user).each do |req_hash|
+      if req_hash.key?('allocated_resources')
+        req_hash['allocated_resources'].each do |onehost|
+          hosts.push(onehost['hostname'])
+        end
+      end
+    end
+
+    hosts
   end
 
   def self.get_active_requests(verbose, url, user)
     conn = Http.get_conn(verbose, url)
     res = conn.get 'status/queue'
+    if valid_json?(res.body)
       requests = JSON.parse(res.body)
+    else
+      FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
+    end
 
     ret_val = []
 
     requests.each do |req|
       next if req == 'null'
 
+      if valid_json?(req)
         req_hash = JSON.parse(req)
+      else
+        FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
+        next
+      end
 
       begin
         next unless user == req_hash['request']['job']['user']
 
         ret_val.push(req_hash)
       rescue NoMethodError
-        FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: "
+        FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
       end
     end
 
@@ -145,30 +166,43 @@ class ABS
     os_list = []
 
     res = conn.get 'status/platforms/vmpooler'
-
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('vmpooler_platforms')
         os_list << '*** VMPOOLER Pools ***'
         os_list += JSON.parse(res_body['vmpooler_platforms'])
+      end
+    end
 
     res = conn.get 'status/platforms/ondemand_vmpooler'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
-    unless res_body['ondemand_vmpooler_platforms'] == '[]'
+      if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
         os_list << ''
         os_list << '*** VMPOOLER ONDEMAND Pools ***'
         os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
       end
+    end
 
     res = conn.get 'status/platforms/nspooler'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('nspooler_platforms')
         os_list << ''
         os_list << '*** NSPOOLER Pools ***'
         os_list += JSON.parse(res_body['nspooler_platforms'])
+      end
+    end
 
     res = conn.get 'status/platforms/aws'
+    if valid_json?(res.body)
       res_body = JSON.parse(res.body)
+      if res_body.key?('aws_platforms')
         os_list << ''
         os_list << '*** AWS Pools ***'
         os_list += JSON.parse(res_body['aws_platforms'])
+      end
+    end
 
     os_list.delete 'ok'
 
@@ -197,7 +231,7 @@ class ABS
     conn.headers['X-AUTH-TOKEN'] = token if token
 
     saved_job_id = DateTime.now.strftime('%Q')
-
+    vmpooler_config = Utils.get_vmpooler_service_config
     req_obj = {
       :resources => os_types,
       :job       => {
@@ -206,6 +240,7 @@ class ABS
           :user => user,
         },
       },
+      :vm_token => vmpooler_config['token'] # request with this token, on behalf of this user
     }
 
     if options['priority']
@@ -264,12 +299,17 @@ class ABS
 
   def self.check_queue(conn, job_id, req_obj, verbose)
     queue_info_res = conn.get "status/queue/info/#{job_id}"
+    if valid_json?(queue_info_res.body)
       queue_info = JSON.parse(queue_info_res.body)
+    else
+      FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
+      return [nil, nil]
+    end
 
     res = conn.post 'request', req_obj.to_json
     validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
 
-    unless res.body.empty?
+    unless res.body.empty? || !valid_json?(res.body)
       res_body = JSON.parse(res.body)
       return queue_info['queue_place'], res_body
     end
@@ -277,7 +317,7 @@ class ABS
   end
 
   def self.snapshot(_verbose, _url, _hostname, _token)
-    FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
+    raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
   end
 
   def self.status(verbose, url)
@@ -289,20 +329,24 @@ class ABS
   end
 
   def self.summary(verbose, url)
-    conn = Http.get_conn(verbose, url)
+    raise NoMethodError, 'summary is not defined for ABS'
-
-    res = conn.get 'summary'
-    JSON.parse(res.body)
   end
 
-  def self.query(verbose, url, hostname)
+  def self.query(verbose, url, job_id)
-    return @active_hostnames if @active_hostnames
+    # return saved hostnames from the last time list_active was run
+    # preventing having to query the API again.
+    # This works as long as query is called after list_active
+    return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
 
-    FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
+    # If using the cli query job_id
     conn = Http.get_conn(verbose, url)
-
+    queue_info_res = conn.get "status/queue/info/#{job_id}"
-    res = conn.get "host/#{hostname}"
+    if valid_json?(queue_info_res.body)
-    JSON.parse(res.body)
+      queue_info = JSON.parse(queue_info_res.body)
+    else
+      FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
+    end
+    queue_info
   end
 
   def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@@ -333,4 +377,11 @@ class ABS
       raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
     end
   end
+
+  def self.valid_json?(json)
+    JSON.parse(json)
+    return true
+  rescue TypeError, JSON::ParserError => e
+    return false
+  end
 end

lib/vmfloaty/service.rb

@@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
 
 class Service
   attr_reader :config
+  attr_accessor :silent
 
   def initialize(options, config_hash = {})
     options ||= Commander::Command::Options.new
     @config = Utils.get_service_config config_hash, options
     @service_object = Utils.get_service_object @config['type']
+    @silent = false
   end
 
   def method_missing(method_name, *args, &block)
@@ -103,6 +105,7 @@ class Service
   end
 
   def modify(verbose, hostname, modify_hash)
+    maybe_use_vmpooler
     @service_object.modify verbose, url, hostname, token, modify_hash
   end
 
@@ -115,18 +118,35 @@ class Service
   end
 
   def summary(verbose)
+    maybe_use_vmpooler
     @service_object.summary verbose, url
   end
 
   def snapshot(verbose, hostname)
+    maybe_use_vmpooler
     @service_object.snapshot verbose, url, hostname, token
   end
 
   def revert(verbose, hostname, snapshot_sha)
+    maybe_use_vmpooler
     @service_object.revert verbose, url, hostname, token, snapshot_sha
   end
 
   def disk(verbose, hostname, disk)
+    maybe_use_vmpooler
     @service_object.disk(verbose, url, hostname, token, disk)
   end
+
+  # some methods do not exist for ABS, and if possible should target the Pooler service
+  def maybe_use_vmpooler
+    if @service_object.is_a?(ABS.class)
+      if !self.silent
+        FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using vmpooler config from ~/.vmfloaty.yml"
+        self.silent = true
+      end
+
+      @config = Utils.get_vmpooler_service_config
+      @service_object = Pooler
+    end
+  end
 end

lib/vmfloaty/utils.rb

@@ -3,6 +3,7 @@
 require 'vmfloaty/abs'
 require 'vmfloaty/nonstandard_pooler'
 require 'vmfloaty/pooler'
+require 'vmfloaty/conf'
 
 class Utils
   # TODO: Takes the json response body from an HTTP GET
@@ -78,21 +79,28 @@ class Utils
     os_types
   end
 
-  def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false)
+  def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0)
     fetched_data = self.get_host_data(verbose, service, hostnames)
     fetched_data.each do |hostname, host_data|
       case service.type
       when 'ABS'
         # For ABS, 'hostname' variable is the jobID
+        #
+        # Create a vmpooler service to query each hostname there so as to get the metadata too
+
+        vmpooler_service = service.clone
+        vmpooler_service.silent = true
+        vmpooler_service.maybe_use_vmpooler
+        puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>"
         host_data['allocated_resources'].each do |vm_name, _i|
-          puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>"
+          self.pretty_print_hosts(verbose, vmpooler_service, vm_name['hostname'].split('.')[0], print_to_stderr, indent+2)
         end
       when 'Pooler'
         tag_pairs = []
         tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
         duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
         metadata = [host_data['template'], duration, *tag_pairs]
-        puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})"
+        puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent)
       when 'NonstandardPooler'
         line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
         line += ", #{host_data['hours_left_on_reservation']}h remaining"
@@ -241,4 +249,26 @@ class Utils
 
     service_config
   end
+
+  # This method gets the vmpooler service configured in ~/.vmfloaty
+  def self.get_vmpooler_service_config
+    config = Conf.read_config
+    # The top-level url, user, and token values in the config file are treated as defaults
+    service_config = {
+        'url'   => config['url'],
+        'user'  => config['user'],
+        'token' => config['token'],
+        'type'  => 'vmpooler',
+    }
+
+    # at a minimum, the url needs to be configured
+    if config['services'] && config['services']['vmpooler'] && config['services']['vmpooler']['url']
+      # If the service is configured but some values are missing, use the top-level defaults to fill them in
+      service_config.merge! config['services']['vmpooler']
+    else
+      raise ArgumentError, "Could not find a configured service named 'vmpooler' in ~/.vmfloaty.yml use this format:\nservices:\n  vmpooler:\n    url: 'http://vmpooler.com'\n    user: 'superman'\n    token: 'kryptonite'"
+    end
+
+    service_config
+  end
 end