Merge pull request #326 from puppetlabs/optimize-status-api

(maint) Optimize the status api using redis pipeline

lib/vmpooler/api/dashboard.rb

@@ -1,11 +1,20 @@
 module Vmpooler
   class API
     class Dashboard < Sinatra::Base
+
+      helpers do
+        include Vmpooler::API::Helpers
+      end
+
       # handle to the App's configuration information
       def config
         @config ||= Vmpooler::API.settings.config
       end
 
+      def backend
+        Vmpooler::API.settings.redis
+      end
+
       # configuration setting for server hosting graph URLs to view
       def graph_server
         return @graph_server if @graph_server
@@ -53,10 +62,13 @@ module Vmpooler
         content_type :json
         result = {}
 
-        Vmpooler::API.settings.config[:pools].each do |pool|
+        pools = Vmpooler::API.settings.config[:pools]
+        ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
+        pools.each do |pool|
           result[pool['name']] ||= {}
           result[pool['name']]['size'] = pool['size']
-          result[pool['name']]['ready'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])
+          result[pool['name']]['ready'] = ready_hash[pool['name']]
         end
 
         if params[:history]
@@ -91,9 +103,9 @@ module Vmpooler
             rescue
             end
           else
-            Vmpooler::API.settings.config[:pools].each do |pool|
+            pools.each do |pool|
               result[pool['name']] ||= {}
-              result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])]
+              result[pool['name']]['history'] = [ready_hash[pool['name']]]
             end
           end
         end
@@ -104,8 +116,11 @@ module Vmpooler
         content_type :json
         result = {}
 
-        Vmpooler::API.settings.config[:pools].each do |pool|
+        pools = Vmpooler::API.settings.config[:pools]
-          running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
+        running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+
+        pools.each do |pool|
+          running = running_hash[pool['name']]
           pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
           result[pool['major']] ||= {}
           result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i

lib/vmpooler/api/helpers.rb

@@ -151,6 +151,53 @@ module Vmpooler
         backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
       end
 
+      # Takes the pools and a key to run scard on
+      # returns an integer for the total count
+      def get_total_across_pools_redis_scard(pools, key, backend)
+        # using pipelined is much faster than querying each of the pools and adding them
+        # as we get the result.
+        res = backend.pipelined do
+          pools.each do |pool|
+            backend.scard(key + pool['name'])
+          end
+        end
+        res.inject(0){ |m, x| m+x }.to_i
+      end
+
+      # Takes the pools and a key to run scard on
+      # returns a hash with each pool name as key and the value being the count as integer
+      def get_list_across_pools_redis_scard(pools, key, backend)
+        # using pipelined is much faster than querying each of the pools and adding them
+        # as we get the result.
+        temp_hash = {}
+        res = backend.pipelined do
+          pools.each do |pool|
+            backend.scard(key + pool['name'])
+          end
+        end
+        pools.each_with_index do |pool, i|
+          temp_hash[pool['name']] = res[i].to_i
+        end
+        temp_hash
+      end
+
+      # Takes the pools and a key to run hget on
+      # returns a hash with each pool name as key and the value as string
+      def get_list_across_pools_redis_hget(pools, key, backend)
+        # using pipelined is much faster than querying each of the pools and adding them
+        # as we get the result.
+        temp_hash = {}
+        res = backend.pipelined do
+          pools.each do |pool|
+            backend.hget(key, pool['name'])
+          end
+        end
+        pools.each_with_index do |pool, i|
+          temp_hash[pool['name']] = res[i].to_s
+        end
+        temp_hash
+      end
+
       def get_capacity_metrics(pools, backend)
         capacity = {
             current: 0,
@@ -159,12 +206,11 @@ module Vmpooler
         }
 
         pools.each do |pool|
-          pool['capacity'] = backend.scard('vmpooler__ready__' + pool['name']).to_i
-
-          capacity[:current] += pool['capacity']
           capacity[:total]   += pool['size'].to_i
         end
 
+        capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+
         if capacity[:total] > 0
           capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1)
         end
@@ -183,12 +229,10 @@ module Vmpooler
             total:     0
         }
 
-        pools.each do |pool|
+        queue[:pending]   = get_total_across_pools_redis_scard(pools,'vmpooler__pending__', backend)
-          queue[:pending]   += backend.scard('vmpooler__pending__' + pool['name']).to_i
+        queue[:ready]     = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
-          queue[:ready]     += backend.scard('vmpooler__ready__' + pool['name']).to_i
+        queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-          queue[:running]   += backend.scard('vmpooler__running__' + pool['name']).to_i
+        queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
-          queue[:completed] += backend.scard('vmpooler__completed__' + pool['name']).to_i
-        end
 
         queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
         queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i

lib/vmpooler/api/v1.rb

@@ -305,13 +305,18 @@ module Vmpooler
 
       # Check for empty pools
       result[:pools] = {} unless views and not views.include?("pools")
+      ready_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
+      running_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
+      pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
+      lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
+
       pools.each do |pool|
         # REMIND: move this out of the API and into the back-end
-        ready    = backend.scard('vmpooler__ready__' + pool['name']).to_i
+        ready    = ready_hash[pool['name']]
-        running  = backend.scard('vmpooler__running__' + pool['name']).to_i
+        running  = running_hash[pool['name']]
-        pending  = backend.scard('vmpooler__pending__' + pool['name']).to_i
+        pending  = pending_hash[pool['name']]
         max      = pool['size']
-        lastBoot = backend.hget('vmpooler__lastboot',pool['name']).to_s
+        lastBoot = lastBoot_hash[pool['name']]
         aka      = pool['alias']
 
         result[:pools][pool['name']] = {
@@ -382,14 +387,9 @@ module Vmpooler
 
       end
 
-      # using pipelined is much faster than querying each of the pools
+      ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
-      res = backend.pipelined do
-        poolscopy.each do |pool|
-          backend.scard('vmpooler__ready__' + pool['name'])
-        end
-      end
 
-      res.each_with_index { |ready, i| result[:pools][poolscopy[i]['name']][:ready] = ready }
+      ready_hash.each { |k, v| result[:pools][k][:ready] = v }
 
       JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
     end
@@ -401,15 +401,7 @@ module Vmpooler
           running:   0,
       }
 
-      # using pipelined is much faster than querying each of the pools and adding them
+      queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
-      # as we get the result.
-      res = backend.pipelined do
-        pools.each do |pool|
-          backend.scard('vmpooler__running__' + pool['name'])
-        end
-      end
-
-      queue[:running] = res.inject(0){ |m, x| m+x }
 
       JSON.pretty_generate(queue)
     end

spec/unit/api/helpers_spec.rb

@@ -68,6 +68,7 @@ describe Vmpooler::API::Helpers do
 
   describe '#get_capacity_metrics' do
     let(:redis) { double('redis') }
+    let(:backend) { double('backend') }
 
     it 'adds up pools correctly' do
       pools = [
@@ -75,8 +76,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 5}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 1
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 2, total: 10, percent: 20.0})
     end
@@ -87,8 +87,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 5}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,0]
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 0
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 1, total: 10, percent: 10.0})
     end
@@ -99,13 +98,13 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2', 'size' => 0}
       ]
 
-      allow(redis).to receive(:scard).with('vmpooler__ready__p1').and_return 1
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,0]
-      allow(redis).to receive(:scard).with('vmpooler__ready__p2').and_return 0
 
       expect(subject.get_capacity_metrics(pools, redis)).to eq({current: 1, total: 5, percent: 20.0})
     end
 
     it 'handles empty pool array' do
+      allow(redis).to receive(:pipelined).with(no_args).and_return []
       expect(subject.get_capacity_metrics([], redis)).to eq({current: 0, total: 0, percent: 0})
     end
   end
@@ -114,7 +113,7 @@ describe Vmpooler::API::Helpers do
     let(:redis) { double('redis') }
 
     it 'handles empty pool array' do
-      allow(redis).to receive(:scard).and_return 0
+      allow(redis).to receive(:pipelined).with(no_args).and_return [0]
       allow(redis).to receive(:get).and_return 0
 
       expect(subject.get_queue_metrics([], redis)).to eq({pending: 0, cloning: 0, booting: 0, ready: 0, running: 0, completed: 0, total: 0})
@@ -126,11 +125,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      pools.each do |p|
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-        %w(pending ready running completed).each do |action|
-          allow(redis).to receive(:scard).with('vmpooler__' + action + '__' + p['name']).and_return 1
-        end
-      end
       allow(redis).to receive(:get).and_return 1
 
       expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 1, booting: 1, ready: 2, running: 2, completed: 2, total: 8})
@@ -142,11 +137,7 @@ describe Vmpooler::API::Helpers do
           {'name' => 'p2'}
       ]
 
-      pools.each do |p|
+      allow(redis).to receive(:pipelined).with(no_args).and_return [1,1]
-        %w(pending ready running completed).each do |action|
-          allow(redis).to receive(:scard).with('vmpooler__' + action + '__' + p['name']).and_return 1
-        end
-      end
       allow(redis).to receive(:get).and_return 5
 
       expect(subject.get_queue_metrics(pools, redis)).to eq({pending: 2, cloning: 5, booting: 0, ready: 2, running: 2, completed: 2, total: 8})