From 11d94cc3d2129ba15c5b1b94393a7bde623da4e2 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Fri, 12 Apr 2019 14:34:09 -0500 Subject: [PATCH 1/2] (QENG-7201) Vmpooler Status API QueueProcessor Optimization Before this change we used the API /status endpoint to get specific information on pools such as the number of ready VMs and the max. This commit creates two new endpoints to get to that information much quicker 1) poolstat?pool= takes a comma separated list of pools to return, and will provide the max, ready and alias values. 2) /totalrunning will calculate the total number of running VMs across all pools --- lib/vmpooler/api/v1.rb | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index b10cc7b..bfd30da 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -341,6 +341,71 @@ module Vmpooler JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) end + # request statistics for specific pools by passing parameter 'pool' + # with a coma separated list of pools we want to query ?pool=ABC,DEF + # returns the ready, max numbers and the aliases (if set) + get "#{api_prefix}/poolstat/?" do + content_type :json + + result = {} + + poolscopy = [] + + if params[:pool] + subpool = params[:pool].split(",") + poolscopy = pools.select do |p| + subpool.include?(p['name']) || (p['alias'] & subpool).any? + end + end + + result[:pools] = {} + + poolscopy.each do |pool| + result[:pools][pool['name']] = {} + + max = pool['size'] + aka = pool['alias'] + + result[:pools][pool['name']][:max] = max + + if aka + result[:pools][pool['name']][:alias] = aka + end + + end + + # using pipelined is much faster than querying each of the pools + res = backend.pipelined do + poolscopy.each do |pool| + backend.scard('vmpooler__ready__' + pool['name']) + end + end + + res.each_with_index { |ready, i| result[:pools][poolscopy[i]['name']][:ready] = ready } + + JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }]) + end + + # requests the total number of running VMs + get "#{api_prefix}/totalrunning/?" do + content_type :json + queue = { + running: 0, + } + + # using pipelined is much faster than querying each of the pools and adding them + # as we get the result. + res = backend.pipelined do + pools.each do |pool| + backend.scard('vmpooler__running__' + pool['name']) + end + end + + queue[:running] = res.inject(0){ |m, x| m+x } + + JSON.pretty_generate(queue) + end + get "#{api_prefix}/summary/?" do content_type :json From 714a9edf5e3700d358a7c286b6de37998eccf5ce Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Mon, 15 Apr 2019 13:09:53 -0500 Subject: [PATCH 2/2] (QENG-7201) Adding docs and tests --- docs/API.md | 46 +++++++++++++++++++++++ lib/vmpooler/api/v1.rb | 10 ++++- spec/integration/api/v1/status_spec.rb | 52 +++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/docs/API.md b/docs/API.md index 4ceaf79..7111b79 100644 --- a/docs/API.md +++ b/docs/API.md @@ -629,6 +629,52 @@ The valid sections are "boot", "clone" or "tag" eg. `vmpooler.example.com/api/v1 You can further drill-down the data by specifying the second level parameter to query eg `vmpooler.example.com/api/v1/summary/tag/created_by` +##### GET /poolstat?pool=FOO + +For parameter `pool`, containing a comma separated list of pool names to query, this endpoint returns +each of the pool's ready, max and alias information. It can be used to get a fast response for +the required pools instead of using the /status API endpoint + +Return codes +* 200 OK + +``` +$ curl https://vmpooler.example.com/api/v1/poolstat?pool=centos-6-x86_64 +``` +```json +{ + "pools": { + "centos-6-x86_64": { + "ready": 25, + "max": 25, + "alias": [ + "centos-6-64", + "centos-6-amd64" + ] + } + } +} +``` + +##### GET /totalrunning + +Fast endpoint to return the total number of VMs in a 'running' state + +Return codes +* 200 OK + +``` +$ curl https://vmpooler.example.com/api/v1/totalrunning +``` + +```json +{ + + "running": 362 + +} +``` + #### Managing pool configuration via API ##### GET /config diff --git a/lib/vmpooler/api/v1.rb b/lib/vmpooler/api/v1.rb index bfd30da..70aa7d1 100644 --- a/lib/vmpooler/api/v1.rb +++ b/lib/vmpooler/api/v1.rb @@ -354,7 +354,15 @@ module Vmpooler if params[:pool] subpool = params[:pool].split(",") poolscopy = pools.select do |p| - subpool.include?(p['name']) || (p['alias'] & subpool).any? + if subpool.include?(p['name']) + true + elsif !p['alias'].nil? + if p['alias'].is_a?(Array) + (p['alias'] & subpool).any? + elsif p['alias'].is_a?(String) + subpool.include?(p['alias']) + end + end end end diff --git a/spec/integration/api/v1/status_spec.rb b/spec/integration/api/v1/status_spec.rb index 553188b..ffd69aa 100644 --- a/spec/integration/api/v1/status_spec.rb +++ b/spec/integration/api/v1/status_spec.rb @@ -12,7 +12,7 @@ describe Vmpooler::API::V1 do Vmpooler::API end - describe '/status' do + describe 'status and metrics endpoints' do let(:prefix) { '/api/v1' } let(:config) { @@ -186,5 +186,55 @@ describe Vmpooler::API::V1 do expect(result["status"]).to_not be(nil) end end + + describe 'GET /poolstat' do + it 'returns empty list when pool is not set' do + get "#{prefix}/poolstat" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result == {}) + end + it 'returns empty list when pool is not found' do + get "#{prefix}/poolstat?pool=unknownpool" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result == {}) + end + it 'returns one pool when requesting one with alias' do + get "#{prefix}/poolstat?pool=pool1" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 1) + expect(result["pools"]["pool1"]["ready"]).to eq(0) + expect(result["pools"]["pool1"]["max"]).to eq(5) + expect(result["pools"]["pool1"]["alias"]).to eq(['poolone', 'poolun']) + end + it 'returns one pool when requesting one without alias' do + get "#{prefix}/poolstat?pool=pool2" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 1) + expect(result["pools"]["pool2"]["ready"]).to eq(0) + expect(result["pools"]["pool2"]["max"]).to eq(10) + expect(result["pools"]["pool2"]["alias"]).to be(nil) + end + it 'returns multiple pools when requesting csv' do + get "#{prefix}/poolstat?pool=pool1,pool2" + expect(last_response.header['Content-Type']).to eq('application/json') + result = JSON.parse(last_response.body) + expect(result["pools"].size == 2) + end + end + + describe 'GET /totalrunning' do + it 'returns the number of running VMs' do + get "#{prefix}/totalrunning" + expect(last_response.header['Content-Type']).to eq('application/json') + 5.times {|i| create_running_vm("pool1", "vm-#{i}") } + 5.times {|i| create_running_vm("pool3", "vm-#{i}") } + result = JSON.parse(last_response.body) + expect(result["running"] == 10) + end + end end end