(POOLER-134) Ship VM usage stats

This commit updates vmpooler to ship VM usage stats when a VM is destroyed. The stats are gathered for jobs based on user and pool name. If a jenkins build URL is present then this is broken down by user, instance, value stream, branch and project. Additionally, if present then the RMM_COMPONENT_TO_TEST_NAME will be listed after project. Without this change we do not collect stats on per VM usage and its correlation to users and pools.

CHANGELOG.md

@@ -17,6 +17,9 @@ git logs & PR history.
 - Remove a failed VM from the ready queue (POOLER-133)
 - Begin checking ready VMs to ensure alive after 1 minute by default
 
+### Added
+- Add capability to ship VM usage metrics (POOLER-134)
+
 # [0.2.2](https://github.com/puppetlabs/vmpooler/compare/0.2.1...0.2.2)
 
 ### Fixed

docs/configuration.md

@@ -156,6 +156,20 @@ When enabled in the global configuration then purging is enabled for all provide
 Expects a boolean value
 (optional; default: false)
 
+### USAGE\_STATS
+
+Enable shipping of VM usage stats
+When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize
+shipped metrics if there is a jenkins\_build\_url tag set for the VM.
+Without the jenkins\_build\_url tag set the metric will be sent as "usage.$user.$pool\_name".
+When the jenkins\_build\_url tag is set the metric will be sent with additional data. Here is an example
+based off of the following URL;
+https://jenkins.example.com/job/platform\_puppet-agent-extra\_puppet-agent-integration-suite\_pr/RMM\_COMPONENT\_TO\_TEST\_NAME=puppet,SLAVE\_LABEL=beaker,TEST\_TARGET=redhat7-64a/824/
+"usage.$user.$instance.$value\_stream.$branch.$project.$job\_name.$component\_to\_test.$pool\_name", which translates to
+"usage.$user.jenkins\_example\_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool\_name"
+Expects a boolean value
+(optional; default: false)
+
 ## API options <a name="API"></a>
 
 ### AUTH\_PROVIDER

lib/vmpooler.rb

@@ -66,6 +66,7 @@ module Vmpooler
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
     parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
 
     parsed_config[:redis] = parsed_config[:redis] || {}
     parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'

lib/vmpooler/pool_manager.rb

@@ -319,10 +319,66 @@ module Vmpooler
         finish = format('%.2f', Time.now - start)
         $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
         $metrics.timing("destroy.#{pool}", finish)
+        get_vm_usage_labels(vm)
       end
       dereference_mutex(vm)
     end
 
+    def get_vm_usage_labels(vm)
+      return unless $config[:config]['usage_stats']
+      checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
+      return if checkout.nil?
+      jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
+      user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
+      poolname = $redis.hget("vmpooler__vm__#{vm}", "template")
+
+      unless jenkins_build_url
+        $metrics.increment("usage.#{user}.#{poolname}")
+        return
+      end
+
+      url_parts = jenkins_build_url.split('/')[2..-1]
+      instance = url_parts[0].gsub('.', '_')
+      value_stream_parts = url_parts[2].split('_')
+      value_stream = value_stream_parts.shift
+      branch = value_stream_parts.pop
+      project = value_stream_parts.shift
+      job_name = value_stream_parts.join('_')
+      build_metadata_parts = url_parts[3]
+      component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
+
+      metric_parts = [
+        'usage',
+        user,
+        instance,
+        value_stream,
+        branch,
+        project,
+        job_name,
+        component_to_test,
+        poolname
+      ]
+
+      metric_parts = metric_parts.reject { |s| s.nil? }
+
+      $metrics.increment(metric_parts.join('.'))
+    rescue => err
+      logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{err}")
+    end
+
+    def component_to_test(match, labels_string)
+      return if labels_string.nil?
+      labels_string_parts = labels_string.split(',')
+      labels_string_parts.each do |part|
+        key, value = part.split('=')
+        next if value.nil?
+        if key == match
+          return value
+        end
+      end
+      return
+    end
+
     def purge_unused_vms_and_folders
       global_purge = $config[:config]['purge_unconfigured_folders']
       providers = $config[:providers].keys

spec/helpers.rb

@@ -45,10 +45,11 @@ def create_ready_vm(template, name, token = nil)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_running_vm(template, name, token = nil)
-  create_vm(name, token)
+def create_running_vm(template, name, token = nil, user = nil)
+  create_vm(name, token, nil, user)
   redis.sadd("vmpooler__running__#{template}", name)
-  redis.hset("vmpooler__vm__#{name}", "template", template)
+  redis.hset("vmpooler__vm__#{name}", 'template', template)
+  redis.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
 end
 
 def create_pending_vm(template, name, token = nil)
@@ -57,10 +58,11 @@ def create_pending_vm(template, name, token = nil)
   redis.hset("vmpooler__vm__#{name}", "template", template)
 end
 
-def create_vm(name, token = nil, redis_handle = nil)
+def create_vm(name, token = nil, redis_handle = nil, user = nil)
   redis_db = redis_handle ? redis_handle : redis
   redis_db.hset("vmpooler__vm__#{name}", 'checkout', Time.now)
   redis_db.hset("vmpooler__vm__#{name}", 'token:token', token) if token
+  redis_db.hset("vmpooler__vm__#{name}", 'token:user', user) if user
 end
 
 def create_completed_vm(name, pool, active = false, redis_handle = nil)
@@ -81,6 +83,11 @@ def create_migrating_vm(name, pool, redis_handle = nil)
   redis_db.sadd("vmpooler__migrating__#{pool}", name)
 end
 
+def create_tag(vm, tag_name, tag_value, redis_handle = nil)
+  redis_db = redis_handle ? redis-handle : redis
+  redis_db.hset("vmpooler__vm__#{vm}", "tag:#{tag_name}", tag_value)
+end
+
 def add_vm_to_migration_set(name, redis_handle = nil)
   redis_db = redis_handle ? redis_handle : redis
   redis_db.sadd('vmpooler__migration', name)

spec/unit/pool_manager_spec.rb

@@ -757,11 +757,18 @@ EOT
       end
 
       it 'should emit a timing metric' do
+        allow(subject).to receive(:get_vm_usage_labels)
         expect(metrics).to receive(:timing).with("destroy.#{pool}", String)
 
         subject._destroy_vm(vm,pool,provider)
       end
 
+      it 'should check usage labels' do
+        expect(subject).to receive(:get_vm_usage_labels).with(vm)
+
+        subject._destroy_vm(vm,pool,provider)
+      end
+
       it 'should dereference the mutex' do
         expect(subject).to receive(:dereference_mutex)
 
@@ -803,6 +810,171 @@ EOT
     end
   end
 
+  describe '#get_vm_usage_labels' do
+
+    let(:template) { 'pool1' }
+    let(:user) { 'vmpuser' }
+    let(:vm) { 'vm1' }
+
+    context 'when label evaluation is disabled' do
+      it 'should do nothing' do
+        subject.get_vm_usage_labels(vm)
+      end
+    end
+
+    context 'when label evaluation is enabled' do
+
+      before(:each) do
+        config[:config]['usage_stats'] = true
+      end
+
+      context 'when a VM has not been checked out' do
+        before(:each) do
+          create_ready_vm(template, vm)
+        end
+
+        it 'should return' do
+          expect(subject).to receive(:get_vm_usage_labels).and_return(nil)
+
+          subject.get_vm_usage_labels(vm)
+        end
+      end
+
+      context 'when a VM has been checked out' do
+
+        context 'without auth' do
+
+          before(:each) do
+            create_running_vm(template, vm)
+          end
+
+          it 'should emit a metric' do
+            expect(metrics).to receive(:increment).with("usage.unauthenticated.#{template}")
+
+            subject.get_vm_usage_labels(vm)
+          end
+        end
+
+        context 'with auth' do
+
+          before(:each) do
+            create_running_vm(template, vm, token, user)
+          end
+
+          it 'should emit a metric' do
+            expect(metrics).to receive(:increment).with("usage.#{user}.#{template}")
+
+            subject.get_vm_usage_labels(vm)
+          end
+
+          context 'with a jenkins_build_url label' do
+            let(:jenkins_build_url) { 'https://jenkins.example.com/job/enterprise_pe-acceptance-tests_integration-system_pe_full-agent-upgrade_weekend_2018.1.x/LAYOUT=centos6-64mcd-ubuntu1404-32f-64f,LEGACY_AGENT_VERSION=NONE,PLATFORM=NOTUSED,SCM_BRANCH=2018.1.x,UPGRADE_FROM=2018.1.0,UPGRADE_TO_VERSION=NONE,label=beaker/222/' }
+            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
+            let(:instance) { url_parts[0].gsub('.', '_') }
+            let(:value_stream_parts) { url_parts[2].split('_') }
+            let(:value_stream) { value_stream_parts.shift }
+            let(:branch) { value_stream_parts.pop }
+            let(:project) { value_stream_parts.shift }
+            let(:job_name) { value_stream_parts.join('_') }
+
+            before(:each) do
+              create_tag(vm, 'jenkins_build_url', jenkins_build_url)
+            end
+
+            it 'should emit a metric with information from the URL' do
+              expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{template}")
+
+              subject.get_vm_usage_labels(vm)
+            end
+          end
+
+          context 'with a jenkins_build_url that contains RMM_COMPONENT_TO_TEST_NAME' do
+            let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/' }
+            let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
+            let(:instance) { url_parts[0].gsub('.', '_') }
+            let(:value_stream_parts) { url_parts[2].split('_') }
+            let(:value_stream) { value_stream_parts.shift }
+            let(:branch) { value_stream_parts.pop }
+            let(:project) { value_stream_parts.shift }
+            let(:job_name) { value_stream_parts.join('_') }
+            let(:build_metadata) { url_parts[3] }
+            let(:build_component) { subject.component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata) }
+
+            before(:each) do
+              create_tag(vm, 'jenkins_build_url', jenkins_build_url)
+            end
+
+            it 'should emit a metric with information from the URL' do
+              expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{build_component}.#{template}")
+
+              subject.get_vm_usage_labels(vm)
+            end
+
+            context 'when there is no matrix job information' do
+
+              let(:jenkins_build_url) { 'https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/824/' }
+              let(:url_parts) { jenkins_build_url.split('/')[2..-1] }
+              let(:instance) { url_parts[0].gsub('.', '_') }
+              let(:value_stream_parts) { url_parts[2].split('_') }
+              let(:value_stream) { value_stream_parts.shift }
+              let(:branch) { value_stream_parts.pop }
+              let(:project) { value_stream_parts.shift }
+              let(:job_name) { value_stream_parts.join('_') }
+
+              before(:each) do
+                create_tag(vm, 'jenkins_build_url', jenkins_build_url)
+              end
+
+              it 'should emit a metric with information from the URL without a build_component' do
+                expect(metrics).to receive(:increment).with("usage.#{user}.#{instance}.#{value_stream}.#{branch}.#{project}.#{job_name}.#{template}")
+
+                subject.get_vm_usage_labels(vm)
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+
+  describe '#component_to_test' do
+    let(:matching_key) { 'LABEL_ONE' }
+    let(:matching_value) { 'test' }
+    let(:labels_string) { "#{matching_key}=#{matching_value},LABEL_TWO=test2,LABEL_THREE=test3" }
+    let(:nonmatrix_string) { 'test,stuff,and,things' }
+
+    context 'when string contains a matching key' do
+      it 'should print the corresponding value' do
+        expect(subject.component_to_test(matching_key, labels_string)).to eq(matching_value)
+      end
+
+      context 'when match contains no value' do
+        it 'should return nil' do
+          expect(subject.component_to_test(matching_key, matching_key)).to be nil
+        end
+      end
+    end
+
+    context 'when string contains no key value pairs' do
+      it 'should return' do
+        expect(subject.component_to_test(matching_key, nonmatrix_string)).to be nil
+      end
+    end
+
+    context 'when labels_string is a job number' do
+      it 'should return nil' do
+        expect(subject.component_to_test(matching_key, '25')).to be nil
+      end
+    end
+
+    context 'when labels_string is nil' do
+      it 'should return nil' do
+        expect(subject.component_to_test(matching_key, nil)).to be nil
+      end
+    end
+  end
+
   describe '#purge_unused_vms_and_folders' do
     let(:config) { YAML.load(<<-EOT
 ---

vmpooler.yaml.example

@@ -485,6 +485,19 @@
 #     Expects a hash value
 #     (optional)
 #
+#   - usage_stats
+#     Enable shipping of VM usage stats
+#     When enabled a metric is emitted when a machine is destroyed. Tags are inspected and used to organize
+#     shipped metrics if there is a jenkins_build_url tag set for the VM.
+#     Without the jenkins_build_url tag set the metric will be sent as "usage.$user.$pool_name".
+#     When the jenkins_build_url tag is set the metric will be sent with additional data. Here is an example
+#     based off of the following URL, and requested by the user ABS;
+#     https://jenkins.example.com/job/platform_puppet-agent-extra_puppet-agent-integration-suite_pr/RMM_COMPONENT_TO_TEST_NAME=puppet,SLAVE_LABEL=beaker,TEST_TARGET=redhat7-64a/824/
+#     "usage.$user.$instance.$value_stream.$branch.$project.$job_name.$component_to_test.$pool_name", which translates to
+#     "usage.$user.jenkins_example_com.platform.pr.puppet-agent-extra.puppet-agent-integration-suite.puppet.$pool_name"
+#     Expects a boolean value
+#     (optional; default: false)
+#
 # Example:
 
 :config: