Add distributed tracing (#399)

This change utilizes OpenTelemetry's automatic instrumentation to add
distributed tracing capabilities to VMPooler. This is a non-breaking
change as traces are processed in noop mode by default.

Gemfile

@@ -16,6 +16,13 @@ gem 'nokogiri', '~> 1.10'
 gem 'spicy-proton', '~> 2.1'
 gem 'concurrent-ruby', '~> 1.1'
 
+gem 'opentelemetry-api', '~> 0.6.0'
+gem 'opentelemetry-exporter-jaeger', '~> 0.6.0'
+gem 'opentelemetry-instrumentation-concurrent_ruby', '~> 0.6.0'
+gem 'opentelemetry-instrumentation-redis', '~> 0.6.0'
+gem 'opentelemetry-instrumentation-sinatra', '~> 0.6.0'
+gem 'opentelemetry-sdk', '~> 0.6.0'
+
 group :development do
   gem 'pry'
 end

Vagrantfile

@@ -1,8 +1,10 @@
 # vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
 Vagrant.configure("2") do |config|
   config.vm.box = "genebean/centos-7-rvm-multi"
-  config.vm.network "forwarded_port", guest: 4567, host: 4567
-  config.vm.network "forwarded_port", guest: 8080, host: 8080
+  config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose
+  config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose
+  config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose
+  config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose
   config.vm.provision "shell", inline: <<-SCRIPT
     mkdir /var/log/vmpooler
     chown vagrant:vagrant /var/log/vmpooler
@@ -11,9 +13,18 @@ Vagrant.configure("2") do |config|
     usermod -aG docker vagrant
     systemctl enable docker
     systemctl start docker
-    docker build -t vmpooler /vagrant
+    curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+    chmod +x /usr/local/bin/docker-compose
+    ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+    docker-compose --version
+    cd /vagrant
+    docker-compose -f docker/docker-compose.yml build
     docker images
-    echo 'To use the container with the dummy provider do this after "vagrant ssh":'
-    echo "docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler"
   SCRIPT
+
+  # config.vm.provider "virtualbox" do |v|
+  #   v.memory = 2048
+  #   v.cpus = 2
+  # end
+  
 end

bin/vmpooler

@@ -2,30 +2,39 @@
 # frozen_string_literal: true
 
 require 'vmpooler'
+require 'vmpooler/version'
 
 config = Vmpooler.config
+logger_file = config[:config]['logfile']
+prefix = config[:config]['prefix']
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
 redis_connection_pool_size = config[:redis]['connection_pool_size']
 redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
 redis_reconnect_attempts = config[:redis]['reconnect_attempts']
-logger_file = config[:config]['logfile']
+tracing_enabled = config[:tracing]['enabled']
+tracing_jaeger_host = config[:tracing]['jaeger_host']
 
 logger = Vmpooler::Logger.new logger_file
 metrics = Vmpooler::Metrics.init(logger, config)
 
+version = Vmpooler::VERSION
+
+startup_args = ARGV
+Vmpooler.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+
 torun_threads = []
 if ARGV.count == 0
   torun = %i[api manager]
 else
   torun = []
-  torun << :api if ARGV.include? 'api'
-  torun << :manager if ARGV.include? 'manager'
+  torun << :api if ARGV.include?('api')
+  torun << :manager if ARGV.include?('manager')
   exit(2) if torun.empty?
 end
 
-if torun.include? :api
+if torun.include?(:api)
   api = Thread.new do
     redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
     Vmpooler::API.execute(torun, config, redis, metrics, logger)
@@ -39,7 +48,7 @@ elsif metrics.respond_to?(:setup_prometheus_metrics)
   torun_threads << prometheus_only_api
 end
 
-if torun.include? :manager
+if torun.include?(:manager)
   manager = Thread.new do
     Vmpooler::PoolManager.new(
       config,

docker/Dockerfile_local

@@ -15,7 +15,16 @@ COPY ./ ./
 
 ENV RACK_ENV=production
 
-RUN gem install bundler && bundle install && gem build vmpooler.gemspec && gem install vmpooler*.gem && \
+RUN apt-get update -qq && \
+    apt-get install -y --no-install-recommends make && \
+    apt-get clean autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN gem install bundler && \
+    bundle install && \
+    gem build vmpooler.gemspec && \
+    gem install vmpooler*.gem && \
     chmod +x /usr/local/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["docker-entrypoint.sh"]

docker/docker-compose.yml

@@ -1,7 +1,7 @@
 # For local development run with a dummy provider
-version: '3.2'
+version: '3.8'
 services:
-  vmpooler:
+  vmpooler-api:
     build:
       context: ../
       dockerfile: docker/Dockerfile_local
@@ -10,7 +10,7 @@ services:
         source: ${PWD}/vmpooler.yaml
         target: /etc/vmpooler/vmpooler.yaml
     ports:
-      - "4567:4567"
+      - "8080:4567"
     networks:
       - redis-net
     environment:
@@ -18,7 +18,35 @@ services:
       - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
       - REDIS_SERVER=redislocal
       - LOGFILE=/dev/null
+      - JRUBY_OPTS=-Xinvokedynamic.yield=false
+      - VMPOOLER_TRACING_ENABLED=true
+      - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces
     image: vmpooler-local
+    command: api
+    depends_on:
+      - redislocal
+  vmpooler-manager:
+    build:
+      context: ../
+      dockerfile: docker/Dockerfile_local
+    volumes:
+      - type: bind
+        source: ${PWD}/vmpooler.yaml
+        target: /etc/vmpooler/vmpooler.yaml
+    ports:
+      - "8081:4567"
+    networks:
+      - redis-net
+    environment:
+      - VMPOOLER_DEBUG=true # for use of dummy auth
+      - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml
+      - REDIS_SERVER=redislocal
+      - LOGFILE=/dev/null
+      - JRUBY_OPTS=-Xinvokedynamic.yield=false
+      - VMPOOLER_TRACING_ENABLED=true
+      - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces
+    image: vmpooler-local
+    command: manager
     depends_on:
       - redislocal
   redislocal:
@@ -29,6 +57,17 @@ services:
       - "6379:6379"
     networks:
       - redis-net
+  jaeger-aio:
+    image: jaegertracing/all-in-one:1.18
+    ports:
+      - "14250:14250"
+      - "8082:16686"
+    networks:
+      - redis-net
+    user: '1001'
+    read_only: true
+    cap_drop:
+      - ALL
 
 networks:
   redis-net:

lib/vmpooler.rb

@@ -15,6 +15,14 @@ module Vmpooler
   require 'timeout'
   require 'yaml'
 
+  # Dependencies for tracing
+  require 'opentelemetry-api'
+  require 'opentelemetry/exporter/jaeger'
+  require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-redis'
+  require 'opentelemetry-instrumentation-sinatra'
+  require 'opentelemetry-sdk'
+
   %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
@@ -103,6 +111,10 @@ module Vmpooler
     parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
     parsed_config[:graphite]['port']   = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
 
+    parsed_config[:tracing]                = parsed_config[:tracing] || {}
+    parsed_config[:tracing]['enabled']     = ENV['VMPOOLER_TRACING_ENABLED'] || parsed_config[:tracing]['enabled'] || 'false'
+    parsed_config[:tracing]['jaeger_host'] = ENV['VMPOOLER_TRACING_JAEGER_HOST'] || parsed_config[:tracing]['jaeger_host'] || 'http://localhost:14268/api/traces'
+
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
     if parsed_config.key? :auth
       parsed_config[:auth]['provider']           = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
@@ -213,4 +225,42 @@ module Vmpooler
     parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/
     parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones']
   end
+
+  def self.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+    if startup_args.length == 1 && startup_args.include?('api')
+      service_name = 'vmpooler-api'
+    elsif startup_args.length == 1 && startup_args.include?('manager')
+      service_name = 'vmpooler-manager'
+    else
+      service_name = 'vmpooler'
+    end
+
+    service_name += "-#{prefix}" unless prefix.empty?
+
+    if tracing_enabled.eql?('false')
+      puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        exporter: OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
+      )
+    else
+      puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        exporter: OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
+      )
+    end
+
+    OpenTelemetry::SDK.configure do |c|
+      c.use 'OpenTelemetry::Instrumentation::Sinatra'
+      c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::Redis'
+
+      c.add_span_processor(span_processor)
+      c.resource = OpenTelemetry::SDK::Resources::Resource.create(
+        {
+          OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:name] => service_name,
+          OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:version] => version
+        }
+      )
+    end
+  end
 end