From bdc4314b2749d853165bf955323349a3c4522869 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Fri, 11 Sep 2020 18:31:12 -0400 Subject: [PATCH] Add distributed tracing This change utilizes OpenTelemetry's automatic instrumentation to add distributed tracing capabilities to VMPooler. This is a non-breaking change as traces are processed in noop mode by default. --- Gemfile | 7 ++++++ Vagrantfile | 21 ++++++++++++---- bin/vmpooler | 19 +++++++++++---- docker/Dockerfile_local | 13 ++++++++-- docker/docker-compose.yml | 45 ++++++++++++++++++++++++++++++++--- lib/vmpooler.rb | 50 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 15 deletions(-) diff --git a/Gemfile b/Gemfile index 26eef16..f5c81be 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,13 @@ gem 'nokogiri', '~> 1.10' gem 'spicy-proton', '~> 2.1' gem 'concurrent-ruby', '~> 1.1' +gem 'opentelemetry-api', '~> 0.6.0' +gem 'opentelemetry-exporter-jaeger', '~> 0.6.0' +gem 'opentelemetry-instrumentation-concurrent_ruby', '~> 0.6.0' +gem 'opentelemetry-instrumentation-redis', '~> 0.6.0' +gem 'opentelemetry-instrumentation-sinatra', '~> 0.6.0' +gem 'opentelemetry-sdk', '~> 0.6.0' + group :development do gem 'pry' end diff --git a/Vagrantfile b/Vagrantfile index e0a9e66..aa667a6 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,8 +1,10 @@ # vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby Vagrant.configure("2") do |config| config.vm.box = "genebean/centos-7-rvm-multi" - config.vm.network "forwarded_port", guest: 4567, host: 4567 - config.vm.network "forwarded_port", guest: 8080, host: 8080 + config.vm.network "forwarded_port", guest: 4567, host: 4567 # for when not running docker-compose + config.vm.network "forwarded_port", guest: 8080, host: 8080 # VMPooler api in docker-compose + config.vm.network "forwarded_port", guest: 8081, host: 8081 # VMPooler manager in docker-compose + config.vm.network "forwarded_port", guest: 8082, host: 8082 # Jaeger in docker-compose config.vm.provision "shell", inline: <<-SCRIPT mkdir /var/log/vmpooler chown vagrant:vagrant /var/log/vmpooler @@ -11,9 +13,18 @@ Vagrant.configure("2") do |config| usermod -aG docker vagrant systemctl enable docker systemctl start docker - docker build -t vmpooler /vagrant + curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + docker-compose --version + cd /vagrant + docker-compose -f docker/docker-compose.yml build docker images - echo 'To use the container with the dummy provider do this after "vagrant ssh":' - echo "docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler" SCRIPT + + # config.vm.provider "virtualbox" do |v| + # v.memory = 2048 + # v.cpus = 2 + # end + end diff --git a/bin/vmpooler b/bin/vmpooler index 4d8af45..3483349 100755 --- a/bin/vmpooler +++ b/bin/vmpooler @@ -2,30 +2,39 @@ # frozen_string_literal: true require 'vmpooler' +require 'vmpooler/version' config = Vmpooler.config +logger_file = config[:config]['logfile'] +prefix = config[:config]['prefix'] redis_host = config[:redis]['server'] redis_port = config[:redis]['port'] redis_password = config[:redis]['password'] redis_connection_pool_size = config[:redis]['connection_pool_size'] redis_connection_pool_timeout = config[:redis]['connection_pool_timeout'] redis_reconnect_attempts = config[:redis]['reconnect_attempts'] -logger_file = config[:config]['logfile'] +tracing_enabled = config[:tracing]['enabled'] +tracing_jaeger_host = config[:tracing]['jaeger_host'] logger = Vmpooler::Logger.new logger_file metrics = Vmpooler::Metrics.init(logger, config) +version = Vmpooler::VERSION + +startup_args = ARGV +Vmpooler.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version) + torun_threads = [] if ARGV.count == 0 torun = %i[api manager] else torun = [] - torun << :api if ARGV.include? 'api' - torun << :manager if ARGV.include? 'manager' + torun << :api if ARGV.include?('api') + torun << :manager if ARGV.include?('manager') exit(2) if torun.empty? end -if torun.include? :api +if torun.include?(:api) api = Thread.new do redis = Vmpooler.new_redis(redis_host, redis_port, redis_password) Vmpooler::API.execute(torun, config, redis, metrics, logger) @@ -39,7 +48,7 @@ elsif metrics.respond_to?(:setup_prometheus_metrics) torun_threads << prometheus_only_api end -if torun.include? :manager +if torun.include?(:manager) manager = Thread.new do Vmpooler::PoolManager.new( config, diff --git a/docker/Dockerfile_local b/docker/Dockerfile_local index 7a47516..c1b3f4e 100644 --- a/docker/Dockerfile_local +++ b/docker/Dockerfile_local @@ -15,7 +15,16 @@ COPY ./ ./ ENV RACK_ENV=production -RUN gem install bundler && bundle install && gem build vmpooler.gemspec && gem install vmpooler*.gem && \ - chmod +x /usr/local/bin/docker-entrypoint.sh +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends make && \ + apt-get clean autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +RUN gem install bundler && \ + bundle install && \ + gem build vmpooler.gemspec && \ + gem install vmpooler*.gem && \ + chmod +x /usr/local/bin/docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dd00a99..21b179d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,7 +1,7 @@ # For local development run with a dummy provider -version: '3.2' +version: '3.8' services: - vmpooler: + vmpooler-api: build: context: ../ dockerfile: docker/Dockerfile_local @@ -10,7 +10,7 @@ services: source: ${PWD}/vmpooler.yaml target: /etc/vmpooler/vmpooler.yaml ports: - - "4567:4567" + - "8080:4567" networks: - redis-net environment: @@ -18,7 +18,35 @@ services: - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - REDIS_SERVER=redislocal - LOGFILE=/dev/null + - JRUBY_OPTS=-Xinvokedynamic.yield=false + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces image: vmpooler-local + command: api + depends_on: + - redislocal + vmpooler-manager: + build: + context: ../ + dockerfile: docker/Dockerfile_local + volumes: + - type: bind + source: ${PWD}/vmpooler.yaml + target: /etc/vmpooler/vmpooler.yaml + ports: + - "8081:4567" + networks: + - redis-net + environment: + - VMPOOLER_DEBUG=true # for use of dummy auth + - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml + - REDIS_SERVER=redislocal + - LOGFILE=/dev/null + - JRUBY_OPTS=-Xinvokedynamic.yield=false + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces + image: vmpooler-local + command: manager depends_on: - redislocal redislocal: @@ -29,6 +57,17 @@ services: - "6379:6379" networks: - redis-net + jaeger-aio: + image: jaegertracing/all-in-one:1.18 + ports: + - "14250:14250" + - "8082:16686" + networks: + - redis-net + user: '1001' + read_only: true + cap_drop: + - ALL networks: redis-net: diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb index d6e5522..7f5ddf4 100644 --- a/lib/vmpooler.rb +++ b/lib/vmpooler.rb @@ -15,6 +15,14 @@ module Vmpooler require 'timeout' require 'yaml' + # Dependencies for tracing + require 'opentelemetry-api' + require 'opentelemetry/exporter/jaeger' + require 'opentelemetry-instrumentation-concurrent_ruby' + require 'opentelemetry-instrumentation-redis' + require 'opentelemetry-instrumentation-sinatra' + require 'opentelemetry-sdk' + %w[api metrics logger pool_manager generic_connection_pool].each do |lib| require "vmpooler/#{lib}" end @@ -103,6 +111,10 @@ module Vmpooler parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX'] parsed_config[:graphite]['port'] = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT'] + parsed_config[:tracing] = parsed_config[:tracing] || {} + parsed_config[:tracing]['enabled'] = ENV['VMPOOLER_TRACING_ENABLED'] || parsed_config[:tracing]['enabled'] || 'false' + parsed_config[:tracing]['jaeger_host'] = ENV['VMPOOLER_TRACING_JAEGER_HOST'] || parsed_config[:tracing]['jaeger_host'] || 'http://localhost:14268/api/traces' + parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER'] if parsed_config.key? :auth parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER'] @@ -213,4 +225,42 @@ module Vmpooler parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/ parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones'] end + + def self.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version) + if startup_args.length == 1 && startup_args.include?('api') + service_name = 'vmpooler-api' + elsif startup_args.length == 1 && startup_args.include?('manager') + service_name = 'vmpooler-manager' + else + service_name = 'vmpooler' + end + + service_name += "-#{prefix}" unless prefix.empty? + + if tracing_enabled.eql?('false') + puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'" + span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + exporter: OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new + ) + else + puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}" + span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + exporter: OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host) + ) + end + + OpenTelemetry::SDK.configure do |c| + c.use 'OpenTelemetry::Instrumentation::Sinatra' + c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby' + c.use 'OpenTelemetry::Instrumentation::Redis' + + c.add_span_processor(span_processor) + c.resource = OpenTelemetry::SDK::Resources::Resource.create( + { + OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:name] => service_name, + OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:version] => version + } + ) + end + end end