From a08cba099f867b1db01a50940ec3ae9239245db5 Mon Sep 17 00:00:00 2001 From: Gene Liverman Date: Mon, 29 Nov 2021 15:14:42 -0500 Subject: [PATCH] Initial commit migrating from VMPooler proper This copies in all needed files from the main VMPooler repo. Version 1.3.0 of VMPooler was used as the basis for this code. --- .github/dependabot.yml | 8 + .github/workflows/testing.yml | 47 + .gitignore | 10 + CODEOWNERS | 10 + Gemfile | 13 + Gemfile.lock | 191 + README.md | 3 + Rakefile | 25 + lib/vmpooler-vsphere-provider/version.rb | 5 + lib/vmpooler/providers/vsphere.rb | 1173 +++++ scripts/create_template_deltas.rb | 98 + spec/fixtures/vmpooler.yaml | 43 + spec/fixtures/vmpooler2.yaml | 43 + spec/helpers.rb | 154 + spec/rbvmomi_helper.rb | 922 ++++ spec/spec_helper.rb | 19 + spec/unit/pool_manager_spec.rb | 5027 ++++++++++++++++++++++ spec/unit/providers/vsphere_spec.rb | 3599 ++++++++++++++++ vmpooler-vsphere-provider.gemspec | 32 + 19 files changed, 11422 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/testing.yml create mode 100644 .gitignore create mode 100644 CODEOWNERS create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 README.md create mode 100644 Rakefile create mode 100644 lib/vmpooler-vsphere-provider/version.rb create mode 100644 lib/vmpooler/providers/vsphere.rb create mode 100755 scripts/create_template_deltas.rb create mode 100644 spec/fixtures/vmpooler.yaml create mode 100644 spec/fixtures/vmpooler2.yaml create mode 100644 spec/helpers.rb create mode 100644 spec/rbvmomi_helper.rb create mode 100644 spec/spec_helper.rb create mode 100644 spec/unit/pool_manager_spec.rb create mode 100644 spec/unit/providers/vsphere_spec.rb create mode 100644 vmpooler-vsphere-provider.gemspec diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c8f8016 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: bundler + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..501403f --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Testing + +on: + pull_request: + branches: + - main + +jobs: + rubocop: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run Rubocop + run: bundle exec rake rubocop + + spec_tests: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + - 'jruby-9.2.12.0' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run spec tests + run: bundle exec rake test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41f92b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.bundle/ +.vagrant/ +coverage/ +vendor/ +.dccache +.ruby-version +Gemfile.local +results.xml +/vmpooler.yaml + diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..9c4ecfc --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ + +# This will cause DIO to be assigned review of any opened PRs against +# the branches containing this file. +# See https://help.github.com/en/articles/about-code-owners for info on how to +# take ownership of parts of the code base that should be reviewed by another +# team. + +# DIO will be the default owners for everything in the repo. +* @puppetlabs/dio + diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..122d6b5 --- /dev/null +++ b/Gemfile @@ -0,0 +1,13 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gemspec + +# Evaluate Gemfile.local if it exists +if File.exists? "#{__FILE__}.local" + instance_eval(File.read("#{__FILE__}.local")) +end + +# Evaluate ~/.gemfile if it exists +if File.exists?(File.join(Dir.home, '.gemfile')) + instance_eval(File.read(File.join(Dir.home, '.gemfile'))) +end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..83f626e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,191 @@ +PATH + remote: . + specs: + vmpooler-vsphere-provider (1.3.0) + rbvmomi (>= 2.1, < 4.0) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + bindata (2.4.10) + builder (3.2.4) + climate_control (1.0.1) + coderay (1.1.3) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) + diff-lcs (1.4.4) + docile (1.4.0) + faraday (1.8.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0.1) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.1) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + google-cloud-env (1.5.0) + faraday (>= 0.17.3, < 2.0) + json (2.6.1) + method_source (1.0.0) + mock_redis (0.29.0) + ruby2_keywords + multipart-post (2.1.1) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) + net-ldap (0.17.0) + nio4r (2.5.8) + nokogiri (1.12.5-x86_64-linux) + racc (~> 1.4) + opentelemetry-api (0.17.0) + opentelemetry-common (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-exporter-jaeger (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-sdk (~> 0.17.0) + thrift + opentelemetry-instrumentation-base (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-concurrent_ruby (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-instrumentation-redis (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-instrumentation-sinatra (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-resource_detectors (0.17.0) + google-cloud-env + opentelemetry-sdk + opentelemetry-sdk (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + optimist (3.0.1) + parallel (1.21.0) + parser (3.0.3.1) + ast (~> 2.4.1) + pickup (0.0.11) + prometheus-client (2.1.0) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) + puma (5.5.2) + nio4r (~> 2.0) + racc (1.6.0) + rack (2.2.3) + rack-protection (2.1.0) + rack + rack-test (1.1.0) + rack (>= 1.0, < 3) + rainbow (3.0.0) + rake (13.0.6) + rbvmomi (3.0.0) + builder (~> 3.2) + json (~> 2.3) + nokogiri (~> 1.10) + optimist (~> 3.0) + redis (4.5.1) + regexp_parser (2.1.1) + rexml (3.2.5) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.3) + rubocop (1.1.0) + parallel (~> 1.10) + parser (>= 2.7.1.5) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8) + rexml + rubocop-ast (>= 1.0.1) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 2.0) + rubocop-ast (1.13.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) + ruby2_keywords (0.0.5) + simplecov (0.21.2) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.3) + sinatra (2.1.0) + mustermann (~> 1.0) + rack (~> 2.2) + rack-protection (= 2.1.0) + tilt (~> 2.0) + spicy-proton (2.1.13) + bindata (~> 2.3) + statsd-ruby (1.5.0) + thor (1.1.0) + thrift (0.15.0) + tilt (2.0.10) + unicode-display_width (1.8.0) + vmpooler (1.3.0) + concurrent-ruby (~> 1.1) + connection_pool (~> 2.2) + net-ldap (~> 0.16) + nokogiri (~> 1.10) + opentelemetry-exporter-jaeger (= 0.17.0) + opentelemetry-instrumentation-concurrent_ruby (= 0.17.0) + opentelemetry-instrumentation-redis (= 0.17.0) + opentelemetry-instrumentation-sinatra (= 0.17.0) + opentelemetry-resource_detectors (= 0.17.0) + opentelemetry-sdk (= 0.17.0) + pickup (~> 0.0.11) + prometheus-client (~> 2.0) + puma (~> 5.0, >= 5.0.4) + rack (~> 2.2) + rake (~> 13.0) + rbvmomi (>= 2.1, < 4.0) + redis (~> 4.1) + sinatra (~> 2.0) + spicy-proton (~> 2.1) + statsd-ruby (~> 1.4) + yarjuf (2.0.0) + builder + rspec (~> 3) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + climate_control (>= 0.2.0) + mock_redis (>= 0.17.0) + pry + rack-test (>= 0.6) + rspec (>= 3.2) + rubocop (~> 1.1.0) + simplecov (>= 0.11.2) + thor (~> 1.0, >= 1.0.1) + vmpooler (~> 1.3, >= 1.3.0) + vmpooler-vsphere-provider! + yarjuf (>= 2.0) + +BUNDLED WITH + 2.2.22 diff --git a/README.md b/README.md new file mode 100644 index 0000000..8cbda60 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# vmpooler-vsphere-provider + +This is a WIP - do not use yet. The goal is to extract the vSphere provider from the main VMPooler codebase. Force pushes to this repo may happen while initial development is happening. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..76d6a80 --- /dev/null +++ b/Rakefile @@ -0,0 +1,25 @@ +require 'rspec/core/rake_task' + +rubocop_available = Gem::Specification::find_all_by_name('rubocop').any? +require 'rubocop/rake_task' if rubocop_available + +desc 'Run rspec tests with coloring.' +RSpec::Core::RakeTask.new(:test) do |t| + t.rspec_opts = %w[--color --format documentation] + t.pattern = 'spec/' +end + +desc 'Run rspec tests and save JUnit output to results.xml.' +RSpec::Core::RakeTask.new(:junit) do |t| + t.rspec_opts = %w[-r yarjuf -f JUnit -o results.xml] + t.pattern = 'spec/' +end + +if rubocop_available + desc 'Run RuboCop' + RuboCop::RakeTask.new(:rubocop) do |task| + task.options << '--display-cop-names' + end +end + +task :default => [:test] diff --git a/lib/vmpooler-vsphere-provider/version.rb b/lib/vmpooler-vsphere-provider/version.rb new file mode 100644 index 0000000..e9c9fad --- /dev/null +++ b/lib/vmpooler-vsphere-provider/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module VmpoolerVsphereProvider + VERSION = '1.3.0' +end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb new file mode 100644 index 0000000..b860743 --- /dev/null +++ b/lib/vmpooler/providers/vsphere.rb @@ -0,0 +1,1173 @@ +# frozen_string_literal: true + +require 'bigdecimal' +require 'bigdecimal/util' +require 'rbvmomi' +require 'vmpooler/providers/base' + +module Vmpooler + class PoolManager + class Provider + class VSphere < Vmpooler::PoolManager::Provider::Base + # The connection_pool method is normally used only for testing + attr_reader :connection_pool + + def initialize(config, logger, metrics, redis_connection_pool, name, options) + super(config, logger, metrics, redis_connection_pool, name, options) + + task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i + # The default connection pool size is: + # Whatever is biggest from: + # - How many pools this provider services + # - Maximum number of cloning tasks allowed + # - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc. + default_connpool_size = [provided_pools.count, task_limit, 2].max + connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i + # The default connection pool timeout should be quite large - 60 seconds + connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i + logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}") + @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + connpool_type: 'provider_connection_pool', + connpool_provider: name, + size: connpool_size, + timeout: connpool_timeout + ) do + logger.log('d', "[#{name}] Connection Pool - Creating a connection object") + # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve + # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection + # object. Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the + # Hash can change, and is preserved across invocations. + new_conn = connect_to_vsphere + { connection: new_conn } + end + @provider_hosts = {} + @provider_hosts_lock = Mutex.new + @redis = redis_connection_pool + end + + # name of the provider class + def name + 'vsphere' + end + + def folder_configured?(folder_title, base_folder, configured_folders, whitelist) + return true if whitelist&.include?(folder_title) + return false unless configured_folders.keys.include?(folder_title) + return false unless configured_folders[folder_title] == base_folder + + true + end + + def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl) + try = 0 if try.nil? + max_tries = 3 + @redis.with_metrics do |redis| + redis.multi + redis.srem("vmpooler__completed__#{pool}", vm_name) + redis.hdel("vmpooler__active__#{pool}", vm_name) + redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now) + + # Auto-expire metadata key + redis.expire("vmpooler__vm__#{vm_name}", (data_ttl * 60 * 60)) + redis.exec + end + + start = Time.now + + if vm_object.is_a? RbVmomi::VIM::Folder + logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying") + raise('Expected VM, but received a folder object') + end + vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn' + vm_object.Destroy_Task.wait_for_completion + + finish = format('%