commit a08cba099f867b1db01a50940ec3ae9239245db5 Author: Gene Liverman Date: Mon Nov 29 15:14:42 2021 -0500 Initial commit migrating from VMPooler proper This copies in all needed files from the main VMPooler repo. Version 1.3.0 of VMPooler was used as the basis for this code. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c8f8016 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: bundler + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..501403f --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Testing + +on: + pull_request: + branches: + - main + +jobs: + rubocop: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run Rubocop + run: bundle exec rake rubocop + + spec_tests: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + - 'jruby-9.2.12.0' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run spec tests + run: bundle exec rake test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41f92b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.bundle/ +.vagrant/ +coverage/ +vendor/ +.dccache +.ruby-version +Gemfile.local +results.xml +/vmpooler.yaml + diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..9c4ecfc --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ + +# This will cause DIO to be assigned review of any opened PRs against +# the branches containing this file. +# See https://help.github.com/en/articles/about-code-owners for info on how to +# take ownership of parts of the code base that should be reviewed by another +# team. + +# DIO will be the default owners for everything in the repo. +* @puppetlabs/dio + diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..122d6b5 --- /dev/null +++ b/Gemfile @@ -0,0 +1,13 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gemspec + +# Evaluate Gemfile.local if it exists +if File.exists? "#{__FILE__}.local" + instance_eval(File.read("#{__FILE__}.local")) +end + +# Evaluate ~/.gemfile if it exists +if File.exists?(File.join(Dir.home, '.gemfile')) + instance_eval(File.read(File.join(Dir.home, '.gemfile'))) +end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..83f626e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,191 @@ +PATH + remote: . + specs: + vmpooler-vsphere-provider (1.3.0) + rbvmomi (>= 2.1, < 4.0) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + bindata (2.4.10) + builder (3.2.4) + climate_control (1.0.1) + coderay (1.1.3) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) + diff-lcs (1.4.4) + docile (1.4.0) + faraday (1.8.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0.1) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.1) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + google-cloud-env (1.5.0) + faraday (>= 0.17.3, < 2.0) + json (2.6.1) + method_source (1.0.0) + mock_redis (0.29.0) + ruby2_keywords + multipart-post (2.1.1) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) + net-ldap (0.17.0) + nio4r (2.5.8) + nokogiri (1.12.5-x86_64-linux) + racc (~> 1.4) + opentelemetry-api (0.17.0) + opentelemetry-common (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-exporter-jaeger (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-sdk (~> 0.17.0) + thrift + opentelemetry-instrumentation-base (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-concurrent_ruby (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-instrumentation-redis (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-instrumentation-sinatra (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + opentelemetry-resource_detectors (0.17.0) + google-cloud-env + opentelemetry-sdk + opentelemetry-sdk (0.17.0) + opentelemetry-api (~> 0.17.0) + opentelemetry-common (~> 0.17.0) + opentelemetry-instrumentation-base (~> 0.17.0) + optimist (3.0.1) + parallel (1.21.0) + parser (3.0.3.1) + ast (~> 2.4.1) + pickup (0.0.11) + prometheus-client (2.1.0) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) + puma (5.5.2) + nio4r (~> 2.0) + racc (1.6.0) + rack (2.2.3) + rack-protection (2.1.0) + rack + rack-test (1.1.0) + rack (>= 1.0, < 3) + rainbow (3.0.0) + rake (13.0.6) + rbvmomi (3.0.0) + builder (~> 3.2) + json (~> 2.3) + nokogiri (~> 1.10) + optimist (~> 3.0) + redis (4.5.1) + regexp_parser (2.1.1) + rexml (3.2.5) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.3) + rubocop (1.1.0) + parallel (~> 1.10) + parser (>= 2.7.1.5) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8) + rexml + rubocop-ast (>= 1.0.1) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 2.0) + rubocop-ast (1.13.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) + ruby2_keywords (0.0.5) + simplecov (0.21.2) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.3) + sinatra (2.1.0) + mustermann (~> 1.0) + rack (~> 2.2) + rack-protection (= 2.1.0) + tilt (~> 2.0) + spicy-proton (2.1.13) + bindata (~> 2.3) + statsd-ruby (1.5.0) + thor (1.1.0) + thrift (0.15.0) + tilt (2.0.10) + unicode-display_width (1.8.0) + vmpooler (1.3.0) + concurrent-ruby (~> 1.1) + connection_pool (~> 2.2) + net-ldap (~> 0.16) + nokogiri (~> 1.10) + opentelemetry-exporter-jaeger (= 0.17.0) + opentelemetry-instrumentation-concurrent_ruby (= 0.17.0) + opentelemetry-instrumentation-redis (= 0.17.0) + opentelemetry-instrumentation-sinatra (= 0.17.0) + opentelemetry-resource_detectors (= 0.17.0) + opentelemetry-sdk (= 0.17.0) + pickup (~> 0.0.11) + prometheus-client (~> 2.0) + puma (~> 5.0, >= 5.0.4) + rack (~> 2.2) + rake (~> 13.0) + rbvmomi (>= 2.1, < 4.0) + redis (~> 4.1) + sinatra (~> 2.0) + spicy-proton (~> 2.1) + statsd-ruby (~> 1.4) + yarjuf (2.0.0) + builder + rspec (~> 3) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + climate_control (>= 0.2.0) + mock_redis (>= 0.17.0) + pry + rack-test (>= 0.6) + rspec (>= 3.2) + rubocop (~> 1.1.0) + simplecov (>= 0.11.2) + thor (~> 1.0, >= 1.0.1) + vmpooler (~> 1.3, >= 1.3.0) + vmpooler-vsphere-provider! + yarjuf (>= 2.0) + +BUNDLED WITH + 2.2.22 diff --git a/README.md b/README.md new file mode 100644 index 0000000..8cbda60 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# vmpooler-vsphere-provider + +This is a WIP - do not use yet. The goal is to extract the vSphere provider from the main VMPooler codebase. Force pushes to this repo may happen while initial development is happening. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..76d6a80 --- /dev/null +++ b/Rakefile @@ -0,0 +1,25 @@ +require 'rspec/core/rake_task' + +rubocop_available = Gem::Specification::find_all_by_name('rubocop').any? +require 'rubocop/rake_task' if rubocop_available + +desc 'Run rspec tests with coloring.' +RSpec::Core::RakeTask.new(:test) do |t| + t.rspec_opts = %w[--color --format documentation] + t.pattern = 'spec/' +end + +desc 'Run rspec tests and save JUnit output to results.xml.' +RSpec::Core::RakeTask.new(:junit) do |t| + t.rspec_opts = %w[-r yarjuf -f JUnit -o results.xml] + t.pattern = 'spec/' +end + +if rubocop_available + desc 'Run RuboCop' + RuboCop::RakeTask.new(:rubocop) do |task| + task.options << '--display-cop-names' + end +end + +task :default => [:test] diff --git a/lib/vmpooler-vsphere-provider/version.rb b/lib/vmpooler-vsphere-provider/version.rb new file mode 100644 index 0000000..e9c9fad --- /dev/null +++ b/lib/vmpooler-vsphere-provider/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module VmpoolerVsphereProvider + VERSION = '1.3.0' +end diff --git a/lib/vmpooler/providers/vsphere.rb b/lib/vmpooler/providers/vsphere.rb new file mode 100644 index 0000000..b860743 --- /dev/null +++ b/lib/vmpooler/providers/vsphere.rb @@ -0,0 +1,1173 @@ +# frozen_string_literal: true + +require 'bigdecimal' +require 'bigdecimal/util' +require 'rbvmomi' +require 'vmpooler/providers/base' + +module Vmpooler + class PoolManager + class Provider + class VSphere < Vmpooler::PoolManager::Provider::Base + # The connection_pool method is normally used only for testing + attr_reader :connection_pool + + def initialize(config, logger, metrics, redis_connection_pool, name, options) + super(config, logger, metrics, redis_connection_pool, name, options) + + task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i + # The default connection pool size is: + # Whatever is biggest from: + # - How many pools this provider services + # - Maximum number of cloning tasks allowed + # - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc. + default_connpool_size = [provided_pools.count, task_limit, 2].max + connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i + # The default connection pool timeout should be quite large - 60 seconds + connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i + logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}") + @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + connpool_type: 'provider_connection_pool', + connpool_provider: name, + size: connpool_size, + timeout: connpool_timeout + ) do + logger.log('d', "[#{name}] Connection Pool - Creating a connection object") + # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve + # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection + # object. Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the + # Hash can change, and is preserved across invocations. + new_conn = connect_to_vsphere + { connection: new_conn } + end + @provider_hosts = {} + @provider_hosts_lock = Mutex.new + @redis = redis_connection_pool + end + + # name of the provider class + def name + 'vsphere' + end + + def folder_configured?(folder_title, base_folder, configured_folders, whitelist) + return true if whitelist&.include?(folder_title) + return false unless configured_folders.keys.include?(folder_title) + return false unless configured_folders[folder_title] == base_folder + + true + end + + def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl) + try = 0 if try.nil? + max_tries = 3 + @redis.with_metrics do |redis| + redis.multi + redis.srem("vmpooler__completed__#{pool}", vm_name) + redis.hdel("vmpooler__active__#{pool}", vm_name) + redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now) + + # Auto-expire metadata key + redis.expire("vmpooler__vm__#{vm_name}", (data_ttl * 60 * 60)) + redis.exec + end + + start = Time.now + + if vm_object.is_a? RbVmomi::VIM::Folder + logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying") + raise('Expected VM, but received a folder object') + end + vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime&.powerState && vm_object.runtime.powerState == 'poweredOn' + vm_object.Destroy_Task.wait_for_completion + + finish = format('%