commit 588e29b6e100327336bf0910ae16b6a85ffe279a Author: Samuel Beaulieu Date: Wed Dec 1 16:16:40 2021 -0600 (DIO-2768) Initial gce provider diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c8f8016 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: bundler + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..501403f --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Testing + +on: + pull_request: + branches: + - main + +jobs: + rubocop: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run Rubocop + run: bundle exec rake rubocop + + spec_tests: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: + - '2.5.8' + - 'jruby-9.2.12.0' + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run spec tests + run: bundle exec rake test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b97fbf --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.bundle/ +.vagrant/ +coverage/ +vendor/ +.dccache +.ruby-version +Gemfile.local +results.xml +/vmpooler.yaml +.idea diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..9c4ecfc --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ + +# This will cause DIO to be assigned review of any opened PRs against +# the branches containing this file. +# See https://help.github.com/en/articles/about-code-owners for info on how to +# take ownership of parts of the code base that should be reviewed by another +# team. + +# DIO will be the default owners for everything in the repo. +* @puppetlabs/dio + diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..122d6b5 --- /dev/null +++ b/Gemfile @@ -0,0 +1,13 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gemspec + +# Evaluate Gemfile.local if it exists +if File.exists? "#{__FILE__}.local" + instance_eval(File.read("#{__FILE__}.local")) +end + +# Evaluate ~/.gemfile if it exists +if File.exists?(File.join(Dir.home, '.gemfile')) + instance_eval(File.read(File.join(Dir.home, '.gemfile'))) +end diff --git a/README.md b/README.md new file mode 100644 index 0000000..57fbd75 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# vmpooler-provider-gce + +This is a WIP - do not use yet. Provider for GCE VMs in vmpooler. + +Vm has a label 'pool' that represent the VMpooler OS/pool it is part of +Boot disk had a label 'vm' that represent the vm_name it is attached to \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..76d6a80 --- /dev/null +++ b/Rakefile @@ -0,0 +1,25 @@ +require 'rspec/core/rake_task' + +rubocop_available = Gem::Specification::find_all_by_name('rubocop').any? +require 'rubocop/rake_task' if rubocop_available + +desc 'Run rspec tests with coloring.' +RSpec::Core::RakeTask.new(:test) do |t| + t.rspec_opts = %w[--color --format documentation] + t.pattern = 'spec/' +end + +desc 'Run rspec tests and save JUnit output to results.xml.' +RSpec::Core::RakeTask.new(:junit) do |t| + t.rspec_opts = %w[-r yarjuf -f JUnit -o results.xml] + t.pattern = 'spec/' +end + +if rubocop_available + desc 'Run RuboCop' + RuboCop::RakeTask.new(:rubocop) do |task| + task.options << '--display-cop-names' + end +end + +task :default => [:test] diff --git a/lib/vmpooler-provider-gce/version.rb b/lib/vmpooler-provider-gce/version.rb new file mode 100644 index 0000000..f51a6f5 --- /dev/null +++ b/lib/vmpooler-provider-gce/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module VmpoolerProviderGce + VERSION = '0.1.0' +end diff --git a/lib/vmpooler/providers/gce.rb b/lib/vmpooler/providers/gce.rb new file mode 100644 index 0000000..044f745 --- /dev/null +++ b/lib/vmpooler/providers/gce.rb @@ -0,0 +1,357 @@ +# frozen_string_literal: true +require 'googleauth' +require 'google/apis/compute_v1' +require 'bigdecimal' +require 'bigdecimal/util' +require 'vmpooler/providers/base' + +module Vmpooler + class PoolManager + class Provider + class Gce < Vmpooler::PoolManager::Provider::Base + # The connection_pool method is normally used only for testing + attr_reader :connection_pool + + def initialize(config, logger, metrics, redis_connection_pool, name, options) + super(config, logger, metrics, redis_connection_pool, name, options) + + task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i + # The default connection pool size is: + # Whatever is biggest from: + # - How many pools this provider services + # - Maximum number of cloning tasks allowed + # - Need at least 2 connections so that a pool can have inventory functions performed while cloning etc. + default_connpool_size = [provided_pools.count, task_limit, 2].max + connpool_size = provider_config['connection_pool_size'].nil? ? default_connpool_size : provider_config['connection_pool_size'].to_i + # The default connection pool timeout should be quite large - 60 seconds + connpool_timeout = provider_config['connection_pool_timeout'].nil? ? 60 : provider_config['connection_pool_timeout'].to_i + logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}") + @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new( + metrics: metrics, + connpool_type: 'provider_connection_pool', + connpool_provider: name, + size: connpool_size, + timeout: connpool_timeout + ) do + logger.log('d', "[#{name}] Connection Pool - Creating a connection object") + # Need to wrap the vSphere connection object in another object. The generic connection pooler will preserve + # the object reference for the connection, which means it cannot "reconnect" by creating an entirely new connection + # object. Instead by wrapping it in a Hash, the Hash object reference itself never changes but the content of the + # Hash can change, and is preserved across invocations. + new_conn = connect_to_gce + { connection: new_conn } + end + @redis = redis_connection_pool + end + + # name of the provider class + def name + 'gce' + end + + # main configuration options + def project + provider_config['project'] + end + + def network_name + provider_config['network_name'] + end + + # main configuration options, overridable for each pool + def zone(pool_name) + return pool_config(pool_name)['zone'] if pool_config(pool_name)['zone'] + return provider_config['zone'] if provider_config['zone'] + end + + def machine_type(pool_name) + return pool_config(pool_name)['machine_type'] if pool_config(pool_name)['machine_type'] + return provider_config['machine_type'] if provider_config['machine_type'] + end + + #Base methods that are implemented: + # + # + + def vms_in_pool(pool_name) + vms = [] + pool = pool_config(pool_name) + raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? + zone = zone(pool_name) + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + filter = "(labels.pool = #{pool_name})" + instance_list = connection.list_instances(project, zone, filter: filter) + + return vms if instance_list.items.nil? + + instance_list.items.each do |vm| + vms << { 'name' => vm.name } + end + end + vms + end + + # inputs + # [String]pool_name : Name of the pool + # [String] vm_name : Name of the VM + # returns + # [String] : Name of the host computer running the vm. If this is not a Virtual Machine, it returns the vm_name + def get_vm_host(_pool_name, _vm_name) + raise("#{self.class.name} does not implement get_vm_host") + end + + # inputs + # [String] pool_name : Name of the pool + # [String] vm_name : Name of the VM to find + # returns + # nil if VM doesn't exist + # [Hastable] of the VM + # [String] name : The name of the resource, provided by the client when initially creating the resource + # [String] hostname : Specifies the hostname of the instance. The specified hostname must be RFC1035 compliant. If hostname is not specified, + # the default hostname is [ INSTANCE_NAME].c.[PROJECT_ID].internal when using the global DNS, and + # [ INSTANCE_NAME].[ZONE].c.[PROJECT_ID].internal when using zonal DNS + # [String] template : This is the name of template exposed by the API. It must _match_ the poolname ??? TODO + # [String] poolname : Name of the pool the VM as per labels + # [Time] boottime : Time when the VM was created/booted + # [String] status : One of the following values: PROVISIONING, STAGING, RUNNING, STOPPING, SUSPENDING, SUSPENDED, REPAIRING, and TERMINATED + # [String] zone : URL of the zone where the instance resides. + # [String] machine_type : Full or partial URL of the machine type resource to use for this instance, in the format: zones/zone/machineTypes/machine-type. + def get_vm(pool_name, vm_name) + vm_hash = nil + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + vm_object = connection.get_instance(project, zone(pool_name), vm_name) + return vm_hash if vm_object.nil? + + vm_hash = generate_vm_hash(vm_object, pool_name) + end + vm_hash + end + + # inputs + # [String] pool : Name of the pool + # [String] new_vmname : Name to give the new VM + # returns + # [Hashtable] of the VM as per get_vm + # Raises RuntimeError if the pool_name is not supported by the Provider + def create_vm(pool_name, new_vmname) + pool = pool_config(pool_name) + raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? + + vm_hash = nil + # harcoded network info + network_interfaces = Google::Apis::ComputeV1::NetworkInterface.new( + :network => network_name + ) + initParams = { + :source_image => pool['template'], #The source image to create this disk. + :labels => {'vm' => new_vmname, 'pool' => pool_name} + } + disk = Google::Apis::ComputeV1::AttachedDisk.new( + :auto_delete => true, + :boot => true, + :initialize_params => Google::Apis::ComputeV1::AttachedDiskInitializeParams.new(initParams) + ) + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + # Assume all pool config is valid i.e. not missing + template_path = pool['template'] + client = ::Google::Apis::ComputeV1::Instance.new( + :name => new_vmname, + :machine_type => pool['machine_type'], + :disks => [disk], + :network_interfaces => [network_interfaces], + :labels => {'pool' => pool_name} + ) + result = connection.insert_instance(project, zone(pool_name), client) + result = wait_for_operation(project, pool_name, result, connection) + if result.error + error_message = "" + # array of errors, combine them all + result.error.each do |error| + error_message = "#{error_message} #{error.code}:#{error.message}" + end + raise "Pool #{pool_name} operation: #{result.description} failed with error: #{error_message}" + end + vm_hash = get_vm(pool_name, new_vmname) + end + vm_hash + rescue ::Google::Apis::ClientError => e + raise e unless e.status_code == 404 + nil + end + + #TODO + def create_disk(pool_name, vm_name, disk_size) + pool = pool_config(pool_name) + raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil? + + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + vm_object = find_vm(pool_name, vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? + + #TODO part 2 + end + true + end + + def create_snapshot(pool_name, vm_name, new_snapshot_name) + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + vm_object = find_vm(pool_name, vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? + + old_snap = find_snapshot(vm_object, new_snapshot_name) + raise("Snapshot #{new_snapshot_name} for VM #{vm_name} in pool #{pool_name} already exists for the provider #{name}") unless old_snap.nil? + + snapshot_obj = ::Google::Apis::ComputeV1::Snapshot.new(name: "#{new_snapshot_name}_boot", labels: {"snapshot_name" => new_snapshot_name, "vm" => vm_name}) + boot_disk = vm_object.disks[0] + result = connection.create_disk_snapshot(project, zone(pool_name), boot_disk, snapshot_obj) + wait_for_operation(project, pool_name, result, connection) + #TODO snapshot other disks if there are any + end + true + rescue ::Google::Apis::ClientError => e + raise e unless e.status_code == 404 + nil + end + + #TODO + def revert_snapshot(pool_name, vm_name, snapshot_name) + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + vm_object = find_vm(pool_name, vm_name, connection) + raise("VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if vm_object.nil? + + snapshot_object = find_snapshot(vm_object, snapshot_name) + raise("Snapshot #{snapshot_name} for VM #{vm_name} in pool #{pool_name} does not exist for the provider #{name}") if snapshot_object.nil? + + #TODO part 2 + end + true + end + + def destroy_vm(pool_name, vm_name) + @connection_pool.with_metrics do |pool_object| + connection = ensured_gce_connection(pool_object) + vm_object = find_vm(pool_name, vm_name, connection) + # If a VM doesn't exist then it is effectively deleted + return true if vm_object.nil? + + start = Time.now + + result = connection.delete_instance(project, @options[:vm_zone], name) + wait_for_operation(project, pool_name, result, connection) + + finish = format('%