diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4ae585a..81e0069 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,3 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - open-pull-requests-limit: 10 diff --git a/.github/workflows/auto_release_prep.yml b/.github/workflows/auto_release_prep.yml deleted file mode 100644 index ad905fa..0000000 --- a/.github/workflows/auto_release_prep.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Automated release prep - -on: - workflow_dispatch: - -jobs: - auto_release_prep: - uses: puppetlabs/release-engineering-repo-standards/.github/workflows/auto_release_prep.yml@v1 - secrets: inherit - with: - project-type: ruby - version-file-path: lib/vmpooler-provider-gce/version.rb diff --git a/.github/workflows/dependabot_merge.yml b/.github/workflows/dependabot_merge.yml deleted file mode 100644 index 75b9cea..0000000 --- a/.github/workflows/dependabot_merge.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Dependabot auto-merge - -on: pull_request - -jobs: - dependabot_merge: - uses: puppetlabs/release-engineering-repo-standards/.github/workflows/dependabot_merge.yml@v1 - secrets: inherit diff --git a/.github/workflows/ensure_label.yml b/.github/workflows/ensure_label.yml deleted file mode 100644 index 50a5fa8..0000000 --- a/.github/workflows/ensure_label.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Ensure label - -on: pull_request - -jobs: - ensure_label: - uses: puppetlabs/release-engineering-repo-standards/.github/workflows/ensure_label.yml@v1 - secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index add1656..0e30eae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'puppetlabs/vmpooler-provider-gce' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Get Current Version - uses: actions/github-script@v7 + uses: actions/github-script@v6 id: cv with: script: | @@ -70,10 +70,10 @@ jobs: prerelease: false # This step should closely match what is used in `docker/Dockerfile` in vmpooler-deployment - - name: Install Ruby jruby-9.4.3.0 + - name: Install Ruby jruby-9.4.1.0 uses: ruby/setup-ruby@v1 with: - ruby-version: 'jruby-9.4.3.0' + ruby-version: 'jruby-9.4.1.0' - name: Build gem run: gem build *.gemspec diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index ba273f5..666c602 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout repo content - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 1 - name: setup ruby @@ -22,7 +22,7 @@ jobs: - name: check lock run: '[ -f "Gemfile.lock" ] && echo "package lock file exists, skipping" || bundle lock' # install java - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v3 with: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '17' diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1f2f421..eac55b3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -18,9 +18,9 @@ jobs: strategy: matrix: ruby-version: - - 'jruby-9.4.3.0' + - 'jruby-9.4.1.0' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -34,9 +34,9 @@ jobs: strategy: matrix: ruby-version: - - 'jruby-9.4.3.0' + - 'jruby-9.4.1.0' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index ed1eead..3bc3ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,44 +1,5 @@ # Changelog -## [1.3.0](https://github.com/puppetlabs/vmpooler-provider-gce/tree/1.3.0) (2023-08-18) - -[Full Changelog](https://github.com/puppetlabs/vmpooler-provider-gce/compare/1.2.0...1.3.0) - -**Implemented enhancements:** - -- \(POD-10\) Log reason for failed VM checks. [\#33](https://github.com/puppetlabs/vmpooler-provider-gce/pull/33) ([isaac-hammes](https://github.com/isaac-hammes)) - -## [1.2.0](https://github.com/puppetlabs/vmpooler-provider-gce/tree/1.2.0) (2023-08-10) - -[Full Changelog](https://github.com/puppetlabs/vmpooler-provider-gce/compare/1.1.0...1.2.0) - -**Implemented enhancements:** - -- Bump jruby to 9.4.3.0 and update lockfile [\#31](https://github.com/puppetlabs/vmpooler-provider-gce/pull/31) ([yachub](https://github.com/yachub)) - -**Merged pull requests:** - -- Bump thor from 1.2.1 to 1.2.2 [\#29](https://github.com/puppetlabs/vmpooler-provider-gce/pull/29) ([dependabot[bot]](https://github.com/apps/dependabot)) - -## [1.1.0](https://github.com/puppetlabs/vmpooler-provider-gce/tree/1.1.0) (2023-05-01) - -[Full Changelog](https://github.com/puppetlabs/vmpooler-provider-gce/compare/1.0.0...1.1.0) - -**Merged pull requests:** - -- Migrate issue management to Jira [\#27](https://github.com/puppetlabs/vmpooler-provider-gce/pull/27) ([yachub](https://github.com/yachub)) -- Bump jruby to 9.4.2.0 [\#26](https://github.com/puppetlabs/vmpooler-provider-gce/pull/26) ([yachub](https://github.com/yachub)) -- Bump rack-test from 2.0.2 to 2.1.0 [\#24](https://github.com/puppetlabs/vmpooler-provider-gce/pull/24) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Update googleauth requirement from \>= 0.16.2, \< 1.3.0 to \>= 0.16.2, \< 1.4.0 [\#18](https://github.com/puppetlabs/vmpooler-provider-gce/pull/18) ([dependabot[bot]](https://github.com/apps/dependabot)) - -## [1.0.0](https://github.com/puppetlabs/vmpooler-provider-gce/tree/1.0.0) (2023-04-19) - -[Full Changelog](https://github.com/puppetlabs/vmpooler-provider-gce/compare/0.5.0...1.0.0) - -**Breaking changes:** - -- \(RE-15124\) Decouple DNS Record Management into DNS Plugins [\#21](https://github.com/puppetlabs/vmpooler-provider-gce/pull/21) ([yachub](https://github.com/yachub)) - ## [0.5.0](https://github.com/puppetlabs/vmpooler-provider-gce/tree/0.5.0) (2023-03-06) [Full Changelog](https://github.com/puppetlabs/vmpooler-provider-gce/compare/0.4.0...0.5.0) diff --git a/Gemfile.lock b/Gemfile.lock index 749532c..2258b05 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,15 +1,16 @@ PATH remote: . specs: - vmpooler-provider-gce (1.3.0) + vmpooler-provider-gce (0.5.0) google-apis-compute_v1 (~> 0.14) - googleauth (>= 0.16.2, < 1.4.0) - vmpooler (~> 3.0) + google-cloud-dns (~> 0.35.1) + googleauth (>= 0.16.2, < 1.3.0) + vmpooler (~> 2.3, >= 1.3.0) GEM remote: https://rubygems.org/ specs: - addressable (2.8.5) + addressable (2.8.1) public_suffix (>= 2.0.2, < 6.0) ast (2.4.2) bindata (2.4.15) @@ -17,19 +18,19 @@ GEM climate_control (1.2.0) coderay (1.1.3) concurrent-ruby (1.2.2) - connection_pool (2.4.1) + connection_pool (2.3.0) declarative (0.0.20) deep_merge (1.2.2) diff-lcs (1.5.0) docile (1.4.0) - faraday (2.7.10) + faraday (2.7.4) faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) ffi (1.15.5-java) - google-apis-compute_v1 (0.75.0) + google-apis-compute_v1 (0.62.0) google-apis-core (>= 0.11.0, < 2.a) - google-apis-core (0.11.1) + google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -38,9 +39,20 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick + google-apis-dns_v1 (0.31.0) + google-apis-core (>= 0.11.0, < 2.a) + google-cloud-core (1.6.0) + google-cloud-env (~> 1.0) + google-cloud-errors (~> 1.0) + google-cloud-dns (0.35.1) + google-apis-dns_v1 (~> 0.1) + google-cloud-core (~> 1.6) + googleauth (>= 0.16.2, < 2.a) + zonefile (~> 1.04) google-cloud-env (1.6.0) faraday (>= 0.17.3, < 3.0) - googleauth (1.3.0) + google-cloud-errors (1.3.1) + googleauth (1.2.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -48,110 +60,95 @@ GEM os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) httpclient (2.8.3) - jwt (2.7.1) + jwt (2.7.0) memoist (0.16.2) method_source (1.0.0) - mini_mime (1.1.5) - mock_redis (0.37.0) + mini_mime (1.1.2) + mock_redis (0.36.0) + ruby2_keywords multi_json (1.15.0) - mustermann (3.0.0) + mustermann (2.0.2) ruby2_keywords (~> 0.0.1) - net-ldap (0.18.0) - nio4r (2.5.9) - nio4r (2.5.9-java) - opentelemetry-api (1.2.1) - opentelemetry-common (0.20.0) + net-ldap (0.17.1) + nio4r (2.5.8-java) + opentelemetry-api (1.1.0) + opentelemetry-common (0.19.6) opentelemetry-api (~> 1.0) - opentelemetry-exporter-jaeger (0.23.0) - opentelemetry-api (~> 1.1) - opentelemetry-common (~> 0.20) - opentelemetry-sdk (~> 1.2) - opentelemetry-semantic_conventions - thrift - opentelemetry-instrumentation-base (0.22.2) + opentelemetry-exporter-jaeger (0.20.1) opentelemetry-api (~> 1.0) - opentelemetry-registry (~> 0.1) - opentelemetry-instrumentation-concurrent_ruby (0.21.1) - opentelemetry-api (~> 1.0) - opentelemetry-instrumentation-base (~> 0.22.1) - opentelemetry-instrumentation-http_client (0.22.2) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.20.0) - opentelemetry-instrumentation-base (~> 0.22.1) - opentelemetry-instrumentation-rack (0.23.4) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.20.0) - opentelemetry-instrumentation-base (~> 0.22.1) - opentelemetry-instrumentation-redis (0.25.3) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.20.0) - opentelemetry-instrumentation-base (~> 0.22.1) - opentelemetry-instrumentation-sinatra (0.23.2) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.20.0) - opentelemetry-instrumentation-base (~> 0.22.1) - opentelemetry-instrumentation-rack (~> 0.21) - opentelemetry-registry (0.3.0) - opentelemetry-api (~> 1.1) - opentelemetry-resource_detectors (0.24.1) - google-cloud-env + opentelemetry-common (~> 0.19.2) opentelemetry-sdk (~> 1.0) - opentelemetry-sdk (1.3.0) + thrift + opentelemetry-instrumentation-base (0.19.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-concurrent_ruby (0.19.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-http_client (0.19.4) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-redis (0.21.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-instrumentation-sinatra (0.19.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.19.0) + opentelemetry-registry (0.2.0) opentelemetry-api (~> 1.1) - opentelemetry-common (~> 0.20) + opentelemetry-resource_detectors (0.19.1) + google-cloud-env + opentelemetry-sdk + opentelemetry-sdk (1.2.0) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.19.3) opentelemetry-registry (~> 0.2) opentelemetry-semantic_conventions - opentelemetry-semantic_conventions (1.10.0) + opentelemetry-semantic_conventions (1.8.0) opentelemetry-api (~> 1.0) os (1.1.4) - parallel (1.23.0) - parser (3.2.2.3) + parallel (1.22.1) + parser (3.2.1.0) ast (~> 2.4.1) - racc pickup (0.0.11) - prometheus-client (4.2.1) - pry (0.14.2) - coderay (~> 1.1) - method_source (~> 1.0) + prometheus-client (2.1.0) pry (0.14.2-java) coderay (~> 1.1) method_source (~> 1.0) spoon (~> 0.0) - public_suffix (5.0.3) - puma (6.3.0) + public_suffix (5.0.1) + puma (5.6.5-java) nio4r (~> 2.0) - puma (6.3.0-java) - nio4r (~> 2.0) - racc (1.7.1) - racc (1.7.1-java) - rack (2.2.8) - rack-protection (3.1.0) - rack (~> 2.2, >= 2.2.4) - rack-test (2.1.0) + rack (2.2.6.2) + rack-protection (2.2.4) + rack + rack-test (2.0.2) rack (>= 1.3) rainbow (3.1.1) rake (13.0.6) redis (4.8.1) - regexp_parser (2.8.1) + regexp_parser (2.7.0) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.2.6) + rexml (3.2.5) rspec (3.12.0) rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) rspec-mocks (~> 3.12.0) - rspec-core (3.12.2) + rspec-core (3.12.1) rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) + rspec-expectations (3.12.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) + rspec-mocks (3.12.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-support (3.12.1) + rspec-support (3.12.0) rubocop (1.1.0) parallel (~> 1.10) parser (>= 2.7.1.5) @@ -161,9 +158,9 @@ GEM rubocop-ast (>= 1.0.1) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.29.0) + rubocop-ast (1.27.0) parser (>= 3.2.1.0) - ruby-progressbar (1.13.0) + ruby-progressbar (1.12.0) ruby2_keywords (0.0.5) signet (0.17.0) addressable (~> 2.8) @@ -176,52 +173,52 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - sinatra (3.1.0) - mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.1.0) + sinatra (2.2.4) + mustermann (~> 2.0) + rack (~> 2.2) + rack-protection (= 2.2.4) tilt (~> 2.0) spicy-proton (2.1.15) bindata (~> 2.3) spoon (0.0.6) ffi statsd-ruby (1.5.0) - thor (1.2.2) + thor (1.2.1) thrift (0.18.1) - tilt (2.2.0) + tilt (2.1.0) trailblazer-option (0.1.2) uber (0.1.0) unicode-display_width (1.8.0) - vmpooler (3.2.0) + vmpooler (2.4.0) concurrent-ruby (~> 1.1) connection_pool (~> 2.2) deep_merge (~> 1.2) net-ldap (~> 0.16) - opentelemetry-exporter-jaeger (= 0.23.0) - opentelemetry-instrumentation-concurrent_ruby (= 0.21.1) - opentelemetry-instrumentation-http_client (= 0.22.2) - opentelemetry-instrumentation-redis (= 0.25.3) - opentelemetry-instrumentation-sinatra (= 0.23.2) - opentelemetry-resource_detectors (= 0.24.1) - opentelemetry-sdk (~> 1.3, >= 1.3.0) + opentelemetry-exporter-jaeger (= 0.20.1) + opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) + opentelemetry-instrumentation-http_client (= 0.19.4) + opentelemetry-instrumentation-redis (= 0.21.3) + opentelemetry-instrumentation-sinatra (= 0.19.3) + opentelemetry-resource_detectors (= 0.19.1) + opentelemetry-sdk (~> 1.0, >= 1.0.2) pickup (~> 0.0.11) - prometheus-client (>= 2, < 5) - puma (>= 5.0.4, < 7) - rack (>= 2.2, < 4.0) + prometheus-client (~> 2.0) + puma (~> 5.0, >= 5.0.4) + rack (~> 2.2) rake (~> 13.0) redis (~> 4.1) - sinatra (>= 2, < 4) + sinatra (~> 2.0) spicy-proton (~> 2.1) statsd-ruby (~> 1.4) webrick (1.8.1) yarjuf (2.0.0) builder rspec (~> 3) + zonefile (1.06) PLATFORMS universal-java-1.8 universal-java-11 - x86_64-linux DEPENDENCIES climate_control (>= 0.2.0) @@ -236,4 +233,4 @@ DEPENDENCIES yarjuf (>= 2.0) BUNDLED WITH - 2.4.7 + 2.4.5 diff --git a/README.md b/README.md index d7e4e98..29cb7a6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ - [vmpooler-provider-gce](#vmpooler-provider-gce) - [Usage](#usage) - - [Migrating to v1](#migrating-to-v1) - [DNS](#dns) - [Labels](#labels) - [Pre-requisite](#pre-requisite) @@ -24,42 +23,6 @@ GCE authorization is handled via a service account (or personal account) private 1. GOOGLE_APPLICATION_CREDENTIALS environment variable eg GOOGLE_APPLICATION_CREDENTIALS=/my/home/directory/my_account_key.json -### Migrating to v1 - -Starting with the v1.x release, management of DNS records has been extracted from this compute provider and implemented as DNS plugins, similar to compute providers. This means each pool configuration should be pointing to a configuration object in `:dns_config` to determine it's method of record management. - -For those using DNS management via this provider, the DNS related options should be moved under `:dns_configs:` with the value for `dns_class`. - -For example, the following keys in a v0.x GCE provider config: - -```yaml -:providers: - :gce: - domain: vmpooler.example.com - dns_zone_resource_name: vmpooler-example-com -``` - -Would be moved to: - -```yaml -:dns_configs: - :example: - dns_class: gcp-clouddns - project: jake-vmpooler-dev - domain: vmpooler.example.com - zone_name: vmpooler-example-com -``` - -Then any pools that should have records created via the dns config above should now reference the named dns config in the `dns_plugin` key: - -```yaml -:pools: - - name: 'debian-11-x86_64' - dns_plugin: 'example' -``` - -For complete examples on how to use the GCP DNS plugin see [vmpooler-dns-gcp](https://github.com/puppetlabs/vmpooler-dns-gcp). - ### DNS DNS is integrated via Google's CloudDNS service. To enable, a CloudDNS zone name must be provided in the config (see the example yaml file dns_zone_resource_name) diff --git a/install-gemfile-lock b/install-gemfile-lock deleted file mode 100755 index a479b0d..0000000 --- a/install-gemfile-lock +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment -docker run -it --rm \ - -v $(pwd):/app \ - jruby:9.4.3.0-jdk11 \ - /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends make git netbase && cd /app && gem install bundler && bundle install --jobs 3; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' diff --git a/lib/vmpooler-provider-gce/version.rb b/lib/vmpooler-provider-gce/version.rb index b97bce2..effca7b 100644 --- a/lib/vmpooler-provider-gce/version.rb +++ b/lib/vmpooler-provider-gce/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module VmpoolerProviderGce - VERSION = '1.3.0' + VERSION = '0.5.0' end diff --git a/lib/vmpooler/cloud_dns.rb b/lib/vmpooler/cloud_dns.rb new file mode 100644 index 0000000..a98d9ef --- /dev/null +++ b/lib/vmpooler/cloud_dns.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'googleauth' +require 'google/cloud/dns' + +module Vmpooler + class PoolManager + # This class interacts with GCP Cloud DNS to create or delete records. + class CloudDns + def initialize(project, dns_zone_resource_name) + @dns = Google::Cloud::Dns.new(project_id: project) + @dns_zone_resource_name = dns_zone_resource_name + end + + def dns_create_or_replace(created_instance) + dns_zone = @dns.zone(@dns_zone_resource_name) if @dns_zone_resource_name + return unless dns_zone && created_instance && created_instance['name'] && created_instance['ip'] + + retries = 0 + name = created_instance['name'] + begin + change = dns_zone.add(name, 'A', 60, [created_instance['ip']]) + debug_logger("#{change.id} - #{change.started_at} - #{change.status} DNS address added") if change + rescue Google::Cloud::AlreadyExistsError => _e + # DNS setup is done only for new instances, so in the rare case where a DNS record already exists (it is stale) and we replace it. + # the error is Google::Cloud::AlreadyExistsError: alreadyExists: The resource 'entity.change.additions[0]' named 'instance-8.test.vmpooler.net. (A)' already exists + change = dns_zone.replace(name, 'A', 60, [created_instance['ip']]) + debug_logger("#{change.id} - #{change.started_at} - #{change.status} DNS address previously existed and was replaced") if change + rescue Google::Cloud::FailedPreconditionError => e + # this error was experienced intermittently, will retry to see if it can complete successfully + # the error is Google::Cloud::FailedPreconditionError: conditionNotMet: Precondition not met for 'entity.change.deletions[0]' + debug_logger("DNS create failed, retrying error: #{e}") + sleep 5 + retry if (retries += 1) < 30 + end + end + + def dns_teardown(created_instance) + dns_zone = @dns.zone(@dns_zone_resource_name) if @dns_zone_resource_name + return unless dns_zone && created_instance + + retries = 0 + name = created_instance['name'] + change = dns_zone.remove(name, 'A') + debug_logger("#{change.id} - #{change.started_at} - #{change.status} DNS address removed") if change + rescue Google::Cloud::FailedPreconditionError => e + # this error was experienced intermittently, will retry to see if it can complete successfully + # the error is Google::Cloud::FailedPreconditionError: conditionNotMet: Precondition not met for 'entity.change.deletions[1]' + debug_logger("DNS teardown failed, retrying error: #{e}") + sleep 5 + retry if (retries += 1) < 30 + end + + # used in local dev environment, set DEBUG_FLAG=true + # this way the upstream vmpooler manager does not get polluted with logs + def debug_logger(message, send_to_upstream: false) + # the default logger is simple and does not enforce debug levels (the first argument) + puts message if ENV['DEBUG_FLAG'] + logger.log('[g]', message) if send_to_upstream + end + end + end +end diff --git a/lib/vmpooler/providers/gce.rb b/lib/vmpooler/providers/gce.rb index 747872e..7c95859 100644 --- a/lib/vmpooler/providers/gce.rb +++ b/lib/vmpooler/providers/gce.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true +require 'googleauth' +require 'google/apis/compute_v1' +require 'vmpooler/cloud_dns' require 'bigdecimal' require 'bigdecimal/util' -require 'google/apis/compute_v1' -require 'googleauth' -require 'vmpooler/dns/base' require 'vmpooler/providers/base' module Vmpooler @@ -82,9 +82,12 @@ module Vmpooler return provider_config['machine_type'] if provider_config['machine_type'] end - def domain(pool_name) - dns_plugin_name = pool_config(pool_name)['dns_plugin'] - dns_config(dns_plugin_name) + def domain + provider_config['domain'] + end + + def dns_zone_resource_name + provider_config['dns_zone_resource_name'] end # Base methods that are implemented: @@ -188,7 +191,7 @@ module Vmpooler boot: true, initialize_params: init_params ) - append_domain = domain(pool_name) + append_domain = domain || global_config[:config]['domain'] fqdn = "#{new_vmname}.#{append_domain}" if append_domain # Assume all pool config is valid i.e. not missing @@ -205,12 +208,9 @@ module Vmpooler debug_logger('trigger insert_instance') result = connection.insert_instance(project, zone(pool_name), client) wait_for_operation(project, pool_name, result) - get_vm(pool_name, new_vmname) - end - - def get_vm_ip_address(vm_name, pool_name) - vm_object = get_vm(pool_name, vm_name) - vm_object['ip'] + created_instance = get_vm(pool_name, new_vmname) + dns_setup(created_instance) + created_instance end # create_disk creates an additional disk for an existing VM. It will name the new @@ -424,8 +424,10 @@ module Vmpooler unless deleted debug_logger("trigger delete_instance #{vm_name}") + vm_hash = get_vm(pool_name, vm_name) result = connection.delete_instance(project, zone(pool_name), vm_name) wait_for_operation(project, pool_name, result, 10) + dns_teardown(vm_hash) end # list and delete any leftover disk, for instance if they were detached from the instance @@ -460,17 +462,13 @@ module Vmpooler true end - def vm_ready?(pool_name, vm_name, redis) - debug_logger('vm_ready?') + def vm_ready?(_pool_name, vm_name) begin # TODO: we could use a healthcheck resource attached to instance - domain = domain(pool_name) - open_socket(vm_name, domain) - rescue StandardError => e - redis.hset("vmpooler__vm__#{vm_name}", 'open_socket_error', e.to_s) + open_socket(vm_name, domain || global_config[:config]['domain']) + rescue StandardError => _e return false end - redis.hdel("vmpooler__vm__#{vm_name}", 'open_socket_error') true end @@ -499,6 +497,9 @@ module Vmpooler debug_logger("trigger async delete_instance #{vm.name}") result = connection.delete_instance(project, zone, vm.name) + vm_pool = vm.labels&.key?('pool') ? vm.labels['pool'] : nil + existing_vm = generate_vm_hash(vm, vm_pool) + dns_teardown(existing_vm) result_list << result end # now check they are done @@ -559,6 +560,16 @@ module Vmpooler # END BASE METHODS + def dns_setup(created_instance) + dns = Vmpooler::PoolManager::CloudDns.new(project, dns_zone_resource_name) + dns.dns_create_or_replace(created_instance) + end + + def dns_teardown(created_instance) + dns = Vmpooler::PoolManager::CloudDns.new(project, dns_zone_resource_name) + dns.dns_teardown(created_instance) + end + def should_be_ignored(item, allowlist) return false if allowlist.nil? diff --git a/release-prep b/release-prep index 502ad2b..59c10f7 100755 --- a/release-prep +++ b/release-prep @@ -3,13 +3,13 @@ # The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment # # Update Gemfile.lock -docker run -t --rm \ +docker run -it --rm \ -v $(pwd):/app \ - jruby:9.4.3.0-jdk11 \ + jruby:9.4.1.0-jdk11 \ /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase && cd /app && gem install bundler && bundle install --jobs 3; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' # Update Changelog -docker run -t --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \ +docker run -it --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \ githubchangeloggenerator/github-changelog-generator:1.16.2 \ github_changelog_generator --future-release $(grep VERSION lib/vmpooler-provider-gce/version.rb |rev |cut -d "'" -f2 |rev) diff --git a/spec/dnsservice_helper.rb b/spec/dnsservice_helper.rb new file mode 100644 index 0000000..c2752c0 --- /dev/null +++ b/spec/dnsservice_helper.rb @@ -0,0 +1,9 @@ +MockDNS = Struct.new( + # https://rubydoc.info/gems/google-cloud-dns/0.35.1/Google/Cloud/Dns + :change, :credentials, :project, :record, :zone, + keyword_init: true +) do + def zone(zone) + self.zone = zone + end +end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 603f438..6cd4447 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -10,6 +10,7 @@ require 'vmpooler' require 'redis' require 'vmpooler/metrics' require 'computeservice_helper' +require 'dnsservice_helper' def project_root_dir File.dirname(File.dirname(__FILE__)) diff --git a/spec/unit/providers/gce_spec.rb b/spec/unit/providers/gce_spec.rb index 1e219b9..c4b8f25 100644 --- a/spec/unit/providers/gce_spec.rb +++ b/spec/unit/providers/gce_spec.rb @@ -18,11 +18,6 @@ describe 'Vmpooler::PoolManager::Provider::Gce' do :config: max_tries: 3 retry_factor: 10 - :dns_configs: - :gcp-clouddns: - project: vmpooler-test - domain: vmpooler.example.com - dns_zone_resource_name: vmpooler-example-com :providers: :gce: connection_pool_timeout: 1 @@ -37,7 +32,6 @@ describe 'Vmpooler::PoolManager::Provider::Gce' do timeout: 10 ready_ttl: 1440 provider: 'gce' - dns_config: 'gcp-clouddns' machine_type: 'zones/#{zone}/machineTypes/e2-micro' EOT ) @@ -57,6 +51,8 @@ EOT subject { Vmpooler::PoolManager::Provider::Gce.new(config, logger, metrics, redis_connection_pool, 'gce', provider_options) } + before(:each) { allow(subject).to receive(:dns).and_return(MockDNS.new()) } + describe '#name' do it 'should be gce' do expect(subject.name).to eq('gce') @@ -65,6 +61,7 @@ EOT describe '#manual tests live' do context 'in itsysops' do + before(:each) { allow(subject).to receive(:dns).and_call_original } let(:vmname) { "instance-31" } let(:project) { 'vmpooler-test' } let(:config) { YAML.load(<<~EOT @@ -72,17 +69,14 @@ EOT :config: max_tries: 3 retry_factor: 10 - :dns_configs: - :gcp-clouddns: - project: vmpooler-test - domain: vmpooler.example.com - dns_zone_resource_name: vmpooler-example-com :providers: :gce: connection_pool_timeout: 1 project: '#{project}' zone: '#{zone}' network_name: 'projects/itsysopsnetworking/global/networks/shared1' + dns_zone_resource_name: 'vmpooler-test-puppet-net' + domain: 'vmpooler-test.puppet.net' :pools: - name: '#{poolname}' alias: [ 'mockpool' ] @@ -91,7 +85,6 @@ EOT timeout: 10 ready_ttl: 1440 provider: 'gce' - dns_config: 'gcp-clouddns' subnetwork_name: 'projects/itsysopsnetworking/regions/us-west1/subnetworks/vmpooler-test' machine_type: 'zones/#{zone}/machineTypes/e2-micro' disk_type: 'pd-ssd' @@ -99,6 +92,10 @@ EOT ) } skip 'gets a vm' do result = subject.create_vm(poolname, vmname) + #result = subject.destroy_vm(poolname, vmname) + # subject.get_vm(poolname, vmname) + subject.dns_teardown({'name' => vmname}) + # subject.dns_setup({'name' => vmname, 'ip' => '1.2.3.5'}) end end end @@ -270,6 +267,7 @@ EOT result = MockResult.new result.status = 'DONE' allow(connection).to receive(:insert_instance).and_return(result) + allow(subject).to receive(:dns_setup).and_return(true) end it 'should return a hash' do @@ -316,6 +314,7 @@ EOT result.status = 'DONE' allow(subject).to receive(:wait_for_operation).and_return(result) allow(connection).to receive(:delete_instance).and_return(result) + allow(subject).to receive(:dns_teardown).and_return(true) end it 'should return true' do @@ -350,20 +349,14 @@ EOT end describe '#vm_ready?' do - let(:domain) { 'vmpooler.example.com' } - before(:each) do - allow(subject).to receive(:domain).and_return('vmpooler.example.com') - end - + let(:domain) { nil } context 'When a VM is ready' do before(:each) do expect(subject).to receive(:open_socket).with(vmname, domain) end it 'should return true' do - redis_connection_pool.with_metrics do |redis| - expect(subject.vm_ready?(poolname, vmname, redis)).to be true - end + expect(subject.vm_ready?(poolname, vmname)).to be true end end @@ -373,9 +366,7 @@ EOT end it 'should return false' do - redis_connection_pool.with_metrics do |redis| - expect(subject.vm_ready?(poolname, vmname, redis)).to be false - end + expect(subject.vm_ready?(poolname, vmname)).to be false end end end @@ -595,6 +586,7 @@ EOT before(:each) do allow(subject).to receive(:connect_to_gce).and_return(connection) + allow(subject).to receive(:dns_teardown).and_return(true) end context 'with empty allowlist' do diff --git a/update-gemfile-lock b/update-gemfile-lock index 31986cc..46c4873 100755 --- a/update-gemfile-lock +++ b/update-gemfile-lock @@ -3,5 +3,5 @@ # The container tag should closely match what is used in `docker/Dockerfile` in vmpooler-deployment docker run -it --rm \ -v $(pwd):/app \ - jruby:9.4.3.0-jdk11 \ + jruby:9.4.1.0-jdk11 \ /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends git make netbase && cd /app && gem install bundler && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' diff --git a/vmpooler-provider-gce.gemspec b/vmpooler-provider-gce.gemspec index ac5f4b8..3be4d75 100644 --- a/vmpooler-provider-gce.gemspec +++ b/vmpooler-provider-gce.gemspec @@ -16,8 +16,9 @@ Gem::Specification.new do |s| s.files = Dir[ "lib/**/*" ] s.require_paths = ["lib"] s.add_dependency "google-apis-compute_v1", "~> 0.14" - s.add_dependency "googleauth", ">= 0.16.2", "< 1.4.0" - s.add_dependency 'vmpooler', '~> 3.0' + s.add_dependency "googleauth", ">= 0.16.2", "< 1.3.0" + s.add_dependency "google-cloud-dns", "~> 0.35.1" + s.add_dependency 'vmpooler', '>= 1.3.0', '~> 2.3' # Testing dependencies s.add_development_dependency 'climate_control', '>= 0.2.0'