From 71b4edcd1de64da38f0d7d37272437e092ebfb9c Mon Sep 17 00:00:00 2001 From: Jake Spain Date: Mon, 23 Jan 2023 17:15:18 -0500 Subject: [PATCH] Consolidate dockerfiles and add docs for development This will reduce the number of files that need to be managed and dependencies that need updated at the same time. --- .github/dependabot.yml | 32 +-- .gitignore | 3 + README.md | 101 +++++++- build-chart.sh | 4 + docker/.dockerignore | 6 + docker/{prod-all-providers => }/Dockerfile | 5 +- docker/Dockerfile.local | 43 ++++ docker/Gemfile | 12 + docker/Gemfile.local | 6 + docker/{prod-all-providers => }/Gemfile.lock | 0 docker/data/redis/.gitkeep | 0 docker/data/vmpooler.yaml | 37 +++ docker/dev/.dockerignore | 3 - docker/dev/Dockerfile | 38 ---- docker/dev/Dockerfile-aio | 39 ---- docker/dev/docker-compose.yml | 71 ------ docker/docker-compose.local.yml | 79 +++++++ docker/docker-compose.yml | 89 ++++++++ docker/{dev => }/docker-entrypoint.sh | 0 docker/prod-all-providers/.bundle/config | 2 - docker/prod-all-providers/.dockerignore | 3 - docker/prod-all-providers/Gemfile | 6 - docker/prod-all-providers/docker-compose.yml | 71 ------ .../prod-all-providers/docker-entrypoint.sh | 6 - .../prod-all-providers/update-gemfile-lock.sh | 6 - docker/test-all-providers/.bundle/config | 2 - docker/test-all-providers/.dockerignore | 3 - docker/test-all-providers/Dockerfile | 38 ---- docker/test-all-providers/Gemfile | 7 - docker/test-all-providers/Gemfile.lock | 215 ------------------ docker/test-all-providers/docker-compose.yml | 71 ------ .../test-all-providers/docker-entrypoint.sh | 6 - ...te-gemfile-lock.sh => update-gemfile-lock} | 0 update-changelog | 5 + update-chart-lock | 4 + 35 files changed, 386 insertions(+), 627 deletions(-) create mode 100755 build-chart.sh create mode 100644 docker/.dockerignore rename docker/{prod-all-providers => }/Dockerfile (92%) create mode 100644 docker/Dockerfile.local create mode 100644 docker/Gemfile create mode 100644 docker/Gemfile.local rename docker/{prod-all-providers => }/Gemfile.lock (100%) create mode 100644 docker/data/redis/.gitkeep create mode 100644 docker/data/vmpooler.yaml delete mode 100644 docker/dev/.dockerignore delete mode 100644 docker/dev/Dockerfile delete mode 100644 docker/dev/Dockerfile-aio delete mode 100644 docker/dev/docker-compose.yml create mode 100644 docker/docker-compose.local.yml create mode 100644 docker/docker-compose.yml rename docker/{dev => }/docker-entrypoint.sh (100%) delete mode 100644 docker/prod-all-providers/.bundle/config delete mode 100644 docker/prod-all-providers/.dockerignore delete mode 100644 docker/prod-all-providers/Gemfile delete mode 100644 docker/prod-all-providers/docker-compose.yml delete mode 100644 docker/prod-all-providers/docker-entrypoint.sh delete mode 100755 docker/prod-all-providers/update-gemfile-lock.sh delete mode 100644 docker/test-all-providers/.bundle/config delete mode 100644 docker/test-all-providers/.dockerignore delete mode 100644 docker/test-all-providers/Dockerfile delete mode 100644 docker/test-all-providers/Gemfile delete mode 100644 docker/test-all-providers/Gemfile.lock delete mode 100644 docker/test-all-providers/docker-compose.yml delete mode 100644 docker/test-all-providers/docker-entrypoint.sh rename docker/{test-all-providers/update-gemfile-lock.sh => update-gemfile-lock} (100%) create mode 100755 update-changelog create mode 100755 update-chart-lock diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d228804..76020ad 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,35 +1,13 @@ version: 2 updates: -# /docker/dev folder - package-ecosystem: docker - directory: "/docker/dev/" + directory: "/docker/" schedule: - interval: daily - time: "13:00" + interval: weekly open-pull-requests-limit: 10 -# /docker/prod-all-providers folder + - package-ecosystem: bundler - directory: "/docker/prod-all-providers/" + directory: "/docker/" schedule: - interval: daily - time: "13:00" - open-pull-requests-limit: 10 -- package-ecosystem: docker - directory: "/docker/prod-all-providers/" - schedule: - interval: daily - time: "13:00" - open-pull-requests-limit: 10 -# /docker/test-all-providers folder -- package-ecosystem: bundler - directory: "/docker/test-all-providers/" - schedule: - interval: daily - time: "13:00" - open-pull-requests-limit: 10 -- package-ecosystem: docker - directory: "/docker/test-all-providers/" - schedule: - interval: daily - time: "13:00" + interval: weekly open-pull-requests-limit: 10 diff --git a/.gitignore b/.gitignore index f86fe23..ba34006 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ .vagrant/ .idea/ helm-charts/vmpooler/charts/ +/docker/data/redis/** +!docker/data/**/*.yaml +!docker/data/**/.gitkeep \ No newline at end of file diff --git a/README.md b/README.md index d15f5e2..7bf0423 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,35 @@ # vmpooler-deployment +- [vmpooler-deployment](#vmpooler-deployment) + - [VMPooler Components](#vmpooler-components) + - [Docker Registry](#docker-registry) + - [Helm Repository](#helm-repository) + - [Adding / updating charts](#adding--updating-charts) + - [Development](#development) + - [Docker Compose URLs](#docker-compose-urls) + - [Deploy Chart for Testing](#deploy-chart-for-testing) + - [Releasing](#releasing) + - [Contributing](#contributing) + - [License](#license) + This repo contains Dockerfiles and a Helm chart that can be used to deploy [VMPooler](https://github.com/puppetlabs/vmpooler). The Release Engineering team at Puppet uses the code here as part of operating our VMPooler instances. +## VMPooler Components + +The docker image gnerated and hosted by this project contain the following VMPooler components: + +- [VMPooler Core](https://github.com/puppetlabs/vmpooler) +- [VMPooler EC2 Provider](https://github.com/puppetlabs/vmpooler-provider-ec2) +- [VMPooler GCE Provider](https://github.com/puppetlabs/vmpooler-provider-gce) +- [VMPooler vSphere Provider](https://github.com/puppetlabs/vmpooler-provider-vsphere) + ## Docker Registry The GitHub Actions in this repository publish images to GitHub Packages. You can browse the VMPooler containers [here](https://github.com/puppetlabs/vmpooler-deployment/pkgs/container/vmpooler-deployment%2Fvmpooler). -Containers are organized using image tags: +The vmpooler officially released docker image tags use a semantic version, where the version increments correlate to the increments in any of the gems at `docker/Gemfile`. -- `x.y.z-prod-all-providers` represents images built using the Dockerfile in `docker/prod-all-providers`. These images use the Gemfile in the same directory to pull in current versions of the `vmpooler` gem and the gems for all known production-grade providers. +Image tags starting with `pr-` are generated when opening and pushing to a pull request and will periodically be cleaned up. ## Helm Repository @@ -21,14 +42,74 @@ $ helm repo add vmpooler-deployment https://puppetlabs.github.io/vmpooler-deploy ### Adding / updating charts -```bash -cd docs/ -helm package ../helm-charts/* -helm repo index --url https://puppetlabs.github.io/vmpooler-deployment/ . -git add . -cd .. -git commit -a -``` +Make the desired changes to the helm chart in helm-charts/vmpooler and run `./update-chart`. + +## Development + +Prerequisites: + +- [Docker](https://docs.docker.com/engine/install/) +- [Docker Compose](https://docs.docker.com/compose/install/) + +1. Become familiar with the configuration file `docker/vmpooler.yaml` as described in [VMPooler](https://github.com/puppetlabs/vmpooler) Core. + - The default configuration file only enables the dummy provider built into [VMPooler](https://github.com/puppetlabs/vmpooler) Core. See each project below for documentation on how to use provider specific options: + - [vmpooler-provider-ec2](https://github.com/puppetlabs/vmpooler-provider-ec2) + - [vmpooler-provider-gce](https://github.com/puppetlabs/vmpooler-provider-gce) + - [vmpooler-provider-vsphere](https://github.com/puppetlabs/vmpooler-provider-vsphere) +2. Chose a Development method: + - Develop via local source: + 1. Clone all of the known vmpooler projects listed at [vmpooler-components](#vmpooler-components) under a common directory, for example: + + ```bash + |-- vmpooler-projects + | |--vmpooler-deployment + | |--vmpooler-provider-ec2 + | |--vmpooler-provider-gce + | |--vmpooler-provider-vsphere + ``` + + 2. From the `vmpooler-projects` directory run `docker build -t vmpooler-local -f vmpooler-deployment/docker/Dockerfile.local .` + 3. Run `docker compose -f vmpooler-deployment/docker/docker-compose.local.yml up` + + - Develop via Git source: + 1. For the component(s) you are developing on, commit and push changes to a branch. + 2. Change to the `docker` directory and modify the `Gemfile` as needed to pull the gem(s) from your git branch. + 3. Run `./update-gemfile-lock` to update the `Gemfile.lock` + 4. Run `docker compose build && docker compose up`. + +When a dependency Helm chart is updated, be sure to run `./update-chart-lock` to update the lockfile, otherwise the test and release workflows will fail. + +### Docker Compose URLs + +These are the default ports used in the docker compose file, to change them edit the `ports` key under the desired service in either `docker/docker-compose.yml` or `docker/docker-compose.local.yml`. + +| App/Endpoint | Path | +|--------------------|-------------------------------------------------------------| +| Redis Server | `localhost:6379` (Password: `vmpooler`) | +| Redis Commander | (Credentials: `admin:admin`) | +| Jaeger | +| VMPooler API | | +| VMPooler Dashboard | | +| Metrics (API) | | +| Metrics (Manager) | | + +### Deploy Chart for Testing + +Helm charts are hosted with GitHub Pages since GitHub Packages does not support Helm, so you will need to build the chart locally and push the chart to your own helm repository. Follow these steps to generate a chart to host internally: + +1. Update the "appVersion" key in `helm-charts/vmpooler/Chart.yaml` to the docker image tag that was generated by opening a pull request. +2. Update the "version" key to a pre-release tag, something like "x.y.z-rc.1", then run `./build-chart` to build the chart locally, and upload it to your desired location. + +## Releasing + +Create a GitHub tag and release, publish a new docker image, and helm chart by opening a release prep pull request and running the release action: + +1. Bump the "appVersion" key in `helm-charts/vmpooler/Chart.yaml` appropriately based on changes to `docker/Gemfile` abd `docker/Gemfile.lock` in merged pull requests since the last release. +2. Bump the "version" key in `helm-charts/vmpooler/Chart.yaml` appropriately based on changes to the chart itself and increments of the "appVersion" in merged pull requests since the last release. +3. Run `./update-changelog` to update `CHANGELOG.md`. +4. Commit and push changes to a new branch, then open a pull request against `main` and be sure to add the "maintenance" label. +5. After the pull request is approved and merged, then navigate to Actions --> Docker and Helm Release --> run workflow --> Branch: main --> Run workflow. + - This action will push a tagged docker image to the GitHub container registry and helm chart to GitHub pages. ## Contributing diff --git a/build-chart.sh b/build-chart.sh new file mode 100755 index 0000000..ffe8010 --- /dev/null +++ b/build-chart.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +docker run --rm -it -v $(pwd):/apps --entrypoint /bin/sh alpine/helm \ + -c 'helm repo add bitnami https://charts.bitnami.com/bitnami && cd /apps/helm-charts/vmpooler && helm dependency update && cd /apps/docs && helm package ../helm-charts/* && helm repo index --url https://puppetlabs.github.io/vmpooler-deployment/ .' diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 0000000..5052e88 --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,6 @@ +# Exclude everything by default; explicitly include necessary files +* +!Gemfile +!Gemfile.lock +!Gemfile.local +!docker-entrypoint.sh diff --git a/docker/prod-all-providers/Dockerfile b/docker/Dockerfile similarity index 92% rename from docker/prod-all-providers/Dockerfile rename to docker/Dockerfile index 1cb1d50..14ec768 100644 --- a/docker/prod-all-providers/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ # RUN: # docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler -FROM jruby:9.3.6-jdk +FROM amd64/jruby:9.3.6-jdk LABEL org.opencontainers.image.description "VMPooler plus all known providers" @@ -23,8 +23,7 @@ RUN apt-get update -qq && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* -RUN gem install bundler && \ - gem update --system 3.2.3 +RUN gem install bundler COPY ./docker-entrypoint.sh /usr/local/bin/ diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local new file mode 100644 index 0000000..970cefe --- /dev/null +++ b/docker/Dockerfile.local @@ -0,0 +1,43 @@ +# For local development of vmpooler core and provider gems instead of using a +# git source ensure the following directory structure: +# |-- vmpooler-projects +# | |--vmpooler-deployment +# | |--vmpooler-provider-ec2 +# | |--vmpooler-provider-gce +# | |--vmpooler-provider-vsphere +# +# Then change directory into vmpooler-projects and build with: + + +FROM amd64/jruby:9.3.6-jdk + +LABEL org.opencontainers.image.description "VMPooler plus all known providers" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LOGFILE=/dev/stdout +ENV RACK_ENV=production + +RUN apt-get update -qq && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends make git netbase && \ + apt-get clean autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +COPY vmpooler /opt/vmpooler +COPY vmpooler-provider-ec2 /opt/vmpooler-provider-ec2 +COPY vmpooler-provider-gce /opt/vmpooler-provider-gce +COPY vmpooler-provider-vsphere /opt/vmpooler-provider-vsphere + +RUN gem install bundler + +COPY vmpooler-deployment/docker/docker-entrypoint.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +COPY vmpooler-deployment/docker/Gemfile.local ./Gemfile + +RUN bundle config set --local jobs 3 && \ + bundle install + +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/Gemfile b/docker/Gemfile new file mode 100644 index 0000000..9bfbe83 --- /dev/null +++ b/docker/Gemfile @@ -0,0 +1,12 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gem 'vmpooler', '~> 2.4' +gem 'vmpooler-provider-ec2', '~> 0.0.1' +gem 'vmpooler-provider-gce', '~> 0.4' +gem 'vmpooler-provider-vsphere', '~> 1.6' + +# For development install via a git branch use something like: +# gem 'vmpooler', git: 'https://github.com/puppetlabs/vmpooler.git', branch: 'my-feature' +# gem 'vmpooler-provider-ec2', git: 'https://github.com/puppetlabs/vmpooler-provider-ec2.git', branch: 'my-feature' +# gem 'vmpooler-provider-gce', git: 'https://github.com/puppetlabs/vmpooler-provider-gce.git', branch: 'my-feature' +# gem 'vmpooler-provider-vsphere', git: 'https://github.com/puppetlabs/vmpooler-provider-vsphere.git', branch: 'my-feature' diff --git a/docker/Gemfile.local b/docker/Gemfile.local new file mode 100644 index 0000000..5ba111f --- /dev/null +++ b/docker/Gemfile.local @@ -0,0 +1,6 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +gem 'vmpooler', path: '/opt/vmpooler' +gem 'vmpooler-provider-ec2', path: '/opt/vmpooler-provider-ec2' +gem 'vmpooler-provider-gce', path: '/opt/vmpooler-provider-gce' +gem 'vmpooler-provider-vsphere', path: '/opt/vmpooler-provider-vsphere' \ No newline at end of file diff --git a/docker/prod-all-providers/Gemfile.lock b/docker/Gemfile.lock similarity index 100% rename from docker/prod-all-providers/Gemfile.lock rename to docker/Gemfile.lock diff --git a/docker/data/redis/.gitkeep b/docker/data/redis/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker/data/vmpooler.yaml b/docker/data/vmpooler.yaml new file mode 100644 index 0000000..4238587 --- /dev/null +++ b/docker/data/vmpooler.yaml @@ -0,0 +1,37 @@ +--- +:providers: + :dummy: + filename: '/tmp/dummy-backing.yaml' + +:prometheus: + prefix: 'vmpooler-local' + +:redis: + server: 'redis-server' + +:auth: + provider: 'dummy' + +:config: + site_name: 'vmpooler' + logfile: '/var/log/vmpooler.log' + task_limit: 10 + timeout: 15 + vm_checktime: 1 + vm_lifetime: 12 + vm_lifetime_auth: 24 + allowed_tags: + - 'created_by' + - 'project' + domain: 'example.com' + # domain: 'localhost' # Flip these out for local requests + +:pools: + - name: 'debian-10-x86_64' + template: 'Templates/debian-7-x86_64' + folder: 'Pooled VMs/debian-7-x86_64' + datastore: 'vmstorage' + size: 5 + timeout: 15 + ready_ttl: 1440 + provider: dummy diff --git a/docker/dev/.dockerignore b/docker/dev/.dockerignore deleted file mode 100644 index 8dad93e..0000000 --- a/docker/dev/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.bundle/ -vendor/ -update-gemfile-lock.sh diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile deleted file mode 100644 index ffea807..0000000 --- a/docker/dev/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler - -FROM jruby:9.3.6-jdk - -ENV DEBIAN_FRONTEND=noninteractive -ENV RACK_ENV=production - -RUN apt-get update -qq && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends make git netbase && \ - apt-get clean autoclean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -COPY docker/docker-entrypoint.sh /usr/local/bin/ -COPY ./Gemfile ./ -COPY ./vmpooler.gemspec ./ -COPY ./lib/vmpooler/version.rb ./lib/vmpooler/version.rb - -RUN gem install bundler && \ - bundle config set --local jobs 3 && \ - bundle install - -COPY ./ ./ - -RUN gem build vmpooler.gemspec && \ - gem install vmpooler*.gem && \ - chmod +x /usr/local/bin/docker-entrypoint.sh - -ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/dev/Dockerfile-aio b/docker/dev/Dockerfile-aio deleted file mode 100644 index 2401e5d..0000000 --- a/docker/dev/Dockerfile-aio +++ /dev/null @@ -1,39 +0,0 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler - -FROM jruby:9.3.6-jdk - -RUN mkdir -p /var/lib/vmpooler - -WORKDIR /var/lib/vmpooler - -RUN echo "deb http://httpredir.debian.org/debian jessie main" >/etc/apt/sources.list.d/jessie-main.list - -RUN apt-get update -qq && \ - apt-get install -y --no-install-recommends make git redis-server && \ - apt-get clean autoclean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -ADD Gemfile* /var/lib/vmpooler/ - -RUN bundle install --system - -RUN ln -s /opt/jruby/bin/jruby /usr/bin/jruby - -COPY . /var/lib/vmpooler - -ENV VMPOOLER_LOG /var/log/vmpooler.log - -CMD \ - /etc/init.d/redis-server start \ - && /var/lib/vmpooler/scripts/vmpooler_init.sh start \ - && while [ ! -f ${VMPOOLER_LOG} ]; do sleep 1; done ; \ - tail -f ${VMPOOLER_LOG} diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml deleted file mode 100644 index 404deff..0000000 --- a/docker/dev/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -# For local development run with a dummy provider -version: '3.8' -services: - vmpooler-api: - build: - context: ../ - dockerfile: docker/Dockerfile_local - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8080:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: api - depends_on: - - redislocal - vmpooler-manager: - build: - context: ../ - dockerfile: docker/Dockerfile_local - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8081:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: manager - depends_on: - - redislocal - redislocal: - image: redis - # Uncomment this if you don't want the redis data to persist - #command: "redis-server --save '' --appendonly no" - ports: - - "6379:6379" - networks: - - redis-net - jaeger-aio: - image: jaegertracing/all-in-one:1.18 - ports: - - "14250:14250" - - "8082:16686" - networks: - - redis-net - user: '1001' - read_only: true - cap_drop: - - ALL - -networks: - redis-net: diff --git a/docker/docker-compose.local.yml b/docker/docker-compose.local.yml new file mode 100644 index 0000000..f5788bf --- /dev/null +++ b/docker/docker-compose.local.yml @@ -0,0 +1,79 @@ +# For local development run with a dummy provider +version: '3.8' +services: + redis-server: + # This should match the major version used in the vmpooler helm chart + image: redis:6 + command: "redis-server --requirepass vmpooler" + volumes: + - type: bind + source: ./data/redis + target: /data + ports: + - "6379:6379" + + redis-commander: + image: rediscommander/redis-commander:latest + environment: + - REDIS_HOST=redis-server + - REDIS_PASSWORD=vmpooler + - HTTP_USER=admin + - HTTP_PASSWORD=admin + ports: + - 8080:8081 + links: + - redis-server + + jaeger-aio: + image: jaegertracing/all-in-one:1 + ports: + - "14250:14250" + - "8081:16686" + user: '1001' + read_only: true + cap_drop: + - ALL + + vmpooler-api: + # See image build instructions in README and Dockerfile.local + image: vmpooler-local + volumes: + - type: bind + source: ./data/vmpooler.yaml + target: /etc/vmpooler/vmpooler.yaml + ports: + - "8082:4567" + environment: + - VMPOOLER_DEBUG=true # for use of dummy auth + - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml + - REDIS_SERVER=redis-server + - REDIS_PASSWORD=vmpooler + - LOGFILE=/dev/null + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces + command: api + links: + - redis-server + - jaeger-aio + + vmpooler-manager: + # See image build instructions in README and Dockerfile.local + image: vmpooler-local + volumes: + - type: bind + source: ./data/vmpooler.yaml + target: /etc/vmpooler/vmpooler.yaml + ports: + - "8083:4567" + environment: + - VMPOOLER_DEBUG=true # for use of dummy auth + - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml + - REDIS_SERVER=redis-server + - REDIS_PASSWORD=vmpooler + - LOGFILE=/dev/null + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces + command: manager + links: + - redis-server + - jaeger-aio diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..3a81546 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,89 @@ +# For local development run with a dummy provider +version: '3.8' +services: + redis-server: + # This should match the major version used in the vmpooler helm chart + image: redis:6 + command: "redis-server --requirepass vmpooler" + volumes: + - type: bind + source: ./data/redis + target: /data + ports: + - "6379:6379" + + redis-commander: + image: rediscommander/redis-commander:latest + environment: + - REDIS_HOST=redis-server + - REDIS_PASSWORD=vmpooler + - HTTP_USER=admin + - HTTP_PASSWORD=admin + ports: + - 8080:8081 + links: + - redis-server + + jaeger-aio: + image: jaegertracing/all-in-one:1 + ports: + - "14250:14250" + - "8081:16686" + user: '1001' + read_only: true + cap_drop: + - ALL + + vmpooler-api: + # Uncomment image and comment out build, which will allow you to use an + # image with local gem source instead of from a git source. + # See build instructions in Dockerfile.local + # image: vmpooler-local + build: + context: ./ + dockerfile: Dockerfile + volumes: + - type: bind + source: ./data/vmpooler.yaml + target: /etc/vmpooler/vmpooler.yaml + ports: + - "8082:4567" + environment: + - VMPOOLER_DEBUG=true # for use of dummy auth + - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml + - REDIS_SERVER=redis-server + - REDIS_PASSWORD=vmpooler + - LOGFILE=/dev/null + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces + command: api + links: + - redis-server + - jaeger-aio + + vmpooler-manager: + # Uncomment image and comment out build, which will allow you to use an + # image with local gem source instead of from a git source. + # See build instructions in Dockerfile.local + # image: vmpooler-local + build: + context: ./ + dockerfile: Dockerfile + volumes: + - type: bind + source: ./data/vmpooler.yaml + target: /etc/vmpooler/vmpooler.yaml + ports: + - "8083:4567" + environment: + - VMPOOLER_DEBUG=true # for use of dummy auth + - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml + - REDIS_SERVER=redis-server + - REDIS_PASSWORD=vmpooler + - LOGFILE=/dev/null + - VMPOOLER_TRACING_ENABLED=true + - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces + command: manager + links: + - redis-server + - jaeger-aio diff --git a/docker/dev/docker-entrypoint.sh b/docker/docker-entrypoint.sh similarity index 100% rename from docker/dev/docker-entrypoint.sh rename to docker/docker-entrypoint.sh diff --git a/docker/prod-all-providers/.bundle/config b/docker/prod-all-providers/.bundle/config deleted file mode 100644 index 2369228..0000000 --- a/docker/prod-all-providers/.bundle/config +++ /dev/null @@ -1,2 +0,0 @@ ---- -BUNDLE_PATH: "vendor/bundle" diff --git a/docker/prod-all-providers/.dockerignore b/docker/prod-all-providers/.dockerignore deleted file mode 100644 index 8dad93e..0000000 --- a/docker/prod-all-providers/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.bundle/ -vendor/ -update-gemfile-lock.sh diff --git a/docker/prod-all-providers/Gemfile b/docker/prod-all-providers/Gemfile deleted file mode 100644 index 688d20e..0000000 --- a/docker/prod-all-providers/Gemfile +++ /dev/null @@ -1,6 +0,0 @@ -source ENV['GEM_SOURCE'] || 'https://rubygems.org' - -gem 'vmpooler', '~> 2.4' -gem 'vmpooler-provider-ec2', '~> 0.0.1' -gem 'vmpooler-provider-gce', '~> 0.4' -gem 'vmpooler-provider-vsphere', '~> 1.6' diff --git a/docker/prod-all-providers/docker-compose.yml b/docker/prod-all-providers/docker-compose.yml deleted file mode 100644 index d93a8bb..0000000 --- a/docker/prod-all-providers/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -# For local development run with a dummy provider -version: '3.8' -services: - vmpooler-api: - build: - context: ./ - dockerfile: Dockerfile - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8080:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: api - depends_on: - - redislocal - vmpooler-manager: - build: - context: ./ - dockerfile: Dockerfile - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8081:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: manager - depends_on: - - redislocal - redislocal: - image: redis - # Uncomment this if you don't want the redis data to persist - #command: "redis-server --save '' --appendonly no" - ports: - - "6379:6379" - networks: - - redis-net - jaeger-aio: - image: jaegertracing/all-in-one:1.18 - ports: - - "14250:14250" - - "8082:16686" - networks: - - redis-net - user: '1001' - read_only: true - cap_drop: - - ALL - -networks: - redis-net: diff --git a/docker/prod-all-providers/docker-entrypoint.sh b/docker/prod-all-providers/docker-entrypoint.sh deleted file mode 100644 index 07d11de..0000000 --- a/docker/prod-all-providers/docker-entrypoint.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -e - -set -- bundle exec vmpooler "$@" - -exec "$@" diff --git a/docker/prod-all-providers/update-gemfile-lock.sh b/docker/prod-all-providers/update-gemfile-lock.sh deleted file mode 100755 index 14eadf5..0000000 --- a/docker/prod-all-providers/update-gemfile-lock.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -docker run -it --rm \ - -v $(pwd):/app \ - $(grep ^FROM ./Dockerfile |cut -d ' ' -f2) \ - /bin/bash -c 'apt-get update -qq && apt-get install -y --no-install-recommends make git && cd /app && gem install bundler && gem update --system 3.2.3 && bundle install --jobs 3 && bundle update; echo "LOCK_FILE_UPDATE_EXIT_CODE=$?"' diff --git a/docker/test-all-providers/.bundle/config b/docker/test-all-providers/.bundle/config deleted file mode 100644 index 2369228..0000000 --- a/docker/test-all-providers/.bundle/config +++ /dev/null @@ -1,2 +0,0 @@ ---- -BUNDLE_PATH: "vendor/bundle" diff --git a/docker/test-all-providers/.dockerignore b/docker/test-all-providers/.dockerignore deleted file mode 100644 index 8dad93e..0000000 --- a/docker/test-all-providers/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.bundle/ -vendor/ -update-gemfile-lock.sh diff --git a/docker/test-all-providers/Dockerfile b/docker/test-all-providers/Dockerfile deleted file mode 100644 index 010012a..0000000 --- a/docker/test-all-providers/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Run vmpooler in a Docker container! Configuration can either be embedded -# and built within the current working directory, or stored in a -# VMPOOLER_CONFIG environment value and passed to the Docker daemon. -# -# BUILD: -# docker build -t vmpooler . -# -# RUN: -# docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler - -FROM jruby:9.3.6-jdk - -LABEL org.opencontainers.image.description "VMPooler and known providers pulled directly from Git" - -ENV DEBIAN_FRONTEND=noninteractive -ENV LOGFILE=/dev/stdout -ENV RACK_ENV=production - -RUN apt-get update -qq && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends make git netbase && \ - apt-get clean autoclean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -RUN gem install bundler && \ - gem update --system 3.2.3 - -COPY ./docker-entrypoint.sh /usr/local/bin/ - -RUN chmod +x /usr/local/bin/docker-entrypoint.sh - -COPY ./Gemfile* ./ - -RUN bundle config set --local jobs 3 && \ - bundle install - -ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/test-all-providers/Gemfile b/docker/test-all-providers/Gemfile deleted file mode 100644 index 10d1eb9..0000000 --- a/docker/test-all-providers/Gemfile +++ /dev/null @@ -1,7 +0,0 @@ -source ENV['GEM_SOURCE'] || 'https://rubygems.org' - -gem 'vmpooler', git: 'https://github.com/puppetlabs/vmpooler.git' -gem 'vmpooler-provider-ec2', git: 'https://github.com/puppetlabs/vmpooler-provider-ec2.git' -gem 'vmpooler-provider-gce', git: 'https://github.com/puppetlabs/vmpooler-provider-gce.git' -gem 'vmpooler-provider-vsphere', git: 'https://github.com/puppetlabs/vmpooler-provider-vsphere.git' - diff --git a/docker/test-all-providers/Gemfile.lock b/docker/test-all-providers/Gemfile.lock deleted file mode 100644 index 805c54c..0000000 --- a/docker/test-all-providers/Gemfile.lock +++ /dev/null @@ -1,215 +0,0 @@ -GIT - remote: https://github.com/puppetlabs/vmpooler-provider-ec2.git - revision: 13d0de6dc035c46dcd099d0b5e9b02bc19e8fcdd - specs: - vmpooler-provider-ec2 (0.0.1) - aws-sdk-ec2 (~> 1) - net-ssh (~> 6.2.0.rc2) - -GIT - remote: https://github.com/puppetlabs/vmpooler-provider-gce.git - revision: 46b9c5aef06184f8a2bd0be6086ae2256b19020a - specs: - vmpooler-provider-gce (0.3.0) - google-apis-compute_v1 (~> 0.14) - google-cloud-dns (~> 0.35.1) - googleauth (>= 0.16.2, < 1.2.0) - -GIT - remote: https://github.com/puppetlabs/vmpooler-provider-vsphere.git - revision: 82e35dfb54705f9a495b64498779da6cfb322f16 - specs: - vmpooler-provider-vsphere (1.6.0) - rbvmomi (>= 2.1, < 4.0) - -GIT - remote: https://github.com/puppetlabs/vmpooler.git - revision: b9a1bb74017c417526cde6586b8dd333435ff74c - specs: - vmpooler (2.4.0) - concurrent-ruby (~> 1.1) - connection_pool (~> 2.2) - deep_merge (~> 1.2) - net-ldap (~> 0.16) - opentelemetry-exporter-jaeger (= 0.20.1) - opentelemetry-instrumentation-concurrent_ruby (= 0.19.2) - opentelemetry-instrumentation-http_client (= 0.19.4) - opentelemetry-instrumentation-redis (= 0.21.3) - opentelemetry-instrumentation-sinatra (= 0.19.3) - opentelemetry-resource_detectors (= 0.19.1) - opentelemetry-sdk (~> 1.0, >= 1.0.2) - pickup (~> 0.0.11) - prometheus-client (~> 2.0) - puma (~> 5.0, >= 5.0.4) - rack (~> 2.2) - rake (~> 13.0) - redis (~> 4.1) - sinatra (~> 2.0) - spicy-proton (~> 2.1) - statsd-ruby (~> 1.4) - -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - aws-eventstream (1.2.0) - aws-partitions (1.610.0) - aws-sdk-core (3.131.3) - aws-eventstream (~> 1, >= 1.0.2) - aws-partitions (~> 1, >= 1.525.0) - aws-sigv4 (~> 1.1) - jmespath (~> 1, >= 1.6.1) - aws-sdk-ec2 (1.324.0) - aws-sdk-core (~> 3, >= 3.127.0) - aws-sigv4 (~> 1.1) - aws-sigv4 (1.5.1) - aws-eventstream (~> 1, >= 1.0.2) - bindata (2.4.10) - builder (3.2.4) - concurrent-ruby (1.1.10) - connection_pool (2.2.5) - declarative (0.0.20) - deep_merge (1.2.2) - faraday (2.3.0) - faraday-net_http (~> 2.0) - ruby2_keywords (>= 0.0.4) - faraday-net_http (2.0.3) - google-apis-compute_v1 (0.44.0) - google-apis-core (>= 0.7, < 2.a) - google-apis-core (0.7.0) - addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.16.2, < 2.a) - httpclient (>= 2.8.1, < 3.a) - mini_mime (~> 1.0) - representable (~> 3.0) - retriable (>= 2.0, < 4.a) - rexml - webrick - google-apis-dns_v1 (0.23.0) - google-apis-core (>= 0.7, < 2.a) - google-cloud-core (1.6.0) - google-cloud-env (~> 1.0) - google-cloud-errors (~> 1.0) - google-cloud-dns (0.35.1) - google-apis-dns_v1 (~> 0.1) - google-cloud-core (~> 1.6) - googleauth (>= 0.16.2, < 2.a) - zonefile (~> 1.04) - google-cloud-env (1.6.0) - faraday (>= 0.17.3, < 3.0) - google-cloud-errors (1.2.0) - googleauth (1.1.3) - faraday (>= 0.17.3, < 3.a) - jwt (>= 1.4, < 3.0) - memoist (~> 0.16) - multi_json (~> 1.11) - os (>= 0.9, < 2.0) - signet (>= 0.16, < 2.a) - httpclient (2.8.3) - jmespath (1.6.1) - json (2.6.2-java) - jwt (2.4.1) - memoist (0.16.2) - mini_mime (1.1.2) - multi_json (1.15.0) - mustermann (2.0.2) - ruby2_keywords (~> 0.0.1) - net-ldap (0.17.1) - net-ssh (6.2.0.rc2) - nio4r (2.5.8-java) - nokogiri (1.13.8-java) - racc (~> 1.4) - opentelemetry-api (1.0.2) - opentelemetry-common (0.19.6) - opentelemetry-api (~> 1.0) - opentelemetry-exporter-jaeger (0.20.1) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.19.2) - opentelemetry-sdk (~> 1.0) - thrift - opentelemetry-instrumentation-base (0.19.0) - opentelemetry-api (~> 1.0) - opentelemetry-instrumentation-concurrent_ruby (0.19.2) - opentelemetry-api (~> 1.0) - opentelemetry-instrumentation-base (~> 0.19.0) - opentelemetry-instrumentation-http_client (0.19.4) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.19.3) - opentelemetry-instrumentation-base (~> 0.19.0) - opentelemetry-instrumentation-redis (0.21.3) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.19.3) - opentelemetry-instrumentation-base (~> 0.19.0) - opentelemetry-instrumentation-sinatra (0.19.3) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.19.3) - opentelemetry-instrumentation-base (~> 0.19.0) - opentelemetry-registry (0.1.0) - opentelemetry-api (~> 1.0.1) - opentelemetry-resource_detectors (0.19.1) - google-cloud-env - opentelemetry-sdk - opentelemetry-sdk (1.1.0) - opentelemetry-api (~> 1.0) - opentelemetry-common (~> 0.19.3) - opentelemetry-registry (~> 0.1) - opentelemetry-semantic_conventions - opentelemetry-semantic_conventions (1.8.0) - opentelemetry-api (~> 1.0) - optimist (3.0.1) - os (1.1.4) - pickup (0.0.11) - prometheus-client (2.1.0) - public_suffix (4.0.7) - puma (5.6.4-java) - nio4r (~> 2.0) - racc (1.6.0-java) - rack (2.2.4) - rack-protection (2.2.2) - rack - rake (13.0.6) - rbvmomi (3.0.0) - builder (~> 3.2) - json (~> 2.3) - nokogiri (~> 1.10) - optimist (~> 3.0) - redis (4.7.1) - representable (3.2.0) - declarative (< 0.1.0) - trailblazer-option (>= 0.1.1, < 0.2.0) - uber (< 0.2.0) - retriable (3.1.2) - rexml (3.2.5) - ruby2_keywords (0.0.5) - signet (0.17.0) - addressable (~> 2.8) - faraday (>= 0.17.5, < 3.a) - jwt (>= 1.5, < 3.0) - multi_json (~> 1.10) - sinatra (2.2.2) - mustermann (~> 2.0) - rack (~> 2.2) - rack-protection (= 2.2.2) - tilt (~> 2.0) - spicy-proton (2.1.14) - bindata (~> 2.3) - statsd-ruby (1.5.0) - thrift (0.16.0) - tilt (2.0.11) - trailblazer-option (0.1.2) - uber (0.1.0) - webrick (1.7.0) - zonefile (1.06) - -PLATFORMS - universal-java-1.8 - -DEPENDENCIES - vmpooler! - vmpooler-provider-ec2! - vmpooler-provider-gce! - vmpooler-provider-vsphere! - -BUNDLED WITH - 2.3.18 diff --git a/docker/test-all-providers/docker-compose.yml b/docker/test-all-providers/docker-compose.yml deleted file mode 100644 index d93a8bb..0000000 --- a/docker/test-all-providers/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -# For local development run with a dummy provider -version: '3.8' -services: - vmpooler-api: - build: - context: ./ - dockerfile: Dockerfile - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8080:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: api - depends_on: - - redislocal - vmpooler-manager: - build: - context: ./ - dockerfile: Dockerfile - volumes: - - type: bind - source: ${PWD}/vmpooler.yaml - target: /etc/vmpooler/vmpooler.yaml - ports: - - "8081:4567" - networks: - - redis-net - environment: - - VMPOOLER_DEBUG=true # for use of dummy auth - - VMPOOLER_CONFIG_FILE=/etc/vmpooler/vmpooler.yaml - - REDIS_SERVER=redislocal - - LOGFILE=/dev/null - - VMPOOLER_TRACING_ENABLED=true - - VMPOOLER_TRACING_JAEGER_HOST=http://jaeger-aio:14268/api/traces - image: vmpooler-local - command: manager - depends_on: - - redislocal - redislocal: - image: redis - # Uncomment this if you don't want the redis data to persist - #command: "redis-server --save '' --appendonly no" - ports: - - "6379:6379" - networks: - - redis-net - jaeger-aio: - image: jaegertracing/all-in-one:1.18 - ports: - - "14250:14250" - - "8082:16686" - networks: - - redis-net - user: '1001' - read_only: true - cap_drop: - - ALL - -networks: - redis-net: diff --git a/docker/test-all-providers/docker-entrypoint.sh b/docker/test-all-providers/docker-entrypoint.sh deleted file mode 100644 index 07d11de..0000000 --- a/docker/test-all-providers/docker-entrypoint.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -e - -set -- bundle exec vmpooler "$@" - -exec "$@" diff --git a/docker/test-all-providers/update-gemfile-lock.sh b/docker/update-gemfile-lock similarity index 100% rename from docker/test-all-providers/update-gemfile-lock.sh rename to docker/update-gemfile-lock diff --git a/update-changelog b/update-changelog new file mode 100755 index 0000000..3cfe85a --- /dev/null +++ b/update-changelog @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +docker run -it --rm -e CHANGELOG_GITHUB_TOKEN -v $(pwd):/usr/local/src/your-app \ + githubchangeloggenerator/github-changelog-generator:1.16.2 \ + github_changelog_generator --future-release $(yq .version helm-charts/vmpooler/Chart.yaml) \ No newline at end of file diff --git a/update-chart-lock b/update-chart-lock new file mode 100755 index 0000000..8748596 --- /dev/null +++ b/update-chart-lock @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +docker run --rm -it -v $(pwd):/apps --entrypoint /bin/sh alpine/helm \ + -c 'helm repo add bitnami https://charts.bitnami.com/bitnami && cd /apps/helm-charts/vmpooler && helm dependency update'