From e1c051df54f6011c41e77f8aafe161e1ebf0207a Mon Sep 17 00:00:00 2001 From: Heath Seals Date: Mon, 5 Aug 2019 09:51:46 -0500 Subject: [PATCH 1/4] (maint) setup updates --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b6dba18..80a794e 100755 --- a/setup.py +++ b/setup.py @@ -27,9 +27,9 @@ setuptools.setup( "simplepup", ], - tests_requires = [ + tests_require = [ "pytest", - ] + ], include_package_data = True, entry_points = { From 854daa0d468997363dd6e10953824462de1d0cb6 Mon Sep 17 00:00:00 2001 From: Heath Seals Date: Fri, 23 Aug 2019 13:33:03 -0500 Subject: [PATCH 2/4] Add Docker file and generate wrapper --- CODEOWNERS | 8 ++++++++ Dockerfile | 6 ++++++ generate.py | 17 +++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 CODEOWNERS create mode 100644 Dockerfile create mode 100755 generate.py diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..7832d20 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,8 @@ +# This will cause InfraCore to be assigned review of any opened PRs against +# the branches containing this file. +# See https://help.github.com/en/articles/about-code-owners for info on how to +# take ownership of parts of the code base that should be reviewed by another +# team. + +* @puppetlabs/infracore + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..736157e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3 +ADD generate.py / +RUN pip install --upgrade pip +#RUN pip install --upgrade --extra-index-url https://artifactory.delivery.puppetlabs.net/artifactory/api/pypi/pypi/simple infinitory +RUN pip install git+git://github.com/puppetlabs/infinitory.git@setup_fixup +CMD [ "python", "generate.py", "pe-master-infranext-prod-1.infc-aws.puppet.net" ] diff --git a/generate.py b/generate.py new file mode 100755 index 0000000..4a9853b --- /dev/null +++ b/generate.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import sys +import subprocess + +if len(sys.argv) == 1: + puppetdb = 'localhost' +elif len(sys.argv) == 2: + puppetdb = sys.argv[1] +else: + sys.exit("Expected 1 or 0 arguments. Got {}.".format(len(sys.argv) - 1)) + +# Generate the report +subprocess.check_call( + ["infinitory", "--host", "{}".format(puppetdb), + "--output", "/srv/infinitory/output"], + timeout=120) From fd1ad61fce2209e8150d9eef3c23db31b57a2fe3 Mon Sep 17 00:00:00 2001 From: suckatrash Date: Tue, 25 Aug 2020 05:36:41 -0700 Subject: [PATCH 3/4] (DIO-834) Refactor to use pdb queries, output to GCS Bucket --- CODEOWNERS | 2 +- Dockerfile | 8 ++++--- README.md | 31 ++++++++++++++++++++++++ README.rst | 13 ---------- generate.py | 15 +++++------- infinitory/cli.py | 53 +++++++++++++++++++++++++++++++---------- infinitory/errors.py | 29 +++++++++++----------- infinitory/inventory.py | 12 +++++----- setup.py | 4 +++- 9 files changed, 107 insertions(+), 60 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/CODEOWNERS b/CODEOWNERS index 7832d20..6c509ca 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -4,5 +4,5 @@ # take ownership of parts of the code base that should be reviewed by another # team. -* @puppetlabs/infracore +* @puppetlabs/dio diff --git a/Dockerfile b/Dockerfile index 736157e..92b2711 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM python:3 ADD generate.py / +ENV TOKEN $TOKEN +ENV BUCKET $BUCKET +ENV GOOGLE_APPLICATION_CREDENTIALS $GOOGLE_APPLICATION_CREDENTIALS RUN pip install --upgrade pip -#RUN pip install --upgrade --extra-index-url https://artifactory.delivery.puppetlabs.net/artifactory/api/pypi/pypi/simple infinitory -RUN pip install git+git://github.com/puppetlabs/infinitory.git@setup_fixup -CMD [ "python", "generate.py", "pe-master-infranext-prod-1.infc-aws.puppet.net" ] +RUN pip install -i https://artifactory.delivery.puppetlabs.net/artifactory/api/pypi/pypi/simple -v infinitory==0.1.6 +ENTRYPOINT python generate.py ${PDB_HOST} ${TOKEN} ${BUCKET} diff --git a/README.md b/README.md new file mode 100644 index 0000000..5530705 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +SRE Inventory Report +==================== + +Generate a report on SRE inventory, including hosts, roles, and +services. + +## Running in Docker + +``` +docker run -e GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS -e BUCKET= -e TOKEN= -v /tmp:/output:rw --add-host : infinitory-app +``` + +Using `GOOGLE_APPLICATION_CREDENTIALS` may require an extra volume mount in some cases: + +``` +-v /path/to/creds.json:/creds.json +``` + +...where your ENV variable points to that file: + +``` +export GOOGLE_APPLICATION_CREDENTIALS=/creds.json +``` + +## Developing + +Use python setup.py develop to install dependencies + +Run in Dev: + +bin/infinitory -h pdb.ops.puppetlabs.net -t -o /tmp/output -b diff --git a/README.rst b/README.rst deleted file mode 100644 index 4c9cc9a..0000000 --- a/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -SRE Inventory Report -==================== - -Generate a report on SRE inventory, including hosts, roles, and services. - -Developing -========== - -Use `python setup.py develop` to install dependencies - -Run in Dev: - -bin/infinitory -h pdb.ops.puppetlabs.net -o /tmp/output diff --git a/generate.py b/generate.py index 4a9853b..b8aeec6 100755 --- a/generate.py +++ b/generate.py @@ -3,15 +3,12 @@ import sys import subprocess -if len(sys.argv) == 1: - puppetdb = 'localhost' -elif len(sys.argv) == 2: - puppetdb = sys.argv[1] -else: - sys.exit("Expected 1 or 0 arguments. Got {}.".format(len(sys.argv) - 1)) +puppetdb = sys.argv[1] +token = sys.argv[2] +bucket = sys.argv[3] # Generate the report subprocess.check_call( - ["infinitory", "--host", "{}".format(puppetdb), - "--output", "/srv/infinitory/output"], - timeout=120) + ["infinitory", "--host", puppetdb, "--token", token, "--bucket", bucket, + "--output", "/output/infinitory"], + timeout=300) diff --git a/infinitory/cli.py b/infinitory/cli.py index 7cb29f8..1d11947 100755 --- a/infinitory/cli.py +++ b/infinitory/cli.py @@ -19,9 +19,10 @@ import sys from infinitory import cellformatter from infinitory.inventory import Inventory from simplepup import puppetdb +from pypuppetdb import connect +from google.cloud import storage - -def output_html(inventory, directory): +def output_html(inventory, directory, bucket_name): if os.path.isdir(directory): shutil.rmtree(directory) os.mkdir(directory, 0o755) @@ -32,6 +33,7 @@ def output_html(inventory, directory): with open("{}/pygments.css".format(directory), "w", encoding="utf-8") as css: css.write(pygments.formatters.HtmlFormatter().get_style_defs('.codehilite')) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/pygments.css".format(directory), destination_blob_name="pygments.css") os.mkdir("{}/errors".format(directory), 0o755) os.mkdir("{}/nodes".format(directory), 0o755) @@ -43,6 +45,7 @@ def output_html(inventory, directory): render_template("home.html", path="", generation_time=generation_time)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/index.html".format(directory), destination_blob_name="index.html") report_columns = [ cellformatter.Fqdn("facts", "fqdn"), @@ -71,6 +74,7 @@ def output_html(inventory, directory): generation_time=generation_time, columns=unique_error_columns, errors=unique_errors)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/errors/index.html".format(directory), destination_blob_name="errors/index.html") all_error_columns = [ cellformatter.Base("other", "message"), @@ -87,6 +91,8 @@ def output_html(inventory, directory): generation_time=generation_time, columns=all_error_columns, errors=unique_errors)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/errors/all.html".format(directory), destination_blob_name="errors/all.html") + with open("{}/nodes/index.html".format(directory), "w", encoding="utf-8") as html: html.write( @@ -95,6 +101,8 @@ def output_html(inventory, directory): generation_time=generation_time, columns=report_columns, nodes=nodes)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/nodes/index.html".format(directory), destination_blob_name="nodes/index.html") + all_columns = [ cellformatter.Base("facts", "fqdn"), @@ -124,6 +132,8 @@ def output_html(inventory, directory): csv_writer.writerow([cell.head_csv() for cell in all_columns]) for node in nodes: csv_writer.writerow([cell.body_csv(node) for cell in all_columns]) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/nodes.csv".format(directory), destination_blob_name="nodes.csv") + write_json(nodes, directory, "index") @@ -136,6 +146,7 @@ def output_html(inventory, directory): generation_time=generation_time, columns=all_columns[1:], node=node)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/nodes/{}.html".format(directory, node["certname"]), destination_blob_name="nodes/{}.html".format(node["certname"])) os.mkdir("{}/roles".format(directory), 0o755) with open("{}/roles/index.html".format(directory), "w", encoding="utf-8") as html: @@ -144,6 +155,8 @@ def output_html(inventory, directory): path="../", generation_time=generation_time, roles=inventory.sorted_roles())) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/roles/index.html".format(directory), destination_blob_name="roles/index.html") + os.mkdir("{}/services".format(directory), 0o755) sorted_services = inventory.sorted_services() @@ -154,6 +167,8 @@ def output_html(inventory, directory): path="../", generation_time=generation_time, services=sorted_services)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/services/index.html".format(directory), destination_blob_name="services/index.html") + for service in sorted_services: path = "{}/services/{}.html".format(directory, service["class_name"]) @@ -163,6 +178,7 @@ def output_html(inventory, directory): path="../", generation_time=generation_time, service=service)) + gcs_upload(bucket_name=bucket_name, source_file_name="{}/services/{}.html".format(directory, service["class_name"]), destination_blob_name="services/{}.html".format(service["class_name"])) def render_template(template_name, **kwargs): @@ -216,13 +232,24 @@ def write_json(nodes, directory, filename): with open(path, "w", encoding="utf-8") as json_out: json_out.write(json.dumps(nodes)) +def gcs_upload(bucket_name, source_file_name, destination_blob_name): + """Uploads a file to the bucket.""" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_filename(source_file_name) + @click.command() @click.option("--output", "-o", required=True, metavar="PATH", help="Directory to put report in. WARNING: this directory will be removed if it already exists.") @click.option("--host", "-h", default="localhost", metavar="HOST", help="PuppetDB host to query") +@click.option("--token", "-t", default="123token", metavar="TOKEN", help="RBAC auth token to use") @click.option("--verbose", "-v", default=False, is_flag=True) @click.option("--debug", "-d", default=False, is_flag=True) +@click.option("--bucket", "-b", default="bucket", metavar="BUCKET", help="Bucket to save files to, such as GCS") @click.version_option() -def main(host, output, verbose, debug): +def main(host, token, output, bucket, verbose, debug ): """Generate SRE inventory report""" if debug: set_up_logging(logging.DEBUG) @@ -231,20 +258,20 @@ def main(host, output, verbose, debug): else: set_up_logging(logging.WARNING) + pupdb = connect(host=host, port=8081, timeout=30, token=token) try: inventory = Inventory(debug=debug) inventory.add_active_filter() + + inventory.load_nodes(pupdb) + inventory.load_errors(pupdb) + inventory.load_backups(pupdb) + inventory.load_logging(pupdb) + inventory.load_metrics(pupdb) + inventory.load_monitoring(pupdb) + inventory.load_roles(pupdb) - with puppetdb.AutomaticConnection(host) as pupdb: - inventory.load_nodes(pupdb) - inventory.load_errors(pupdb) - inventory.load_backups(pupdb) - inventory.load_logging(pupdb) - inventory.load_metrics(pupdb) - inventory.load_monitoring(pupdb) - inventory.load_roles(pupdb) - - output_html(inventory, output) + output_html(inventory, output, bucket_name=bucket) except socket.gaierror as e: sys.exit("PuppetDB connection (Socket): {}".format(e)) except paramiko.ssh_exception.SSHException as e: diff --git a/infinitory/errors.py b/infinitory/errors.py index eb300fb..15f17cd 100644 --- a/infinitory/errors.py +++ b/infinitory/errors.py @@ -32,21 +32,22 @@ class ErrorParser(object): def load_reports(self, pupdb): """ I didn't use a subquery because it takes much longer than loading the reports one by one """ - for report in pupdb.query('nodes[certname, latest_report_hash] { }'): - cache_file = "%s/%s" % (self.reports_cache_path, report["latest_report_hash"]) - if os.path.isfile(cache_file): - full_report = pickle.load(open(cache_file, "rb")) - if self.debug: - sys.stdout.write('#') - else: - query = 'reports[] { hash = "%s" }' % report["latest_report_hash"] - full_report = pupdb.query(query) - pickle.dump( full_report, open(cache_file, "wb" ) ) - if self.debug: - sys.stdout.write('.') - sys.stdout.flush() + for report in pupdb._query('nodes', query='["extract", ["certname", "latest_report_hash"]]'): + if report["latest_report_hash"] != None: + cache_file = "%s/%s" % (self.reports_cache_path, report["latest_report_hash"]) + if os.path.isfile(cache_file): + full_report = pickle.load(open(cache_file, "rb")) + if self.debug: + sys.stdout.write('#') + else: + query = '["=", "hash", "%s"]' % report["latest_report_hash"] + full_report = pupdb._query('reports', query=query) + pickle.dump( full_report, open(cache_file, "wb" ) ) + if self.debug: + sys.stdout.write('.') + sys.stdout.flush() - self._reports[report["certname"]] = full_report[0] + self._reports[report["certname"]] = full_report[0] def common_error_prefixes(self): return [ diff --git a/infinitory/inventory.py b/infinitory/inventory.py index 51b926a..1613e75 100644 --- a/infinitory/inventory.py +++ b/infinitory/inventory.py @@ -1,10 +1,10 @@ from collections import defaultdict from operator import itemgetter from simplepup import puppetdb +from pypuppetdb import connect import infinitory.errors as errors - class Inventory(object): def __init__(self, filters=set(), debug=False): self.debug = debug @@ -21,16 +21,16 @@ class Inventory(object): def load_nodes(self, pupdb): self.nodes = dict() - for node in pupdb.query(self.filter('inventory {}')): + for node in pupdb._query('inventory'): node["other"] = defaultdict(list) self.nodes[node["certname"]] = node def query_classes(self, pupdb, class_name): return self.query_resources(pupdb, - 'title="%s" and type="Class"' % class_name) + '["and", ["=", "title", "%s"], ["=", "type", "Class"]]' % class_name) def query_resources(self, pupdb, condition, include_absent=False): - for resource in pupdb.query(self.filter('resources {}', condition)): + for resource in pupdb._query('resources', query=condition): if not include_absent: if resource["parameters"].get("ensure", None) == "absent": continue @@ -41,7 +41,7 @@ class Inventory(object): continue def load_backups(self, pupdb): - for node, resource in self.query_resources(pupdb, 'type="Backup::Job"'): + for node, resource in self.query_resources(pupdb, '["=", "type", "Backup::Job"]'): paths = resource["parameters"]["files"] if type(paths) is list: node["other"]["backups"].extend(paths) @@ -86,7 +86,7 @@ class Inventory(object): def load_roles(self, pupdb): self.roles = defaultdict(list) - condition = 'type = "Class" and title ~ "^Role::"' + condition = '["and", ["=", "type", "Class"], ["~", "title", "^Role::"]]' for node, resource in self.query_resources(pupdb, condition): if resource["title"] not in ("role", "role::delivery"): node["other"]["roles"].append(resource["title"]) diff --git a/setup.py b/setup.py index 80a794e..cb386b6 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools setuptools.setup( name = "infinitory", - version = "0.0.6", + version = "0.1.6", description = "SRE host, role, and service inventory", author = "Daniel Parks", @@ -25,6 +25,8 @@ setuptools.setup( "markdown2", "pygments", "simplepup", + "pypuppetdb", + "google-cloud-storage", ], tests_require = [ From 79fc53cacc7c9d34b4eab0154d0b2ceee9ee8693 Mon Sep 17 00:00:00 2001 From: suckatrash Date: Tue, 29 Sep 2020 10:02:54 -0700 Subject: [PATCH 4/4] (DIO-834) Adds infinitory-flask component --- .gitignore | 2 ++ README.md | 36 ++++++++++++++++++++- infinitory-flask/Dockerfile | 9 ++++++ infinitory-flask/app.py | 64 +++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 infinitory-flask/Dockerfile create mode 100644 infinitory-flask/app.py diff --git a/.gitignore b/.gitignore index 7abcd3d..6b7e54a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ __pycache__ /output bin/ /cache +/infinitory-flask/templates +/infinitory-flask/static \ No newline at end of file diff --git a/README.md b/README.md index 5530705..1024f4c 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,26 @@ SRE Inventory Report Generate a report on SRE inventory, including hosts, roles, and services. +## Architecture + +This app has two components: + +`infinitory` - the data colection portion. Which can be run in cron or otherwise scheduled to collect data from Puppetdb using token authentication. Data can be stored locally as well as pushed to a GCS bucket. + +`infinitory-flask` - the web frontend portion. This can be pointed to resources collected by the `infinitory` (cron) app and serves data from the GCS bucket + ## Running in Docker +`infinitory` ``` docker run -e GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS -e BUCKET= -e TOKEN= -v /tmp:/output:rw --add-host : infinitory-app ``` +`infinitory-flask` +``` +docker run -e GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS -e BUCKET= infinitory-flask +``` + Using `GOOGLE_APPLICATION_CREDENTIALS` may require an extra volume mount in some cases: ``` @@ -26,6 +40,26 @@ export GOOGLE_APPLICATION_CREDENTIALS=/creds.json Use python setup.py develop to install dependencies -Run in Dev: +Running `infinitory` in Dev: +``` bin/infinitory -h pdb.ops.puppetlabs.net -t -o /tmp/output -b +``` + +Running `infinitory-flask` in Dev: + +``` +infinitory-flask/python app.py infinitory-prod +``` + +### Build / release + +`infinitory` - For infinitory, you must first release the python package and then build / push the docker image +``` +## Release a python build +## (with .pypirc in place) +python setup.py sdist upload -r local + +``` + +`infinitory-flask` - Simply build and push the docker image to release this portion of the app. diff --git a/infinitory-flask/Dockerfile b/infinitory-flask/Dockerfile new file mode 100644 index 0000000..5f1a0c3 --- /dev/null +++ b/infinitory-flask/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3 +ADD app.py / +ENV GOOGLE_APPLICATION_CREDENTIALS $GOOGLE_APPLICATION_CREDENTIALS +ENV TZ=America/Los_Angeles +ENV BUCKET $BUCKET +RUN pip install --upgrade pip +RUN pip install flask google-cloud-storage +EXPOSE 5000 +ENTRYPOINT python app.py ${BUCKET} diff --git a/infinitory-flask/app.py b/infinitory-flask/app.py new file mode 100644 index 0000000..a9b1a76 --- /dev/null +++ b/infinitory-flask/app.py @@ -0,0 +1,64 @@ +import os +import logging +import shutil +import sys +from flask import Flask, send_file, Response +from google.cloud import storage +import tempfile + +app = Flask(__name__) +app.config['root_path'] = '/' +app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 +app.config['static_url_path'] = '/static' +app.config['static_folder'] = 'static' + +bucket = sys.argv[1] + +if os.path.isdir('templates'): + shutil.rmtree('templates') +os.mkdir('templates', 0o755) + +if os.path.isdir('static'): + shutil.rmtree('static') +os.mkdir('static', 0o755) + +client = storage.Client() +bucket = client.get_bucket(bucket) + +css = bucket.get_blob('pygments.css') +css.download_to_filename("templates/pygments.css") + +static = bucket.list_blobs(prefix='static') +for b in static: + destination_uri = '{}'.format(b.name) + b.download_to_filename(destination_uri) + +@app.route('/nodes//') +def render_static_node_page(page_name): + return fetch_bucket_resource("nodes/"+page_name) + +@app.route('/roles//') +def render_static_roles_page(page_name): + return fetch_bucket_resource("roles/"+page_name) + +@app.route('/services//') +def render_static_services_page(page_name): + return fetch_bucket_resource("services/"+page_name) + +@app.route('/errors//') +def render_static_errors_page(page_name): + return fetch_bucket_resource("errors/"+page_name) + +@app.route('/') +@app.route('/index.html/') +def render_index(): + return fetch_bucket_resource('index.html') + +def fetch_bucket_resource(blob_path): + blob = bucket.get_blob(blob_path) + with tempfile.NamedTemporaryFile() as temp: + blob.download_to_filename(temp.name) + return send_file(temp.name, mimetype='html') + +if __name__ == '__main__': + app.run(host="0.0.0.0", port=5000)