diff --git a/.gitignore b/.gitignore index 0bf0a89..7abcd3d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ /*.egg-info /output bin/ +/cache diff --git a/README.rst b/README.rst index a12bd9f..4c9cc9a 100644 --- a/README.rst +++ b/README.rst @@ -7,3 +7,7 @@ Developing ========== Use `python setup.py develop` to install dependencies + +Run in Dev: + +bin/infinitory -h pdb.ops.puppetlabs.net -o /tmp/output diff --git a/infinitory/cellformatter.py b/infinitory/cellformatter.py index 765381b..07f1998 100644 --- a/infinitory/cellformatter.py +++ b/infinitory/cellformatter.py @@ -5,6 +5,7 @@ from jinja2 import Markup from operator import itemgetter import re + class Base(object): def __init__(self, section, key, header=None): self.section = section @@ -55,6 +56,21 @@ class Boolean(Base): return "Y" if self.value(record) else "N" +class TruncatedList(Base): + def value_html(self, record): + items = [self.item_html(i) for i in self.value(record)] + return Markup("
    %s
") % Markup("\n").join(items[:5]) + + def item_html(self, item): + return Markup("
  • %s
    • ") % item + + def value_csv(self, record): + return "\n".join([self.item_csv(i) for i in self.value(record)]) + + def item_csv(self, item): + return item + + class List(Base): def value_html(self, record): items = [self.item_html(i) for i in self.value(record)] @@ -173,4 +189,3 @@ class Os(Base): pass return " ".join(os) - diff --git a/infinitory/cli.py b/infinitory/cli.py index b80a608..7cb29f8 100755 --- a/infinitory/cli.py +++ b/infinitory/cli.py @@ -33,6 +33,7 @@ def output_html(inventory, directory): with open("{}/pygments.css".format(directory), "w", encoding="utf-8") as css: css.write(pygments.formatters.HtmlFormatter().get_style_defs('.codehilite')) + os.mkdir("{}/errors".format(directory), 0o755) os.mkdir("{}/nodes".format(directory), 0o755) nodes = inventory.sorted_nodes("facts", "fqdn") generation_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ") @@ -54,6 +55,39 @@ def output_html(inventory, directory): cellformatter.Roles("other", "roles"), ] + unique_error_columns = [ + cellformatter.Base("other", "count"), + cellformatter.Base("other", "level"), + cellformatter.Base("other", "message"), + cellformatter.TruncatedList("other", "certnames"), + ] + + unique_errors = inventory.unique_errors() + + with open("{}/errors/index.html".format(directory), "w", encoding="utf-8") as html: + html.write( + render_template("errors.html", + path="../", + generation_time=generation_time, + columns=unique_error_columns, + errors=unique_errors)) + + all_error_columns = [ + cellformatter.Base("other", "message"), + cellformatter.Base("other", "level"), + cellformatter.Base("other", "certname"), + ] + + all_errors = inventory.all_errors() + + with open("{}/errors/all.html".format(directory), "w", encoding="utf-8") as html: + html.write( + render_template("all_errors.html", + path="../", + generation_time=generation_time, + columns=all_error_columns, + errors=unique_errors)) + with open("{}/nodes/index.html".format(directory), "w", encoding="utf-8") as html: html.write( render_template("nodes.html", @@ -130,6 +164,7 @@ def output_html(inventory, directory): generation_time=generation_time, service=service)) + def render_template(template_name, **kwargs): data_path = os.path.dirname(os.path.abspath(__file__)) environment = jinja2.Environment( @@ -197,16 +232,17 @@ def main(host, output, verbose, debug): set_up_logging(logging.WARNING) try: - inventory = Inventory() + inventory = Inventory(debug=debug) inventory.add_active_filter() - with puppetdb.AutomaticConnection(host) as pdb: - inventory.load_nodes(pdb) - inventory.load_backups(pdb) - inventory.load_logging(pdb) - inventory.load_metrics(pdb) - inventory.load_monitoring(pdb) - inventory.load_roles(pdb) + with puppetdb.AutomaticConnection(host) as pupdb: + inventory.load_nodes(pupdb) + inventory.load_errors(pupdb) + inventory.load_backups(pupdb) + inventory.load_logging(pupdb) + inventory.load_metrics(pupdb) + inventory.load_monitoring(pupdb) + inventory.load_roles(pupdb) output_html(inventory, output) except socket.gaierror as e: @@ -219,5 +255,3 @@ def main(host, output, verbose, debug): sys.exit(e) except requests.exceptions.ConnectionError as e: sys.exit(e) - - diff --git a/infinitory/errors.py b/infinitory/errors.py new file mode 100644 index 0000000..eb300fb --- /dev/null +++ b/infinitory/errors.py @@ -0,0 +1,116 @@ +import logging +import os +import pickle +import sys +import time + +from datetime import datetime + + +class ErrorParser(object): + def __init__(self, debug=False): + self.all_errors = [] + self.reports_cache_path = '/tmp/infinitory_cache' + self.debug = debug + self._logger = logging.getLogger() + self._reports = dict() + self.unique_errors = [] + self.delete_report_cache() + + def delete_report_cache(self): + if not os.path.isdir(self.reports_cache_path): + os.mkdir(self.reports_cache_path) + + for file in os.listdir(self.reports_cache_path): + # Delete cache item if older than 2 hours + absolute_cache_file_path = os.path.join(self.reports_cache_path,file) + time_one_hour_ago = time.mktime(datetime.now().timetuple()) - (1 * 3600) + if os.stat(absolute_cache_file_path).st_mtime < time_one_hour_ago: + print("Deleting File " + absolute_cache_file_path) + os.remove(absolute_cache_file_path) + + def load_reports(self, pupdb): + """ I didn't use a subquery because it takes much longer than loading + the reports one by one """ + for report in pupdb.query('nodes[certname, latest_report_hash] { }'): + cache_file = "%s/%s" % (self.reports_cache_path, report["latest_report_hash"]) + if os.path.isfile(cache_file): + full_report = pickle.load(open(cache_file, "rb")) + if self.debug: + sys.stdout.write('#') + else: + query = 'reports[] { hash = "%s" }' % report["latest_report_hash"] + full_report = pupdb.query(query) + pickle.dump( full_report, open(cache_file, "wb" ) ) + if self.debug: + sys.stdout.write('.') + sys.stdout.flush() + + self._reports[report["certname"]] = full_report[0] + + def common_error_prefixes(self): + return [ + "Could not retrieve catalog from remote server: Error 500 on SERVER: Server Error: Evaluation Error: Error while evaluating a Function Call, Untrusted facts (left) don't match values from certname (right)" + ] + + def matches_stored_error(self, message): + for se in self.common_error_prefixes(): + if message.startswith(se): + return se + + return None + + def clean_error_message(self, error_message): + stored_error = self.matches_stored_error(error_message) + + if stored_error: + return stored_error + + return error_message + + def modify_unique_errors_at(self, i, log_level, certname): + + new_certname_list = self.unique_errors[i]['certnames'] + new_certname_list.add(certname) + + self.unique_errors[i] = { + 'count': self.unique_errors[i]['count'] + 1, + 'level': log_level, + 'certnames': new_certname_list, + 'message': self.unique_errors[i]['message'] + } + + def append_unique_error(self, error_message, log_level, certname): + for i, ue in enumerate(self.unique_errors): + if ue['message'] == error_message: + self.modify_unique_errors_at(i, log_level, certname) + return + + self.unique_errors.append({ + 'count': 1, + 'level': log_level, + 'certnames': set([certname]), + 'message': error_message, + }) + + def extract_errors_from_reports(self): + for node, report in self._reports.items(): + + self._logger.debug("%s -- %s" % (report["certname"], report["status"])) + for log_message in report['logs']['data']: + if log_message['level'] == 'err' or log_message['level'] == 'warning': + error = { + 'level': log_message['level'], + 'hostname': report["certname"], + 'message': log_message['message'] + } + + self.all_errors.append(error) + + error_message = self.clean_error_message(error['message']) + + self.append_unique_error( + error_message, + log_message['level'], + report['certname'] + ) diff --git a/infinitory/inventory.py b/infinitory/inventory.py index fbae118..51b926a 100644 --- a/infinitory/inventory.py +++ b/infinitory/inventory.py @@ -2,8 +2,13 @@ from collections import defaultdict from operator import itemgetter from simplepup import puppetdb +import infinitory.errors as errors + + class Inventory(object): - def __init__(self, filters=set()): + def __init__(self, filters=set(), debug=False): + self.debug = debug + self.errorParser = errors.ErrorParser(debug=debug) self.filter = puppetdb.QueryFilter(filters) self.nodes = None self.roles = None @@ -14,18 +19,18 @@ class Inventory(object): def add_filter(self, filter): self.filter.add(filter) - def load_nodes(self, pdb): + def load_nodes(self, pupdb): self.nodes = dict() - for node in pdb.query(self.filter('inventory {}')): + for node in pupdb.query(self.filter('inventory {}')): node["other"] = defaultdict(list) self.nodes[node["certname"]] = node - def query_classes(self, pdb, class_name): - return self.query_resources(pdb, + def query_classes(self, pupdb, class_name): + return self.query_resources(pupdb, 'title="%s" and type="Class"' % class_name) - def query_resources(self, pdb, condition, include_absent=False): - for resource in pdb.query(self.filter('resources {}', condition)): + def query_resources(self, pupdb, condition, include_absent=False): + for resource in pupdb.query(self.filter('resources {}', condition)): if not include_absent: if resource["parameters"].get("ensure", None) == "absent": continue @@ -35,36 +40,54 @@ class Inventory(object): except KeyError: continue - def load_backups(self, pdb): - for node, resource in self.query_resources(pdb, 'type="Backup::Job"'): + def load_backups(self, pupdb): + for node, resource in self.query_resources(pupdb, 'type="Backup::Job"'): paths = resource["parameters"]["files"] if type(paths) is list: node["other"]["backups"].extend(paths) else: node["other"]["backups"].append(paths) - def load_logging(self, pdb): - for node, resource in self.query_classes(pdb, "Profile::Logging::Rsyslog::Client"): + def load_errors(self, pupdb): + self.errorParser.load_reports(pupdb) + self.errorParser.extract_errors_from_reports() + + def wrap_with_category(self, list_of_hashes, category): + retval = [] + for error in list_of_hashes: + retval.append({ + category: error + }) + return retval + + def unique_errors(self): + return self.wrap_with_category(self.errorParser.unique_errors, "other") + + def all_errors(self): + return self.wrap_with_category(self.errorParser.all_errors, "other") + + def load_logging(self, pupdb): + for node, resource in self.query_classes(pupdb, "Profile::Logging::Rsyslog::Client"): node["other"]["logging"] = True - def load_metrics(self, pdb): - for node, resource in self.query_classes(pdb, "Profile::Metrics"): + def load_metrics(self, pupdb): + for node, resource in self.query_classes(pupdb, "Profile::Metrics"): node["other"]["metrics"] = True - def load_monitoring(self, pdb): - for node, resource in self.query_classes(pdb, "Profile::Server::Monitor"): + def load_monitoring(self, pupdb): + for node, resource in self.query_classes(pupdb, "Profile::Server::Monitor"): node["other"]["monitoring"] = True - for node, resource in self.query_classes(pdb, "Profile::Monitoring::Icinga2::Common"): + for node, resource in self.query_classes(pupdb, "Profile::Monitoring::Icinga2::Common"): node["other"]["icinga_notification_period"] = resource["parameters"]["notification_period"] node["other"]["icinga_environment"] = resource["parameters"]["icinga2_environment"] node["other"]["icinga_owner"] = resource["parameters"]["owner"] - def load_roles(self, pdb): + def load_roles(self, pupdb): self.roles = defaultdict(list) condition = 'type = "Class" and title ~ "^Role::"' - for node, resource in self.query_resources(pdb, condition): + for node, resource in self.query_resources(pupdb, condition): if resource["title"] not in ("role", "role::delivery"): node["other"]["roles"].append(resource["title"]) self.roles[resource["title"]].append(node) @@ -93,4 +116,3 @@ class Inventory(object): services[class_name]["nodes"].append(node) return sorted(services.values(), key=itemgetter("human_name")) - diff --git a/infinitory/templates/all_errors.html b/infinitory/templates/all_errors.html new file mode 100644 index 0000000..f146e78 --- /dev/null +++ b/infinitory/templates/all_errors.html @@ -0,0 +1,27 @@ +{% extends "layout.html" %} +{% block title %}All Errors{% endblock %} +{% block body %} + Unique Errors +

    All Errors

    + + + + {% for cell in columns %} + {{ cell.head_html() }} + {% endfor %} + + + + {% for error in errors %} + + {% for cell in columns %} + {{ cell.body_html(error) }} + {% endfor %} + + {% endfor %} + +
    +{% endblock %} +{% block footer %} + Download CSV +{% endblock %} diff --git a/infinitory/templates/errors.html b/infinitory/templates/errors.html new file mode 100644 index 0000000..45a127a --- /dev/null +++ b/infinitory/templates/errors.html @@ -0,0 +1,27 @@ +{% extends "layout.html" %} +{% block title %}Unique Errors{% endblock %} +{% block body %} + All Errors +

    Unique Errors

    + + + + {% for cell in columns %} + {{ cell.head_html() }} + {% endfor %} + + + + {% for error in errors %} + + {% for cell in columns %} + {{ cell.body_html(error) }} + {% endfor %} + + {% endfor %} + +
    +{% endblock %} +{% block footer %} + Download CSV +{% endblock %} diff --git a/infinitory/templates/layout.html b/infinitory/templates/layout.html index 23df35f..5d3a989 100644 --- a/infinitory/templates/layout.html +++ b/infinitory/templates/layout.html @@ -16,6 +16,7 @@
  • Nodes
  • Roles
  • Services
    • +
  • Errors
    • diff --git a/setup.py b/setup.py index 7cc9a52..b6dba18 100755 --- a/setup.py +++ b/setup.py @@ -24,9 +24,13 @@ setuptools.setup( "Jinja2", "markdown2", "pygments", - "simplepup" + "simplepup", ], + tests_requires = [ + "pytest", + ] + include_package_data = True, entry_points = { "console_scripts": [ diff --git a/test/errors/test_errors.py b/test/errors/test_errors.py new file mode 100644 index 0000000..4dd6e13 --- /dev/null +++ b/test/errors/test_errors.py @@ -0,0 +1,37 @@ +import unittest +import infinitory.errors +import sample + + +class MyTest(unittest.TestCase): + def test_error_message_cleaner(self): + + errorParser = infinitory.errors.ErrorParser() + + self.assertEqual(errorParser.clean_error_message("Hello"), "Hello") + + self.assertEqual( + errorParser.clean_error_message("Could not retrieve catalog from remote server: Error 500 on SERVER: Server Error: Evaluation Error: Error while evaluating a Function Call, Untrusted facts (left) don't match values from certname (right) owaijefoeiawjfoiewjf"), + "Could not retrieve catalog from remote server: Error 500 on SERVER: Server Error: Evaluation Error: Error while evaluating a Function Call, Untrusted facts (left) don't match values from certname (right)" + ) + + def test_other_prefixing(self): + """ The cell formatter expects that all values have a prefix associated + with them. This checks that the errorParser properly adds that + prefix. """ + + errorParser = infinitory.errors.ErrorParser() + + input = ["1", "2"] + + errorParser.set_all_errors(input) + errorParser.set_unique_errors(input) + + self.assertEqual( + [ { "other": "1" }, { "other": "2" } ], + errorParser.all_errors() + ) + self.assertEqual( + [ { "other": "1" }, { "other": "2" } ], + errorParser.unique_errors() + )