Import SRE inventory code

sreinventory/cellformatter.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# vim: set fileencoding=utf-8 :
+
+from jinja2 import Markup
+from operator import itemgetter
+import re
+
+class Base(object):
+    def __init__(self, section, key, header=None):
+        self.section = section
+        self.key = key
+
+        if header is None:
+            self.header = key
+        else:
+            self.header = header
+
+        if re.search(r"[^a-zA-Z0-9_-]", key):
+            raise ValueError("Invalid key: {}".format(key))
+
+    def body_class(self, record):
+        return ["key_{}".format(self.key)]
+
+    def head_html(self):
+        return Markup('<th class="key_%s">%s</th>') % (self.key, self.header)
+
+    def body_html(self, record):
+        return Markup('<td class="%s">%s</td>') % (" ".join(self.body_class(record)), self.value_html(record))
+
+    def value_html(self, record):
+        return self.value(record)
+
+    def head_csv(self):
+        return self.header
+
+    def body_csv(self, record):
+        return self.value_csv(record)
+
+    def value_csv(self, record):
+        return self.value(record)
+
+    def value(self, record):
+        return record[self.section].get(self.key, None) or ""
+
+
+class Boolean(Base):
+    def body_class(self, record):
+        return super(Boolean, self).body_class(record) \
+            + [("true" if self.value(record) else "false")]
+
+    def value_html(self, record):
+        return u"✔︎" if self.value(record) else ""
+
+    def value_csv(self, record):
+        return "Y" if self.value(record) else "N"
+
+
+class List(Base):
+    def value_html(self, record):
+        items = [self.item_html(i) for i in self.value(record)]
+        return Markup("<ol>%s</ol>") % Markup("\n").join(items)
+
+    def item_html(self, item):
+        return Markup("<li>%s</li>") % item
+
+    def value_csv(self, record):
+        return "\n".join([self.item_csv(i) for i in self.value(record)])
+
+    def item_csv(self, item):
+        return item
+
+
+class Set(Base):
+    def value_html(self, record):
+        items = [self.item_html(i) for i in self.value(record)]
+
+        # set() is used here to dedupe things that can't be put into a set in
+        # value(), like a list of dicts()
+        return Markup("<ul>%s</ul>") % Markup("\n").join(set(items))
+
+    def item_html(self, item):
+        return Markup("<li>%s</li>") % item
+
+    def value_csv(self, record):
+        return "\n".join(set([self.item_csv(i) for i in self.value(record)]))
+
+    def item_csv(self, item):
+        return item
+
+    def value(self, record):
+        return sorted(set(record[self.section].get(self.key, [])))
+
+
+class Roles(Set):
+    def item_html(self, role):
+        return Markup('<li><a href="../roles/index.html#%s">%s</a></li>') % (role, role)
+
+
+class Services(Set):
+    def value(self, record):
+        profile_metadata = record["facts"].get("profile_metadata", dict())
+        return sorted(profile_metadata.get("services", list()), key=itemgetter("human_name"))
+
+    def item_html(self, service):
+        return Markup('<li><a href="../services/%s.html">%s</a></li>') % (
+            service["class_name"],
+            service["human_name"])
+
+    def item_csv(self, service):
+        return service["class_name"]
+
+
+class Owners(Services):
+    def item_html(self, service):
+        if service["owner_uid"] == ":undef":
+            return ""
+        else:
+            return Markup('<li>%s</li>') % service["owner_uid"]
+
+    def item_csv(self, service):
+        if service["owner_uid"] == ":undef":
+            return ""
+        else:
+            return service["owner_uid"]
+
+
+class Teams(Services):
+    def item_html(self, service):
+        if service["team"] == ":undef":
+            return ""
+        else:
+            return Markup('<li>%s</li>') % service["team"]
+
+    def item_csv(self, service):
+        if service["team"] == ":undef":
+            return ""
+        else:
+            return service["team"]
+
+
+class Fqdn(Base):
+    def body_html(self, record):
+        # Use th instead of td:
+        return Markup('<th class="%s">%s</th>') % (
+            " ".join(self.body_class(record)),
+            self.value_html(record))
+
+    def value_html(self, record):
+        return Markup(
+            '<a href="%s.html"><b>%s<span>.</span></b><i>%s</i></a>') % (
+                record["certname"],
+                record["facts"]["hostname"],
+                record["facts"]["domain"])
+
+
+class Os(Base):
+    def value(self, record):
+        os_fact = super(Os, self).value(record)
+        os = [os_fact["name"]]
+
+        try:
+            os.append(os_fact["release"]["full"])
+        except KeyError:
+            pass
+
+        return " ".join(os)
+

sreinventory/cli.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+
+import csv
+from datetime import datetime
+import jinja2
+import logging
+import os
+import markdown2
+import paramiko.ssh_exception
+import pygments.formatters
+import re
+import socket
+import shutil
+import sys
+
+from sreinventory import cellformatter
+from sreinventory.inventory import Inventory
+from simplepup import puppetdb
+
+def output_html(inventory, directory):
+    with open("{}/pygments.css".format(directory), "w", encoding="utf-8") as css:
+        css.write(pygments.formatters.HtmlFormatter().get_style_defs('.codehilite'))
+
+    os.mkdir("{}/nodes".format(directory), 0o755)
+    nodes = inventory.sorted_nodes("facts", "fqdn")
+    generation_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")
+
+    with open("{}/index.html".format(directory), "w", encoding="utf-8") as html:
+        html.write(
+            render_template("home.html",
+                path="",
+                generation_time=generation_time))
+
+    report_columns = [
+        cellformatter.Fqdn("facts", "fqdn"),
+        cellformatter.Teams("other", "teams"),
+        cellformatter.Services("other", "services"),
+        cellformatter.Boolean("other", "monitoring"),
+        cellformatter.Boolean("other", "backups"),
+        cellformatter.Boolean("other", "logging"),
+        cellformatter.Boolean("other", "metrics"),
+        cellformatter.Roles("other", "roles"),
+    ]
+
+    with open("{}/nodes/index.html".format(directory), "w", encoding="utf-8") as html:
+        html.write(
+            render_template("nodes.html",
+                path="../",
+                generation_time=generation_time,
+                columns=report_columns,
+                nodes=nodes))
+
+    all_columns = [
+        cellformatter.Base("facts", "fqdn"),
+        cellformatter.Teams("other", "teams"),
+        cellformatter.Owners("other", "owners"),
+        cellformatter.Services("other", "services"),
+        cellformatter.Base("other", "icinga_notification_period", "Icinga notification period"),
+        cellformatter.Base("other", "icinga_stage", header="Icinga stage"),
+        cellformatter.Base("other", "icinga_owner", header="Icinga owner"),
+        cellformatter.Set("other", "backups"),
+        cellformatter.Boolean("other", "logging"),
+        cellformatter.Boolean("other", "metrics"),
+        cellformatter.Base("facts", "whereami"),
+        cellformatter.Base("facts", "primary_ip"),
+        cellformatter.Os("facts", "os"),
+        cellformatter.Roles("other", "roles"),
+        cellformatter.Base("trusted", "certname"),
+        cellformatter.Base("facts", "group"),
+        cellformatter.Base("facts", "function"),
+        cellformatter.Base("facts", "context"),
+        cellformatter.Base("facts", "stage"),
+        cellformatter.Base("facts", "function_number"),
+    ]
+
+    with open("{}/nodes.csv".format(directory), "w", encoding="utf-8") as out:
+        csv_writer = csv.writer(out, lineterminator="\n")
+        csv_writer.writerow([cell.head_csv() for cell in all_columns])
+        for node in nodes:
+            csv_writer.writerow([cell.body_csv(node) for cell in all_columns])
+
+    for node in nodes:
+        path = "{}/nodes/{}.html".format(directory, node["certname"])
+        with open(path, "w", encoding="utf-8") as html:
+            html.write(
+                render_template("node.html",
+                    path="../",
+                    generation_time=generation_time,
+                    columns=all_columns[1:],
+                    node=node))
+
+    os.mkdir("{}/roles".format(directory), 0o755)
+    with open("{}/roles/index.html".format(directory), "w", encoding="utf-8") as html:
+        html.write(
+            render_template("roles.html",
+                path="../",
+                generation_time=generation_time,
+                roles=inventory.sorted_roles()))
+
+    os.mkdir("{}/services".format(directory), 0o755)
+    sorted_services = inventory.sorted_services()
+
+    with open("{}/services/index.html".format(directory), "w", encoding="utf-8") as html:
+        html.write(
+            render_template("services.html",
+                path="../",
+                generation_time=generation_time,
+                services=sorted_services))
+
+    for service in sorted_services:
+        path = "{}/services/{}.html".format(directory, service["class_name"])
+        with open(path, "w", encoding="utf-8") as html:
+            html.write(
+                render_template("service.html",
+                    path="../",
+                    generation_time=generation_time,
+                    service=service))
+
+
+def render_template(template_name, **kwargs):
+    environment = jinja2.Environment(
+        loader=jinja2.FileSystemLoader("templates"),
+        autoescape=jinja2.select_autoescape(default=True))
+
+    md = markdown2.Markdown(extras=[
+        'fenced-code-blocks',
+        'cuddled-lists',
+        'tables'])
+    def markdown_filter(value):
+        return jinja2.Markup(md.convert(value))
+    environment.filters['markdown'] = markdown_filter
+
+    def unundef(value):
+        return "" if value == ":undef" else value
+    environment.filters['unundef'] = unundef
+
+    _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
+    def nl2br(value):
+        return jinja2.Markup(
+            u'\n\n'.join(u'<p>%s</p>'
+                % jinja2.Markup.escape(p) for p in _paragraph_re.split(value)))
+    environment.filters['nl2br'] = nl2br
+
+    body_id = re.sub(r"\W+", "_", re.sub(r"\..*", "", template_name))
+    template = environment.get_template(template_name)
+    return template.render(body_id=body_id, **kwargs)
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger("paramiko").setLevel(logging.FATAL)
+
+    inventory = Inventory()
+    inventory.add_active_filter()
+
+    try:
+        with puppetdb.AutomaticConnection(sys.argv[1]) as pdb:
+            inventory.load_nodes(pdb)
+            inventory.load_backups(pdb)
+            inventory.load_logging(pdb)
+            inventory.load_metrics(pdb)
+            inventory.load_monitoring(pdb)
+            inventory.load_roles(pdb)
+    except socket.gaierror as e:
+        sys.exit("PuppetDB connection (Socket): {}".format(e))
+    except paramiko.ssh_exception.SSHException as e:
+        sys.exit("PuppetDB connection (SSH): {}".format(e))
+
+    if os.path.isdir("output"):
+        shutil.rmtree("output")
+    os.mkdir("output", 0o755)
+    output_html(inventory, "output")

sreinventory/inventory.py

@@ -0,0 +1,96 @@
+from collections import defaultdict
+from operator import itemgetter
+from simplepup import puppetdb
+
+class Inventory(object):
+    def __init__(self, filters=set()):
+        self.filter = puppetdb.QueryFilter(filters)
+        self.nodes = None
+        self.roles = None
+
+    def add_active_filter(self):
+        self.filter.add("nodes { deactivated is null and expired is null }")
+
+    def add_filter(self, filter):
+        self.filter.add(filter)
+
+    def load_nodes(self, pdb):
+        self.nodes = dict()
+        for node in pdb.query(self.filter('inventory {}')):
+            node["other"] = defaultdict(list)
+            self.nodes[node["certname"]] = node
+
+    def query_classes(self, pdb, class_name):
+        return self.query_resources(pdb,
+            'title="%s" and type="Class"' % class_name)
+
+    def query_resources(self, pdb, condition, include_absent=False):
+        for resource in pdb.query(self.filter('resources {}', condition)):
+            if not include_absent:
+                if resource["parameters"].get("ensure", None) == "absent":
+                    continue
+
+            try:
+                yield self.nodes[resource["certname"]], resource
+            except KeyError:
+                continue
+
+    def load_backups(self, pdb):
+        for node, resource in self.query_resources(pdb, 'type="Backup::Job"'):
+            paths = resource["parameters"]["files"]
+            if type(paths) is list:
+                node["other"]["backups"].extend(paths)
+            else:
+                node["other"]["backups"].append(paths)
+
+    def load_logging(self, pdb):
+        for node, resource in self.query_classes(pdb, "Profile::Logging::Rsyslog::Client"):
+            node["other"]["logging"] = True
+
+    def load_metrics(self, pdb):
+        for node, resource in self.query_classes(pdb, "Profile::Metrics"):
+            node["other"]["metrics"] = True
+
+    def load_monitoring(self, pdb):
+        for node, resource in self.query_classes(pdb, "Profile::Server::Monitor"):
+            node["other"]["monitoring"] = True
+
+        for node, resource in self.query_classes(pdb, "Profile::Monitoring::Icinga2::Common"):
+            node["other"]["icinga_notification_period"] = resource["parameters"]["notification_period"]
+            node["other"]["icinga_environment"] = resource["parameters"]["icinga2_environment"]
+            node["other"]["icinga_owner"] = resource["parameters"]["owner"]
+
+    def load_roles(self, pdb):
+        self.roles = defaultdict(list)
+
+        condition = 'type = "Profile::Motd::Register" and file ~ "/site[.]pp$"'
+        for node, resource in self.query_resources(pdb, condition):
+            if resource["title"] not in ("role", "role::delivery"):
+                node["other"]["roles"].append(resource["title"])
+                self.roles[resource["title"]].append(node)
+
+    def sorted_nodes(self, section, key):
+        return sorted(
+            self.nodes.values(),
+            key=lambda node: node[section][key])
+
+    def sorted_roles(self):
+        return sorted(self.roles.items())
+
+    def sorted_services(self):
+        services = dict()
+
+        for node in self.nodes.values():
+            profile_metadata = node["facts"].get("profile_metadata", dict())
+            service_facts = profile_metadata.get("services", list())
+
+            for service_fact in service_facts:
+                class_name = service_fact["class_name"]
+                if class_name not in services:
+                    services[class_name] = service_fact
+                    services[class_name]["nodes"] = list()
+
+                services[class_name]["nodes"].append(node)
+
+        return sorted(services.values(), key=itemgetter("human_name"))
+