Import SRE inventory code

This commit is contained in:
Daniel Parks 2017-09-11 18:02:43 -07:00
commit 7364ccbff8
No known key found for this signature in database
GPG key ID: 7335138B2B9829EB
18 changed files with 5348 additions and 0 deletions

6
.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
*.py[cod]
__pycache__
/build/
/dist/
/*.egg-info
/output

1
MANIFEST.in Normal file
View file

@ -0,0 +1 @@
include README.rst

5
README.rst Normal file
View file

@ -0,0 +1,5 @@
SRE Inventory Report
====================
Generate a report on SRE inventory, including hosts, roles, and services.

35
setup.py Normal file
View file

@ -0,0 +1,35 @@
import setuptools
setuptools.setup(
name = "sreinventory",
version = "0.0.1",
description = "SRE host, role, and service inventory",
author = "Daniel Parks",
author_email = "daniel.parks@puppet.com",
url = "http://github.com/puppetlabs/sreinventory",
long_description = open("README.rst").read(),
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"Natural Language :: English",
"Programming Language :: Python :: 3",
],
packages = [ "sreinventory" ],
install_requires = [
"Jinja2",
"markdown2",
"pygments",
"simplepup"
],
include_package_data = True,
entry_points = {
"console_scripts": [
"sreinventory = sreinventory.cli:main"
]
}
)

0
sreinventory/__init__.py Normal file
View file

View file

@ -0,0 +1,167 @@
# -*- coding: utf-8 -*-
# vim: set fileencoding=utf-8 :
from jinja2 import Markup
from operator import itemgetter
import re
class Base(object):
def __init__(self, section, key, header=None):
self.section = section
self.key = key
if header is None:
self.header = key
else:
self.header = header
if re.search(r"[^a-zA-Z0-9_-]", key):
raise ValueError("Invalid key: {}".format(key))
def body_class(self, record):
return ["key_{}".format(self.key)]
def head_html(self):
return Markup('<th class="key_%s">%s</th>') % (self.key, self.header)
def body_html(self, record):
return Markup('<td class="%s">%s</td>') % (" ".join(self.body_class(record)), self.value_html(record))
def value_html(self, record):
return self.value(record)
def head_csv(self):
return self.header
def body_csv(self, record):
return self.value_csv(record)
def value_csv(self, record):
return self.value(record)
def value(self, record):
return record[self.section].get(self.key, None) or ""
class Boolean(Base):
def body_class(self, record):
return super(Boolean, self).body_class(record) \
+ [("true" if self.value(record) else "false")]
def value_html(self, record):
return u"✔︎" if self.value(record) else ""
def value_csv(self, record):
return "Y" if self.value(record) else "N"
class List(Base):
def value_html(self, record):
items = [self.item_html(i) for i in self.value(record)]
return Markup("<ol>%s</ol>") % Markup("\n").join(items)
def item_html(self, item):
return Markup("<li>%s</li>") % item
def value_csv(self, record):
return "\n".join([self.item_csv(i) for i in self.value(record)])
def item_csv(self, item):
return item
class Set(Base):
def value_html(self, record):
items = [self.item_html(i) for i in self.value(record)]
# set() is used here to dedupe things that can't be put into a set in
# value(), like a list of dicts()
return Markup("<ul>%s</ul>") % Markup("\n").join(set(items))
def item_html(self, item):
return Markup("<li>%s</li>") % item
def value_csv(self, record):
return "\n".join(set([self.item_csv(i) for i in self.value(record)]))
def item_csv(self, item):
return item
def value(self, record):
return sorted(set(record[self.section].get(self.key, [])))
class Roles(Set):
def item_html(self, role):
return Markup('<li><a href="../roles/index.html#%s">%s</a></li>') % (role, role)
class Services(Set):
def value(self, record):
profile_metadata = record["facts"].get("profile_metadata", dict())
return sorted(profile_metadata.get("services", list()), key=itemgetter("human_name"))
def item_html(self, service):
return Markup('<li><a href="../services/%s.html">%s</a></li>') % (
service["class_name"],
service["human_name"])
def item_csv(self, service):
return service["class_name"]
class Owners(Services):
def item_html(self, service):
if service["owner_uid"] == ":undef":
return ""
else:
return Markup('<li>%s</li>') % service["owner_uid"]
def item_csv(self, service):
if service["owner_uid"] == ":undef":
return ""
else:
return service["owner_uid"]
class Teams(Services):
def item_html(self, service):
if service["team"] == ":undef":
return ""
else:
return Markup('<li>%s</li>') % service["team"]
def item_csv(self, service):
if service["team"] == ":undef":
return ""
else:
return service["team"]
class Fqdn(Base):
def body_html(self, record):
# Use th instead of td:
return Markup('<th class="%s">%s</th>') % (
" ".join(self.body_class(record)),
self.value_html(record))
def value_html(self, record):
return Markup(
'<a href="%s.html"><b>%s<span>.</span></b><i>%s</i></a>') % (
record["certname"],
record["facts"]["hostname"],
record["facts"]["domain"])
class Os(Base):
def value(self, record):
os_fact = super(Os, self).value(record)
os = [os_fact["name"]]
try:
os.append(os_fact["release"]["full"])
except KeyError:
pass
return " ".join(os)

171
sreinventory/cli.py Executable file
View file

@ -0,0 +1,171 @@
#!/usr/bin/env python3
import csv
from datetime import datetime
import jinja2
import logging
import os
import markdown2
import paramiko.ssh_exception
import pygments.formatters
import re
import socket
import shutil
import sys
from sreinventory import cellformatter
from sreinventory.inventory import Inventory
from simplepup import puppetdb
def output_html(inventory, directory):
with open("{}/pygments.css".format(directory), "w", encoding="utf-8") as css:
css.write(pygments.formatters.HtmlFormatter().get_style_defs('.codehilite'))
os.mkdir("{}/nodes".format(directory), 0o755)
nodes = inventory.sorted_nodes("facts", "fqdn")
generation_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")
with open("{}/index.html".format(directory), "w", encoding="utf-8") as html:
html.write(
render_template("home.html",
path="",
generation_time=generation_time))
report_columns = [
cellformatter.Fqdn("facts", "fqdn"),
cellformatter.Teams("other", "teams"),
cellformatter.Services("other", "services"),
cellformatter.Boolean("other", "monitoring"),
cellformatter.Boolean("other", "backups"),
cellformatter.Boolean("other", "logging"),
cellformatter.Boolean("other", "metrics"),
cellformatter.Roles("other", "roles"),
]
with open("{}/nodes/index.html".format(directory), "w", encoding="utf-8") as html:
html.write(
render_template("nodes.html",
path="../",
generation_time=generation_time,
columns=report_columns,
nodes=nodes))
all_columns = [
cellformatter.Base("facts", "fqdn"),
cellformatter.Teams("other", "teams"),
cellformatter.Owners("other", "owners"),
cellformatter.Services("other", "services"),
cellformatter.Base("other", "icinga_notification_period", "Icinga notification period"),
cellformatter.Base("other", "icinga_stage", header="Icinga stage"),
cellformatter.Base("other", "icinga_owner", header="Icinga owner"),
cellformatter.Set("other", "backups"),
cellformatter.Boolean("other", "logging"),
cellformatter.Boolean("other", "metrics"),
cellformatter.Base("facts", "whereami"),
cellformatter.Base("facts", "primary_ip"),
cellformatter.Os("facts", "os"),
cellformatter.Roles("other", "roles"),
cellformatter.Base("trusted", "certname"),
cellformatter.Base("facts", "group"),
cellformatter.Base("facts", "function"),
cellformatter.Base("facts", "context"),
cellformatter.Base("facts", "stage"),
cellformatter.Base("facts", "function_number"),
]
with open("{}/nodes.csv".format(directory), "w", encoding="utf-8") as out:
csv_writer = csv.writer(out, lineterminator="\n")
csv_writer.writerow([cell.head_csv() for cell in all_columns])
for node in nodes:
csv_writer.writerow([cell.body_csv(node) for cell in all_columns])
for node in nodes:
path = "{}/nodes/{}.html".format(directory, node["certname"])
with open(path, "w", encoding="utf-8") as html:
html.write(
render_template("node.html",
path="../",
generation_time=generation_time,
columns=all_columns[1:],
node=node))
os.mkdir("{}/roles".format(directory), 0o755)
with open("{}/roles/index.html".format(directory), "w", encoding="utf-8") as html:
html.write(
render_template("roles.html",
path="../",
generation_time=generation_time,
roles=inventory.sorted_roles()))
os.mkdir("{}/services".format(directory), 0o755)
sorted_services = inventory.sorted_services()
with open("{}/services/index.html".format(directory), "w", encoding="utf-8") as html:
html.write(
render_template("services.html",
path="../",
generation_time=generation_time,
services=sorted_services))
for service in sorted_services:
path = "{}/services/{}.html".format(directory, service["class_name"])
with open(path, "w", encoding="utf-8") as html:
html.write(
render_template("service.html",
path="../",
generation_time=generation_time,
service=service))
def render_template(template_name, **kwargs):
environment = jinja2.Environment(
loader=jinja2.FileSystemLoader("templates"),
autoescape=jinja2.select_autoescape(default=True))
md = markdown2.Markdown(extras=[
'fenced-code-blocks',
'cuddled-lists',
'tables'])
def markdown_filter(value):
return jinja2.Markup(md.convert(value))
environment.filters['markdown'] = markdown_filter
def unundef(value):
return "" if value == ":undef" else value
environment.filters['unundef'] = unundef
_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
def nl2br(value):
return jinja2.Markup(
u'\n\n'.join(u'<p>%s</p>'
% jinja2.Markup.escape(p) for p in _paragraph_re.split(value)))
environment.filters['nl2br'] = nl2br
body_id = re.sub(r"\W+", "_", re.sub(r"\..*", "", template_name))
template = environment.get_template(template_name)
return template.render(body_id=body_id, **kwargs)
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger("paramiko").setLevel(logging.FATAL)
inventory = Inventory()
inventory.add_active_filter()
try:
with puppetdb.AutomaticConnection(sys.argv[1]) as pdb:
inventory.load_nodes(pdb)
inventory.load_backups(pdb)
inventory.load_logging(pdb)
inventory.load_metrics(pdb)
inventory.load_monitoring(pdb)
inventory.load_roles(pdb)
except socket.gaierror as e:
sys.exit("PuppetDB connection (Socket): {}".format(e))
except paramiko.ssh_exception.SSHException as e:
sys.exit("PuppetDB connection (SSH): {}".format(e))
if os.path.isdir("output"):
shutil.rmtree("output")
os.mkdir("output", 0o755)
output_html(inventory, "output")

96
sreinventory/inventory.py Normal file
View file

@ -0,0 +1,96 @@
from collections import defaultdict
from operator import itemgetter
from simplepup import puppetdb
class Inventory(object):
def __init__(self, filters=set()):
self.filter = puppetdb.QueryFilter(filters)
self.nodes = None
self.roles = None
def add_active_filter(self):
self.filter.add("nodes { deactivated is null and expired is null }")
def add_filter(self, filter):
self.filter.add(filter)
def load_nodes(self, pdb):
self.nodes = dict()
for node in pdb.query(self.filter('inventory {}')):
node["other"] = defaultdict(list)
self.nodes[node["certname"]] = node
def query_classes(self, pdb, class_name):
return self.query_resources(pdb,
'title="%s" and type="Class"' % class_name)
def query_resources(self, pdb, condition, include_absent=False):
for resource in pdb.query(self.filter('resources {}', condition)):
if not include_absent:
if resource["parameters"].get("ensure", None) == "absent":
continue
try:
yield self.nodes[resource["certname"]], resource
except KeyError:
continue
def load_backups(self, pdb):
for node, resource in self.query_resources(pdb, 'type="Backup::Job"'):
paths = resource["parameters"]["files"]
if type(paths) is list:
node["other"]["backups"].extend(paths)
else:
node["other"]["backups"].append(paths)
def load_logging(self, pdb):
for node, resource in self.query_classes(pdb, "Profile::Logging::Rsyslog::Client"):
node["other"]["logging"] = True
def load_metrics(self, pdb):
for node, resource in self.query_classes(pdb, "Profile::Metrics"):
node["other"]["metrics"] = True
def load_monitoring(self, pdb):
for node, resource in self.query_classes(pdb, "Profile::Server::Monitor"):
node["other"]["monitoring"] = True
for node, resource in self.query_classes(pdb, "Profile::Monitoring::Icinga2::Common"):
node["other"]["icinga_notification_period"] = resource["parameters"]["notification_period"]
node["other"]["icinga_environment"] = resource["parameters"]["icinga2_environment"]
node["other"]["icinga_owner"] = resource["parameters"]["owner"]
def load_roles(self, pdb):
self.roles = defaultdict(list)
condition = 'type = "Profile::Motd::Register" and file ~ "/site[.]pp$"'
for node, resource in self.query_resources(pdb, condition):
if resource["title"] not in ("role", "role::delivery"):
node["other"]["roles"].append(resource["title"])
self.roles[resource["title"]].append(node)
def sorted_nodes(self, section, key):
return sorted(
self.nodes.values(),
key=lambda node: node[section][key])
def sorted_roles(self):
return sorted(self.roles.items())
def sorted_services(self):
services = dict()
for node in self.nodes.values():
profile_metadata = node["facts"].get("profile_metadata", dict())
service_facts = profile_metadata.get("services", list())
for service_fact in service_facts:
class_name = service_fact["class_name"]
if class_name not in services:
services[class_name] = service_fact
services[class_name]["nodes"] = list()
services[class_name]["nodes"].append(node)
return sorted(services.values(), key=itemgetter("human_name"))

174
static/general.css Normal file
View file

@ -0,0 +1,174 @@
@charset "utf-8";
body {
margin: 0;
padding: 0;
font-family: sans-serif;
font-size: 14px;
line-height: 1.5em;
background: #000;
}
a {
text-decoration: none;
color: #00c;
}
nav {
background: #000;
border-bottom: 1px solid #fff;
outline: 1px solid #000;
}
nav > ul {
margin: 0;
padding: 7px 15px;
}
nav > ul > li {
display: inline-block;
margin: 0;
padding: 0 10px;
list-style-type: none;
}
nav a,
footer a {
font-weight: bold;
color: #fff;
}
nav a:after,
footer a:after {
font-family: "Arial Unicode MS", sans-serif;
content: "▸";
vertical-align: -1px;
padding-left: 1px;
}
nav a.backward:after,
footer a.backward:after {
content: "";
padding-left: 0;
}
nav a.backward:before,
footer a.backward:before {
font-family: "Arial Unicode MS", sans-serif;
content: "◂";
vertical-align: -1px;
padding-right: 1px;
}
footer {
position: relative;
overflow: hidden;
margin: 0;
padding: 1em 30px;
background: #000;
color: #fff;
border-top: 1px solid #fff;
outline: 1px solid #000;
}
#generated-at {
display: block;
float: right;
}
main {
background: #fff;
padding: 1px 1em 3em 1em; /* 1px to prevent margin collapse */
}
h1 {
margin: 0.75em 10px 1em 10px;
}
table {
min-width: 750px;
border-collapse: collapse;
}
th, td {
vertical-align: top;
text-align: left;
padding: 5px 10px;
border-bottom: 1px solid #ccc;
}
thead th {
border-bottom: 1px solid #000;
}
thead th.key_fqdn {
min-width: 220px;
}
tbody th.key_fqdn {
line-height: 1.3em;
}
tbody th.key_fqdn b {
float: left;
}
tbody th.key_fqdn span {
color: transparent;
}
tbody th.key_fqdn i {
/* Show hostname and domain on two lines, but copying and pasting should
produce the FQDN without a newline */
clear: left;
float: left;
font-size: 90%;
font-style: normal;
font-weight: normal;
color: #666;
}
body#nodes .key_monitoring,
body#nodes .key_backups,
body#nodes .key_logging,
body#nodes .key_metrics {
text-align: center;
}
td ul,
td li {
margin: 0;
padding: 0;
}
td ul {
list-style-type: none;
}
td p {
margin: 0 0 1em 0;
}
.notes {
max-width: 40em;
}
.notes ol,
.notes ul {
list-style-position: inside;
margin: 1em 0;
padding: 0;
}
.notes ol { list-style-type: decimal }
.notes ul { list-style-type: disc }
.notes li {
margin: 0;
padding: 0;
}
body#node th,
body#service th {
width: 180px;
}

16
static/general.js Normal file
View file

@ -0,0 +1,16 @@
// Update generated at time at bottom of pages
(function(){
try {
var generated_at = document.getElementById("generated-at");
var m = moment(generated_at.innerText);
var update_generated_at = function () {
generated_at.innerText = "Generated on " + m.format("MMMM Do, YYYY")
+ " at " + m.format("h:mm a") + " (" + m.fromNow() + ")";
}
update_generated_at()
setInterval(update_generated_at, 60*1000);
} catch ( e ) {
console.error(e);
}
})();

4463
static/moment.js Normal file

File diff suppressed because it is too large Load diff

10
templates/home.html Normal file
View file

@ -0,0 +1,10 @@
{% extends "layout.html" %}
{% block title %}Puppet SRE Infrastructure{% endblock %}
{% block body %}
<h1>Puppet SRE Infrastructure</h1>
<ul>
<li><a href="nodes/index.html">Node inventory</a></li>
<li><a href="roles/index.html">Role inventory</a></li>
<li><a href="services/index.html">Service inventory</a></li>
</ul>
{% endblock %}

28
templates/layout.html Normal file
View file

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>{% block title %}{% endblock %}</title>
<link rel="stylesheet" href="{{ path }}../static/general.css">
<link rel="stylesheet" href="{{ path }}pygments.css">
<script src="{{ path }}../static/moment.js" defer></script>
<script src="{{ path }}../static/general.js" defer></script>
</head>
<body id="{{ body_id }}">
<nav>
<ul>
<li><a href="{{ path }}index.html" class="backward">Home</a></li>
<li><a href="{{ path }}nodes/index.html">Nodes</a></li>
<li><a href="{{ path }}roles/index.html">Roles</a></li>
<li><a href="{{ path }}services/index.html">Services</a></li>
</ul>
</nav>
<main>
{% block body %}{% endblock %}
</main>
<footer>
{% block footer %}{% endblock %}
<div id="generated-at">{{ generation_time }}</div>
</footer>
</body>
</html>

15
templates/node.html Normal file
View file

@ -0,0 +1,15 @@
{% extends "layout.html" %}
{% block title %}Node {{ node["facts"]["fqdn"] }}{% endblock %}
{% block body %}
<h1>Node {{ node["facts"]["fqdn"] }}</h1>
<table>
<tbody>
{% for cell in columns %}
<tr>
{{ cell.head_html() }}
{{ cell.body_html(node) }}
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}

26
templates/nodes.html Normal file
View file

@ -0,0 +1,26 @@
{% extends "layout.html" %}
{% block title %}Node inventory{% endblock %}
{% block body %}
<h1>Node inventory</h1>
<table>
<thead>
<tr>
{% for cell in columns %}
{{ cell.head_html() }}
{% endfor %}
</tr>
</thead>
<tbody>
{% for node in nodes %}
<tr>
{% for cell in columns %}
{{ cell.body_html(node) }}
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
{% block footer %}
<a href="../nodes.csv">Download CSV</a>
{% endblock %}

27
templates/roles.html Normal file
View file

@ -0,0 +1,27 @@
{% extends "layout.html" %}
{% block title %}Role inventory{% endblock %}
{% block body %}
<h1>Role inventory</h1>
<table>
<thead>
<tr>
<th>Role</th>
<th>Nodes</th>
</tr>
</thead>
<tbody>
{% for role, nodes in roles %}
<tr id="{{ role }}">
<th>{{ role }}</th>
<td>
<ul>
{% for node in nodes | sort(attribute="facts.fqdn") %}
<li><a href="../nodes/{{ node["certname"] }}.html">{{ node["facts"]["fqdn"] }}</a></li>
{% endfor %}
</ul>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}

79
templates/service.html Normal file
View file

@ -0,0 +1,79 @@
{% extends "layout.html" %}
{% block title %}Service {{ service["human_name"] }}{% endblock %}
{% block body %}
<h1>Service {{ service["human_name"] }}</h1>
<table>
<tbody>
<tr>
<th>Class</th>
<td>{{ service["class_name"] }}</td>
</tr>
<tr>
<th>Documentation</th>
<td>
<ul>
{% for url in service["doc_urls"] %}
<li><a href="{{ url }}">{{url}}</a></li>
{% endfor %}
</ul>
</td>
</tr>
<tr>
<th>Downtime impact</th>
<td>{{ service["downtime_impact"] | unundef }}</td>
</tr>
<tr>
<th>End users</th>
<td>
{% if service["end_users"] != ":undef" %}
<ul>
{% for email in service["end_users"] %}
<li><a href="mailto:{{ email }}">{{email}}</a></li>
{% endfor %}
</ul>
{% endif %}
</td>
</tr>
<tr>
<th>Escalation period</th>
<td>{{ service["escalation_period"] | unundef }}</td>
</tr>
<tr>
<th>Notes</th>
<td>
<div class="notes">
{{ service["notes"] | unundef | markdown }}
</div>
</td>
</tr>
<tr>
<th>Other FQDNs</th>
<td>
<ul>
{% for fqdn in service["other_fqdns"] %}
<li>{{ fqdn }}</li>
{% endfor %}
</ul>
</td>
</tr>
<tr>
<th>Owner</th>
<td>{{ service["owner_uid"] | unundef }}</td>
</tr>
<tr>
<th>Team</th>
<td>{{ service["team"] | unundef }}</td>
</tr>
<tr>
<th>Nodes</th>
<td>
<ul>
{% for node in service["nodes"] | sort(attribute='facts.fqdn') %}
<li><a href="../nodes/{{ node["certname"] }}.html">{{ node["facts"]["fqdn"] }}</a></li>
{% endfor %}
</ul>
</td>
</tr>
</tbody>
</table>
{% endblock %}

29
templates/services.html Normal file
View file

@ -0,0 +1,29 @@
{% extends "layout.html" %}
{% block title %}Service inventory{% endblock %}
{% block body %}
<h1>Service inventory</h1>
<table>
<thead>
<tr>
<th>Service</th>
<th>Nodes</th>
</tr>
</thead>
<tbody>
{% for service in services %}
<tr id="{{ service }}">
<th>
<a href="./{{ service["class_name"] }}.html">{{ service["human_name"] }}</a>
</th>
<td>
<ul>
{% for node in service["nodes"] | sort(attribute="facts.fqdn") %}
<li><a href="../nodes/{{ node["certname"] }}.html">{{ node["facts"]["fqdn"] }}</a></li>
{% endfor %}
</ul>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}