mirror of
https://github.com/genebean/dots.git
synced 2026-03-27 01:17:42 -04:00
Monitoring time with Grafana & Prometheus
This commit is contained in:
parent
971acbf730
commit
de73b2dc27
SSH key fingerprint:
SHA256:gMnZbl3rg8nIXl4AomxNeNiOG4mWP/xEywbmQjVfhtY
4 changed files with 177 additions and 14 deletions
|
|
@ -3,6 +3,7 @@ gandi_api: ENC[AES256_GCM,data:YsdDMk75miIKO4LkCZjfwJw6gxfrmsTL,iv:BOPRxB661sPJn
|
|||
restic_env: ENC[AES256_GCM,data:FCYR8tkClRwfcjUotcr28D6uRz7sNihn50nw38CaYnqOD/U9+5kU0iAPSvqAbeuw+xUoKKKAPAfMHI12dPTYt17Wz1N7i4a+MRkiIR9pjyv5KZTK59G+,iv:jStc8GMbZUQUgooZiRdImSZskdckYN1cRm2gsKbUyYY=,tag:HpQQIj1j7fjCmxkSeY/k4g==,type:str]
|
||||
restic_repo: ENC[AES256_GCM,data:kCoNYVKwB87W4h5doa3IXj4n,iv:jKEw/Hki/tp3RSTsRB4dlg593I5B4pCLBav84ADCh70=,tag:+GFF5vHOVw0r/G8BbhcCjw==,type:str]
|
||||
restic_password: ENC[AES256_GCM,data:PfQsxJul1Qpt3WQoUEI941l+yng3lVjhDd8=,iv:U5KjhcVqyksN2ay19RBjNhYIB31tUbfNRIqCEx/+Wbc=,tag:jsoU+B1mjAprPK+M5I0pAQ==,type:str]
|
||||
uptimekuma_grafana_api_key: ENC[AES256_GCM,data:irDlzCrDO5n/pMRHagueBTrXmm3herjOSioAz9E5O4a9dJD4ALhp5LPpmTs=,iv:oJhm8eZWSEvqF3Whv/UrS6sgIRdrZ0eqWF98vdwD0V8=,tag:HvOBFo4CJ9NLSz9FwMCP2w==,type:str]
|
||||
wifi_creds: ENC[AES256_GCM,data:9lgTtI8YHyCHrvqss4W7coLnqfOAoQzrCQne6dLv0x66pt7jLo4Y6YSd3TklRTurS9usvNk3sg==,iv:6g86hOmpnOxf4p4C+wPit7EP0DD+xb+cINiWRJnTRDM=,tag:ZW336IhXtrf5l5n/RJecoQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
|
|
@ -100,8 +101,8 @@ sops:
|
|||
Y3I0d2xaYlRwVU9CdE44UW9vZ1NJeFUK5DQu30MuGjMq5YRSTh2II2uNvWm2XF9B
|
||||
YDcK/E1xKGIA/tKk/DDmpbUZMTIzh+tmYcN72EQQqlT/9a2HyINChg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2025-01-11T02:27:36Z"
|
||||
mac: ENC[AES256_GCM,data:Eezf5E3vh10bZTQCxGMZxioUoJqoc9rNBdMu+Wske1SC34Z8GzWbxy3s1T4RPB8I84woIigSgiwj03bZ/F5dchUqNtZwZDObpB44Ru9SEXMB/zEgM8g8LLha/Dgj1MFNbsVR9j2VhacDVv5XJKkjzr/TY5tKedi2mTSHUkpjAGI=,iv:U6YQWh0ISZ0LV6TrwA9QEGal/+lNyTV+tA6yCfNjd6A=,tag:6AUhKo1PniUxW98Yrq3Ulg==,type:str]
|
||||
lastmodified: "2025-01-14T03:30:50Z"
|
||||
mac: ENC[AES256_GCM,data:lOrSir70ZiZKjajRLUN83FoQQ0+hwLznbul6Z8hVjbxBvXdwvaMfi/BmfG/+wOMFjShU+fEStAjryoKCcaB3RJod2MyncvE4+fY2lmq7U/T1GHEknQ5xm42J6+Dd79P48mDsJ9kUQXO1wpp9CEVkW5hTfzPGYV2tRWY3a9hgz3Y=,iv:+3hE34n4f2zy17TeoDF/lWvFaX2Rd7ZsojlpZq9R4fE=,tag:Bcs3CAKIk+mTwy0dGuzVMg==,type:str]
|
||||
pgp: []
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.9.2
|
||||
|
|
|
|||
|
|
@ -37,8 +37,10 @@ in {
|
|||
jellyfin-web
|
||||
net-snmp
|
||||
nginx
|
||||
nvme-cli
|
||||
podman-compose
|
||||
podman-tui # status of containers in the terminal
|
||||
smartmontools
|
||||
yt-dlp
|
||||
];
|
||||
|
||||
|
|
@ -80,11 +82,14 @@ in {
|
|||
443 # https to local Nginx
|
||||
3000 # PsiTransfer in oci-container
|
||||
3001 # immich-kiosk in compose
|
||||
3002 # grafana
|
||||
3030 # Forgejo
|
||||
8001 # Tube Archivist
|
||||
8384 # Syncthing gui
|
||||
8888 # Atuin
|
||||
8090 # Wallabag in docker compose
|
||||
9090 # Prometheus Server
|
||||
9273 # Telegraf's Prometheus endpoint
|
||||
13378 # Audiobookshelf in oci-container
|
||||
22000 # Syncthing transfers
|
||||
];
|
||||
|
|
@ -194,6 +199,18 @@ in {
|
|||
stateDir = "/orico/forgejo";
|
||||
};
|
||||
fwupd.enable = true;
|
||||
grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
domain = "monitoring.${home_domain}";
|
||||
http_addr = "0.0.0.0";
|
||||
http_port = 3002;
|
||||
root_url = "https://monitoring.${home_domain}/grafana/"; # Not needed if it is `https://your.domain/`
|
||||
serve_from_sub_path = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
jellyfin = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
|
|
@ -300,6 +317,13 @@ in {
|
|||
add_header Content-Type text/html;
|
||||
'';
|
||||
};
|
||||
locations."/server_status" = {
|
||||
extraConfig = ''
|
||||
stub_status;
|
||||
allow 127.0.0.1;
|
||||
deny all;
|
||||
'';
|
||||
};
|
||||
};
|
||||
"ab.${home_domain}" = {
|
||||
listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
|
||||
|
|
@ -393,6 +417,13 @@ in {
|
|||
client_max_body_size 10M;
|
||||
'';
|
||||
};
|
||||
"monitoring.${home_domain}" = {
|
||||
listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
|
||||
enableACME = true;
|
||||
acmeRoot = null;
|
||||
forceSSL = true;
|
||||
locations."/grafana/".proxyPass = "http://${backend_ip}:3002";
|
||||
};
|
||||
"nc.${home_domain}" = {
|
||||
listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
|
||||
enableACME = true;
|
||||
|
|
@ -444,9 +475,45 @@ in {
|
|||
backupAll = true;
|
||||
startAt = "*-*-* 23:00:00";
|
||||
};
|
||||
prometheus = {
|
||||
enable = true;
|
||||
checkConfig = "syntax-only";
|
||||
globalConfig.scrape_interval = "10s"; # "1m"
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "hass";
|
||||
scrape_interval = "30s";
|
||||
metrics_path = "/api/prometheus";
|
||||
static_configs = [{
|
||||
targets = [ "192.168.22.22:8123" ];
|
||||
}];
|
||||
bearer_token_file = config.sops.secrets.home_assistant_token.path;
|
||||
}
|
||||
{
|
||||
job_name = "telegraf";
|
||||
scrape_interval = "5s";
|
||||
static_configs = [{
|
||||
targets = [ "localhost:9273" ];
|
||||
}];
|
||||
}
|
||||
{
|
||||
job_name = "uptimekuma";
|
||||
scheme = "https";
|
||||
scrape_interval = "30s";
|
||||
static_configs = [{
|
||||
targets = [ "utk.technicalissues.us" ];
|
||||
}];
|
||||
basic_auth = {
|
||||
password_file = config.sops.secrets.uptimekuma_grafana_api_key.path;
|
||||
username = "";
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
resolved.enable = true;
|
||||
restic.backups.daily.paths = [
|
||||
config.services.forgejo.stateDir
|
||||
config.services.grafana.dataDir
|
||||
config.services.mealie.settings.DATA_DIR
|
||||
config.services.nextcloud.home
|
||||
"${config.users.users.${username}.home}/compose-files/wallabag"
|
||||
|
|
@ -454,6 +521,7 @@ in {
|
|||
"/orico/jellyfin/data"
|
||||
"/orico/jellyfin/staging/downloaded-files"
|
||||
"/var/backup/postgresql"
|
||||
"/var/lib/prometheus2"
|
||||
];
|
||||
syncthing = {
|
||||
enable = true;
|
||||
|
|
@ -461,6 +529,83 @@ in {
|
|||
openDefaultPorts = true;
|
||||
guiAddress = "0.0.0.0:8384";
|
||||
};
|
||||
telegraf = {
|
||||
enable = true;
|
||||
extraConfig = {
|
||||
agent = {
|
||||
interval = "15s";
|
||||
round_interval = true;
|
||||
collection_jitter = "0s";
|
||||
debug = false;
|
||||
};
|
||||
inputs = {
|
||||
cpu = {
|
||||
percpu = true;
|
||||
totalcpu = true;
|
||||
report_active = true;
|
||||
collect_cpu_time = true;
|
||||
};
|
||||
disk = {
|
||||
ignore_fs = [
|
||||
"aufs"
|
||||
"devfs"
|
||||
"devtmpfs"
|
||||
"iso9660"
|
||||
"overlay"
|
||||
"squashfs"
|
||||
"tmpfs"
|
||||
];
|
||||
};
|
||||
diskio = {
|
||||
skip_serial_number = true;
|
||||
};
|
||||
docker = {
|
||||
endpoint = "unix:///run/docker.sock";
|
||||
container_name_include = [];
|
||||
storage_objects = [
|
||||
"container"
|
||||
"volume"
|
||||
];
|
||||
perdevice_include = [
|
||||
"cpu"
|
||||
"blkio"
|
||||
"network"
|
||||
];
|
||||
total_include = [
|
||||
"cpu"
|
||||
"blkio"
|
||||
"network"
|
||||
];
|
||||
timeout = "5s";
|
||||
gather_services = false;
|
||||
};
|
||||
mem = {};
|
||||
net = {
|
||||
ignore_protocol_stats = true;
|
||||
};
|
||||
nginx = {
|
||||
insecure_skip_verify = true;
|
||||
urls = [ "https://127.0.0.1/server_status" ];
|
||||
};
|
||||
#smart = {};
|
||||
system = {};
|
||||
#systemd_units = {};
|
||||
zfs = {
|
||||
datasetMetrics = true;
|
||||
poolMetrics = true;
|
||||
};
|
||||
};
|
||||
outputs = {
|
||||
prometheus_client = {
|
||||
collectors_exclude = [
|
||||
"gocollector"
|
||||
"process"
|
||||
];
|
||||
listen = ":9273";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
zfs.autoScrub.enable = true;
|
||||
};
|
||||
|
||||
|
|
@ -468,6 +613,10 @@ in {
|
|||
age.keyFile = "${config.users.users.${username}.home}/.config/sops/age/keys.txt";
|
||||
defaultSopsFile = ./secrets.yaml;
|
||||
secrets = {
|
||||
home_assistant_token = {
|
||||
owner = config.users.users.prometheus.name;
|
||||
restartUnits = ["prometheus.service"];
|
||||
};
|
||||
immich_kiosk_basic_auth = {
|
||||
owner = config.users.users.nginx.name;
|
||||
restartUnits = ["nginx.service"];
|
||||
|
|
@ -482,6 +631,11 @@ in {
|
|||
};
|
||||
mealie.mode = "0444";
|
||||
nextcloud_admin_pass.owner = config.users.users.nextcloud.name;
|
||||
uptimekuma_grafana_api_key = {
|
||||
owner = config.users.users.prometheus.name;
|
||||
restartUnits = ["prometheus.service"];
|
||||
sopsFile = ../../common/secrets.yaml;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -507,6 +661,8 @@ in {
|
|||
];
|
||||
};
|
||||
|
||||
users.users.telegraf.extraGroups = [ "docker" ];
|
||||
|
||||
# Enable common container config files in /etc/containers
|
||||
virtualisation.containers.enable = true;
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,11 @@
|
|||
fsType = "zfs";
|
||||
};
|
||||
|
||||
fileSystems."/var/lib/prometheus2" =
|
||||
{ device = "orico/prometheus";
|
||||
fsType = "zfs";
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
tailscale_key: ENC[AES256_GCM,data:aB3KUD4QYm+ZDrjjLcU3gQ8kneVGkVYBsrkVcioOhxunal2FekLDrpKxJwNXuiwx2M5vipnGAEPO,iv:e+tPPfVYkv4U0KRGwspWb1O3ZQom/WFFGm9H9cd/KKE=,tag:ZG5z1C18bj1L7DcGzunQ0w==,type:str]
|
||||
local_git_config: ENC[AES256_GCM,data:Nqwog5C4wnRzNoS4oqaYQ4J1DIj7fUL1y/nXESquR0N7KQ+ebhvuJnM=,iv:Q6o45LZStS3k8iO7s2P6u7OrKFu5alplshZuGgeRKmk=,tag:NcLJrI9AK4eDroODX15lcA==,type:str]
|
||||
local_private_env: ENC[AES256_GCM,data:qOPXTS2uo/1jyVEKCtBvuK/dzZaPf1K5tHuSVF2hBg4fdPYIsDPkM108cGVxJviebB3xVZejn/JVOdUDXQj6,iv:TtyMTOJXaPUrbSaAdtMaGPBlwLl/Y/IBYVCzhhiZozY=,tag:hUyVL8xk3w1iMwNAZw5QUw==,type:str]
|
||||
home_assistant_token: ENC[AES256_GCM,data:fNpoH60rXAsoVx/NBoDobDw/e6IoeoZfef1uDR7HbmnHNI101b+kQKkB8wBXEDLf7MlqlXVc1ExlgYFUo7+h2msx4WZUCGzuHtp1cMP0O9s2PZoQU8KQM2Frd69vOUcOT5y5ShJZPRrf6H2UKrm//3jE9zDOxxy3cQj6Q+jQLXX4AfZ12JAKzSiee9URWdU8eCHbnquUl1RNHF7zZQ8Mr9m41sBOrpBXt7ErsrhxxRhwlk7qprLt,iv:1j/QmOkLYd7nA+wXS49drBHU09HzMP3XxPbPdaErV6E=,tag:ftyOeNaN5PwNebeMPNAmTg==,type:str]
|
||||
immich_kiosk_basic_auth: ENC[AES256_GCM,data:R/vaYXe0SRwSUg8zC367vA/OYhjIcgfbHxR3OhLt1YYUhPNXVuUGmn199zLD,iv:GBEjJKvbwHAS75negE2whTlI4wS8K2nQYNm3015DFoY=,tag:zs+bJEr9JlIrJtikHycN4g==,type:str]
|
||||
mealie: ENC[AES256_GCM,data:HuO2c5na5r271x98GLqYBP/ME+phH58e2scelEH7F3R+2liLRGK9xCA0ciHLJCo8,iv:bo6hTIsWfnWueRCiLcpxYoiol8oGOoPTHDgPQ7FnYqM=,tag:rHRfIv4vnIjO/eotxYmI/Q==,type:str]
|
||||
nextcloud_admin_pass: ENC[AES256_GCM,data:KztB3Tkqlt73PEO41lthGYElrbwVdfqQgT6f,iv:kRwXqGJO4AUOMq+uYzndGhscaJiyvG4ANKabHHd78YM=,tag:dP3PgKafDTv8x7huKJGDqA==,type:str]
|
||||
|
|
@ -22,8 +23,8 @@ sops:
|
|||
bHZlNTZDV2NYU1hQQy9mem80SFF6TFkKfmjkJBfTdh0vTtGaVx1t3tHJvSsAwdYD
|
||||
PF025X9U+yG2oIopwXEVBkxcD70eyuJn3OqH0xoVLBkbhNM9i8LHrA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2025-01-10T16:08:06Z"
|
||||
mac: ENC[AES256_GCM,data:bwPvp28nSVurnSdyuLoFCZ5XbyZlAqeXVjSUk97H1R5EuMx+2UWGNIBm5Llgm+MwsUUkOpEI/V49M1zS8wRuKxsUA1JkmXG4nis856Tx2tl0SDoIkVdCYiq+8nZTWRv1BAYblny6HHUeBzWX3Sgx8NvXC+Qfkl1fuwJcSP2zvCk=,iv:WfzIryQ78UR4Rc9kOJFgBtpV5PkTt2qT3yx/4vDcASI=,tag:49RBwahEAIvicte0S89idw==,type:str]
|
||||
lastmodified: "2025-01-14T04:38:46Z"
|
||||
mac: ENC[AES256_GCM,data:yXoXF3BgOLegwht2BEDhplCxcVB72NdWRKmRzymJMjybuCLdO/IWk7Mk5Kqd5ckpYLq7vuRl2I+QoOGSiZwr8c8FT0eE8YKevMH6QQkGDZfEDAqJIO3FNT5G+ZaVkFLA8OwkAOhIbz3eFNwYU7mAwooKM/mlcEa7v0bmq17RMz0=,iv:LKZ2sP8YD1uGgKVfJT42/QpYPH5PnmzpJd5J+vlEhfg=,tag:Lc0BBOwa0OI3WwEp9Ctu6w==,type:str]
|
||||
pgp: []
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.9.2
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue