Merge pull request #591 from genebean/nominatim

Collection: Gandi, Nominatim, Pocket ID, new monitoring stack

modules/hosts/common/linux/lets-encrypt.nix

@@ -10,8 +10,7 @@
     acceptTerms = true;
     defaults = {
       email = "lets-encrypt@technicalissues.us";
-      credentialFiles = { "GANDIV5_API_KEY_FILE" = "${config.sops.secrets.gandi_api.path}"; };
+      credentialFiles = { "GANDIV5_PERSONAL_ACCESS_TOKEN_FILE" = "${config.sops.secrets.gandi_dns_pat.path}"; };
-      #credentialFiles = { "GANDIV5_PERSONAL_ACCESS_TOKEN_FILE" = gandi_dns_pat; };
       dnsProvider = "gandiv5";
       dnsResolver = "ns1.gandi.net";
       # uncomment below for testing
@@ -21,6 +20,6 @@
 
   sops = {
     age.keyFile = "${config.users.users.${username}.home}/.config/sops/age/keys.txt";
-    secrets.gandi_api.sopsFile = ../secrets.yaml;
+    secrets.gandi_dns_pat.sopsFile = ../secrets.yaml;
   };
 }

modules/hosts/common/secrets.yaml

@@ -1,11 +1,12 @@
-gandi_dns_pat: ENC[AES256_GCM,data:biWxwhrrE1ZOwViDtg0G0eIZz7+k804kBwN1icJWmh5TVi/Ylqbixw==,iv:pip7MXKdf5i0Ks7zdCs2O7UpxLq3HJY0KPNOwgta5+8=,tag:6X98FRXctX8cgBPY1pm+cw==,type:str]
+gandi_dns_pat: ENC[AES256_GCM,data:3L1RDSbCqkmLmguSgsJsf3gdnSi/zxS8xtl+B+kwBeaOnX3X5fmM7A==,iv:SWAfEAC/3klgreTppGZWV5SACrQEEL8tsXUvYFlJXyk=,tag:IiekQLSf1vnjRQr6ZRsVMQ==,type:str]
 gandi_api: ENC[AES256_GCM,data:YsdDMk75miIKO4LkCZjfwJw6gxfrmsTL,iv:BOPRxB661sPJnUH1AUKEALIJfBeyAHZpkWJEDbY+7i8=,tag:TvtW7qhPbOqi9kKDcIe28w==,type:str]
 hetzner_api_token: ENC[AES256_GCM,data:8+bYBnI6vSQ7QIDFv0zplU2A2lW2c7JA9WArCGeAgjg=,iv:Y92uRgjKfuGDY4HMr+j6uDweMmMCx0FBydP3alGgb3M=,tag:cbmeVnP1XcqE+T0qpzJfbw==,type:str]
 hetzner_lego_env: ENC[AES256_GCM,data:xRADnkMC/mTq8/oRpZ+NYTStB9qX2N6V0GNIpGsXNedgO3bTvowgMukyDW4nX19V627ykk5vPC/HTRhZ8ia2KxRJfqa+9n5+Eg83iAFtrQGOe2rvEGEHDUoCTSb/G8YA8XzB3t69Xc+o8g59Grf4rXvNLEEwewn92BP7YWoxvpPaeT3yl/g7/0m4SDXKR/D3LtiN4nikiUFYT6nBG+WipMK3oEw=,iv:dL4hw4/v1FgJKwmCzIpMKvryrm+mMb7SoohPi78paPY=,tag:Lq3vBkyVbv7w5/RIHcsiUg==,type:str]
 restic_env: ENC[AES256_GCM,data:FCYR8tkClRwfcjUotcr28D6uRz7sNihn50nw38CaYnqOD/U9+5kU0iAPSvqAbeuw+xUoKKKAPAfMHI12dPTYt17Wz1N7i4a+MRkiIR9pjyv5KZTK59G+,iv:jStc8GMbZUQUgooZiRdImSZskdckYN1cRm2gsKbUyYY=,tag:HpQQIj1j7fjCmxkSeY/k4g==,type:str]
 restic_repo: ENC[AES256_GCM,data:kCoNYVKwB87W4h5doa3IXj4n,iv:jKEw/Hki/tp3RSTsRB4dlg593I5B4pCLBav84ADCh70=,tag:+GFF5vHOVw0r/G8BbhcCjw==,type:str]
 restic_password: ENC[AES256_GCM,data:PfQsxJul1Qpt3WQoUEI941l+yng3lVjhDd8=,iv:U5KjhcVqyksN2ay19RBjNhYIB31tUbfNRIqCEx/+Wbc=,tag:jsoU+B1mjAprPK+M5I0pAQ==,type:str]
-uptimekuma_grafana_api_key: ENC[AES256_GCM,data:irDlzCrDO5n/pMRHagueBTrXmm3herjOSioAz9E5O4a9dJD4ALhp5LPpmTs=,iv:oJhm8eZWSEvqF3Whv/UrS6sgIRdrZ0eqWF98vdwD0V8=,tag:HvOBFo4CJ9NLSz9FwMCP2w==,type:str]
+vmagent_push_pw: ENC[AES256_GCM,data:Ey82+FQWOdTn8iTcCbn73A2pYIh8dfD6Dw==,iv:0xEtakEOzZd2wbg407aA0rLeZqPo5NpqWdTSD3VQ+yw=,tag:wINlAjbCYnw9L4odxKJFAg==,type:str]
+uptimekuma_grafana_api_key: ENC[AES256_GCM,data:cXdbdiEa4dqigcojFgE8Wf4esjbF7wmrx5BuEQltfwM5fNJQG6LP/8ZcopI=,iv:dUftPkEyy1DMq5fJTQiJlDab2UMAbwJt+81p35eBrAU=,tag:Nh+X6lTKB5EboADaSVK00g==,type:str]
 wifi_creds: ENC[AES256_GCM,data:9lgTtI8YHyCHrvqss4W7coLnqfOAoQzrCQne6dLv0x66pt7jLo4Y6YSd3TklRTurS9usvNk3sg==,iv:6g86hOmpnOxf4p4C+wPit7EP0DD+xb+cINiWRJnTRDM=,tag:ZW336IhXtrf5l5n/RJecoQ==,type:str]
 sops:
     age:
@@ -108,7 +109,7 @@ sops:
             ODFjcWxtRjkweGJvdzdWSEphMHRCdm8Kx0amHgaZZR26c+VRVTyBEnm+w5c5nA7R
             txHj1U349LbfEsovTqZAL1o2WuX+gmXSj1aeXPKW+S0bIagC6dDacA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-10-30T02:44:34Z"
+    lastmodified: "2026-02-01T03:15:26Z"
-    mac: ENC[AES256_GCM,data:CqqfSnNfUK8BI7n6/n7UbtANa0TmWkjmgb4aZwPzc1NPLXtH1xRMdysb8UtNFKwz5pDmGihT4VeVVu11vkOm6iPyS4no7FatkSA1zqGw97vo9kYKZETzKbw6a8nw1Lgbj6MRpxZQYidgir13AOiilzAEsEhzFddAOkNwr9K2NJ8=,iv:1Ns8+JKWeWdwCTIkQk1zTPDm8JtLtZ76gL5JU1A0100=,tag:j58QBexUW/SBZ5+kyoV0Zg==,type:str]
+    mac: ENC[AES256_GCM,data:gGk69AnZNAlA5fxViXZQMpGStn8v2L/IyIUAshyrlAKu8dcTDQfCDfC2kTlo0Q/gwhqchSKYFFKWWze7EUwUWmlYjJu5MEEK4aQ0HEObff27AKQloOln1X3jiIR7bnsKoakbAKJWYpGf6PWClbH64mjoJhBroG5amAwE25bevDE=,iv:bO2kFuGcq/1j2uo2y5/UwOHNQ5A8orStI190NMdAr1E=,tag:7xwClQ28I695qDICc0GS+A==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.11.0

modules/hosts/nixos/hetznix01/post-install/default.nix

@@ -7,6 +7,7 @@ in {
     ../../../common/linux/restic.nix
     ./containers/emqx.nix
     ./matrix-synapse.nix
+    ./monitoring.nix
     #./mosquitto.nix
     ./nginx.nix
   ];
@@ -59,6 +60,13 @@ in {
     dawarich = {
       enable = true;
       configureNginx = true;
+      environment = {
+        NOMINATIM_API_HOST = "nominatim.home.technicalissues.us";
+        NOMINATIM_API_USE_HTTPS = "true";
+      };
+      extraEnvFiles = [
+        "${config.sops.secrets.dawarich_env.path}"
+      ];
       localDomain = "location.technicalissues.us";
       smtp = {
         fromAddress = "location@hetznix01.technicalissues.us";
@@ -166,6 +174,10 @@ in {
         owner = "${username}";
         path = "${config.users.users.${username}.home}/.private-env";
       };
+      dawarich_env = {
+        owner = config.services.dawarich.user;
+        restartUnits = [ "dawarich-web.service" ];
+      };
       matrix_secrets_yaml = {
         owner = config.users.users.matrix-synapse.name;
         restartUnits = ["matrix-synapse.service"];

modules/hosts/nixos/hetznix01/post-install/monitoring.nix

@@ -0,0 +1,129 @@
+{ config, pkgs, ... }: let
+  metrics_server = "https://monitoring.home.technicalissues.us/remotewrite";
+in {
+  services = {
+    vmagent = {
+      enable = true;
+      package = pkgs.victoriametrics;
+
+      # Prometheus-style scrape configuration
+      prometheusConfig = {
+        global.scrape_interval = "15s";
+
+        scrape_configs = [
+          {
+            job_name = "node";
+            static_configs = [
+              { targets = ["127.0.0.1:9100"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                regex = "127.0.0.1.*";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+
+          # Nginx exporter
+          {
+            job_name = "nginx";
+            static_configs = [
+              { targets = ["127.0.0.1:9113"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+        ];
+      };
+
+      # Remote write to VictoriaMetrics
+      remoteWrite = {
+        basicAuthUsername = "metricsshipper";
+        basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path;
+        url = metrics_server;
+      };
+
+      extraArgs = [
+        # Pass other remote write flags the module does not expose natively:
+        "-remoteWrite.flushInterval=10s"
+        "-remoteWrite.maxDiskUsagePerURL=1GB"
+
+        # Prevent vmagent from failing the entire scrape if a target is down:
+        "-promscrape.suppressScrapeErrors"
+
+        # Enable some debugging info suggested by the interface on port 8429
+        "-promscrape.dropOriginalLabels=false"
+      ];
+    };
+
+    # ----------------------------
+    # Exporters (using built-in NixOS modules)
+    # ----------------------------
+    
+    # Node exporter - using the built-in module
+    prometheus.exporters.node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9100;
+      enabledCollectors = [
+        "systemd"
+      ];
+      extraFlags = [
+        "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)"
+        "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$"
+      ];
+    };
+
+    # Nginx exporter - using the built-in module
+    prometheus.exporters.nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9113;
+      scrapeUri = "https://127.0.0.1/server_status";
+      sslVerify = false;
+    };
+  };
+
+  # ----------------------------
+  # Users and groups for service accounts
+  # ----------------------------
+  users.users.vmagent = {
+    isSystemUser = true;
+    group = "vmagent";
+  };
+
+  users.groups.vmagent = {};
+
+  # ----------------------------
+  # SOPS secrets configuration
+  # ----------------------------
+  sops = {
+    secrets = {
+      vmagent_push_pw = {
+        owner = "vmagent";
+        restartUnits = ["vmagent.service"];
+        sopsFile = ../../../common/secrets.yaml;
+      };
+    };
+  };
+}
+

modules/hosts/nixos/hetznix01/secrets.yaml

@@ -1,6 +1,7 @@
 local_git_config: ENC[AES256_GCM,data:BulcGoJ85+BA3maqbMewUdaNOl3feaJMq/4yZL8Y8SLOHqzmA/DUO7k=,iv:V7wpSiEQpt7AhKd+MUyGqTsO6YZovpkj+AaqpLnfRM0=,tag:7f3fFzQX3bpjokVPnUKDPQ==,type:str]
 local_private_env: ENC[AES256_GCM,data:OFcCaE9/hpd6JIoUTTxg0pEFL3rkUE3G+JzP/wjFXpa/AJa2Rr0Kv42Pu+iwgPMWgcpp50ChjVxGvbceNQ==,iv:I2LyWwvdMdE4wKLb3udLVMu3jFsvYR1ruZvaVt9GG7c=,tag:tBPmlNr0iNdLRU1GIRV2mg==,type:str]
 emqx_env: ENC[AES256_GCM,data:NGGMGAtY1s8ojVjMYahS80ichwBFGWrQI3qn2zc3bKr3wdQrcx5p9O0fMPqff6rH3w==,iv:OFEkDybGFnUQXzVJAJ3tTAShfeUvzwE4bLecUQ/YPjQ=,tag:XynNkgXxdga8DmiXZ7Sy9Q==,type:str]
+dawarich_env: ENC[AES256_GCM,data:mS821nAHy+vt8gwex6ZxcYMl8cue8QTnv1tbjfyGIyYygB2U7cpilX7mHMkJ4LjXmkBDII0LB//cJQsJMvP6WwcduwWD3FcXLFg+FoRtWRTYAw9J6lnR8m8cDOQ9tdwQM4NaUjcF44hUpai3Z+CT23XvGj8v3hgfNfcn/C0AqLSZQWO036X8R88MU88CCAaLunlSzwxB8U/H4rhXvTRVJAUUezPw2kFxEbHYC2amd1a0Imxrc5MHMbFLMkYIGVTBbj5y6RR7TsBMYFz29oVcBCv0BilJ/hOj524u9Ijsr9/upWj8otF0KFA5TvagUBWJxlyH1DsSZxE=,iv:HWzA2jAPOq7wrMDaaD73k/v/KRCFQ+Qy2b19Dg7/Teo=,tag:sLrZqXUBk+sYqYsUOae7cg==,type:str]
 matrix_secrets_yaml: ENC[AES256_GCM,data:6DLtAZIYBlL7iQVS/FBeUEhHyAOFZ5JRNqFBqi59GVh7cP0Hp8RBWxKpWAH2eUPYqUqUGCKrSSH3sJqzV+vasSR62tcltV7+13+q+rZVCZNCEf21EwQ5aaxgR3yG4n3YUPqLsCQB6UnWn0tF5HO0ofjYkya0pQ/nX9TBiiqIcPcd4NovbTtf+S0G0VptqyXAuRvJoKCx42ft9IBfV9tF1QsXLemKYlI10hN5l/MgJHwVbwH5xXR2kLKvnlpAyIoST/uJhswQV9DyK9cnl09ZM9ztcXhveBzv6uDW+pme8lFL99SMtMJcbSzxYW/pt+GJgYd1NiaoPbayWM72jdpH0hf2zWchxnIJIyL3H6EzIjD8BE9GnMP7ujQwBZGNZITRSg==,iv:cDtuOhv2v6CZcwiMM3oqjmajIl7D8Im+LkfarcjTM/w=,tag:e7zRQBYslJqESOGN3c4/aw==,type:str]
 matrix_homeserver_signing_key: ENC[AES256_GCM,data:+RflNxFfS2w9LbavT7YnCQIhJWI49kN7pOa9/dH0BpDWxKQaLE4ZYBYq0ikAgcHaF3+rBL3f6KxUacw=,iv:6+nZzuxBUwjM74XHCD89YWfyuMRcoIwQlHLiNN4NWdc=,tag:91yigynRz6QdEd4rF7d/9g==,type:str]
 mosquitto_mountain_mesh: ENC[AES256_GCM,data:LczPsPtAgkTTGcG3KYXMkfeA67e81Q5zJ5Nb8JcSosvvUwJRUi6yDcV/0wsYbMxeWDMrE/p+2KFRI48BVcUbY/LXqyFu5iNbX5IJXxzrexXXSTnOLa2PEamESzQlWI0ZS+K0Q48/5v9ekNVOkPgNQQ==,iv:jfa0QKOp8fyieUYTbMnBJ18VZwPO2CVnYQECHLNCyPI=,tag:9YZU82XQUmLJAFK+AiZ/Vw==,type:str]
@@ -25,7 +26,7 @@ sops:
             WkI4ejBaODI0d0tjWHpTT3VWTXNyaXcKMDtvHN4gcZqBNslyC+NwYW05zgs8QuPV
             W6EktAz+xu6kx5BJbli5GkUFmj52AtEGIqZ1Sr4a0pKQACC87XcTQA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-09-30T01:05:21Z"
+    lastmodified: "2026-01-26T04:52:54Z"
-    mac: ENC[AES256_GCM,data:JqLWrABSqRI5c2h0IE8G5+w7qBBOMflE2zVkcxPaZ8HgtjcStJFtrJu4p4PDkro/PCZ5fh1CgWteRw225xlTIGH9IN7Y+PV4tkgRM1r33ZolnFIfZZEKnEN1+Fyb6F70tfWgsj6lhxZvPUfoVHIOGXGYGTMEncT1VtH9mRghCRA=,iv:MbLd64qHie/8c5h03s3PPVLhJTpP3ZToRGxgsxErPOk=,tag:29oL10fXCvrUl7Myd48diA==,type:str]
+    mac: ENC[AES256_GCM,data:1pDP/ENGdw+bp9euYeiq3V7FNg6/IO88mbBcaYoI+Zi2TCETmCJXGupMKPeyTQ+OJf9C5yHaBHcVISLyJbrDehLp4ndCvNxl0jboytub3YCn0s7QxOLmNI0guRO45EPpPKQekD9TTqv/8QxvKqTvfmIlpdb+Vfii2XTsY5fkNA0=,iv:n7UcVvOYVyvAu74IJUE506GboPTg7rOekgsgoyx8ig4=,tag:gkrV7H/OKAEldFbuD1VEfw==,type:str]
     unencrypted_suffix: _unencrypted
-    version: 3.10.2
+    version: 3.11.0

modules/hosts/nixos/hetznix02/post-install/default.nix

@@ -1,6 +1,7 @@
 { config, username, ... }: {
   imports = [
     ../../../common/linux/lets-encrypt.nix
+    ./monitoring.nix
     ./nginx.nix
   ];
 

modules/hosts/nixos/hetznix02/post-install/monitoring.nix

@@ -0,0 +1,129 @@
+{ config, pkgs, ... }: let
+  metrics_server = "https://monitoring.home.technicalissues.us/remotewrite";
+in {
+  services = {
+    vmagent = {
+      enable = true;
+      package = pkgs.victoriametrics;
+
+      # Prometheus-style scrape configuration
+      prometheusConfig = {
+        global.scrape_interval = "15s";
+
+        scrape_configs = [
+          {
+            job_name = "node";
+            static_configs = [
+              { targets = ["127.0.0.1:9100"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                regex = "127.0.0.1.*";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+
+          # Nginx exporter
+          {
+            job_name = "nginx";
+            static_configs = [
+              { targets = ["127.0.0.1:9113"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+        ];
+      };
+
+      # Remote write to VictoriaMetrics
+      remoteWrite = {
+        basicAuthUsername = "metricsshipper";
+        basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path;
+        url = metrics_server;
+      };
+
+      extraArgs = [
+        # Pass other remote write flags the module does not expose natively:
+        "-remoteWrite.flushInterval=10s"
+        "-remoteWrite.maxDiskUsagePerURL=1GB"
+
+        # Prevent vmagent from failing the entire scrape if a target is down:
+        "-promscrape.suppressScrapeErrors"
+
+        # Enable some debugging info suggested by the interface on port 8429
+        "-promscrape.dropOriginalLabels=false"
+      ];
+    };
+
+    # ----------------------------
+    # Exporters (using built-in NixOS modules)
+    # ----------------------------
+    
+    # Node exporter - using the built-in module
+    prometheus.exporters.node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9100;
+      enabledCollectors = [
+        "systemd"
+      ];
+      extraFlags = [
+        "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)"
+        "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$"
+      ];
+    };
+
+    # Nginx exporter - using the built-in module
+    prometheus.exporters.nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9113;
+      scrapeUri = "https://127.0.0.1/server_status";
+      sslVerify = false;
+    };
+  };
+
+  # ----------------------------
+  # Users and groups for service accounts
+  # ----------------------------
+  users.users.vmagent = {
+    isSystemUser = true;
+    group = "vmagent";
+  };
+
+  users.groups.vmagent = {};
+
+  # ----------------------------
+  # SOPS secrets configuration
+  # ----------------------------
+  sops = {
+    secrets = {
+      vmagent_push_pw = {
+        owner = "vmagent";
+        restartUnits = ["vmagent.service"];
+        sopsFile = ../../../common/secrets.yaml;
+      };
+    };
+  };
+}
+

modules/hosts/nixos/kiosk-entryway/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./disk-config.nix
     ./hardware-configuration.nix
+    ./monitoring.nix
   ];
 
   system.stateVersion = "24.11";

modules/hosts/nixos/kiosk-entryway/monitoring.nix

@@ -0,0 +1,129 @@
+{ config, pkgs, ... }: let
+  metrics_server = "https://monitoring.home.technicalissues.us/remotewrite";
+in {
+  services = {
+    vmagent = {
+      enable = true;
+      package = pkgs.victoriametrics;
+
+      # Prometheus-style scrape configuration
+      prometheusConfig = {
+        global.scrape_interval = "15s";
+
+        scrape_configs = [
+          {
+            job_name = "node";
+            static_configs = [
+              { targets = ["127.0.0.1:9100"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                regex = "127.0.0.1.*";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+
+          # Nginx exporter
+          {
+            job_name = "nginx";
+            static_configs = [
+              { targets = ["127.0.0.1:9113"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+        ];
+      };
+
+      # Remote write to VictoriaMetrics
+      remoteWrite = {
+        basicAuthUsername = "metricsshipper";
+        basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path;
+        url = metrics_server;
+      };
+
+      extraArgs = [
+        # Pass other remote write flags the module does not expose natively:
+        "-remoteWrite.flushInterval=10s"
+        "-remoteWrite.maxDiskUsagePerURL=1GB"
+
+        # Prevent vmagent from failing the entire scrape if a target is down:
+        "-promscrape.suppressScrapeErrors"
+
+        # Enable some debugging info suggested by the interface on port 8429
+        "-promscrape.dropOriginalLabels=false"
+      ];
+    };
+
+    # ----------------------------
+    # Exporters (using built-in NixOS modules)
+    # ----------------------------
+    
+    # Node exporter - using the built-in module
+    prometheus.exporters.node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9100;
+      enabledCollectors = [
+        "systemd"
+      ];
+      extraFlags = [
+        "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)"
+        "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$"
+      ];
+    };
+
+    # Nginx exporter - using the built-in module
+    prometheus.exporters.nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9113;
+      scrapeUri = "https://127.0.0.1/server_status";
+      sslVerify = false;
+    };
+  };
+
+  # ----------------------------
+  # Users and groups for service accounts
+  # ----------------------------
+  users.users.vmagent = {
+    isSystemUser = true;
+    group = "vmagent";
+  };
+
+  users.groups.vmagent = {};
+
+  # ----------------------------
+  # SOPS secrets configuration
+  # ----------------------------
+  sops = {
+    secrets = {
+      vmagent_push_pw = {
+        owner = "vmagent";
+        restartUnits = ["vmagent.service"];
+        sopsFile = ../../../common/secrets.yaml;
+      };
+    };
+  };
+}
+

modules/hosts/nixos/nixnuc/containers/nginx-proxy.nix

@@ -1,181 +0,0 @@
-{ config, ... }: let
-  http_port = 80;
-  https_port = 443;
-  gandi_api = "${config.sops.secrets.gandi_api.path}";
-  #gandi_dns_pat = "${config.sops.secrets.gandi_dns_pat.path}";
-  home_domain = "home.technicalissues.us";
-  backend_ip = "192.168.20.190";
-  mini_watcher = "192.168.23.20";
-in {
-  sops.secrets.gandi_api = {
-    sopsFile = ../../../../system/common/secrets.yaml;
-    restartUnits = [
-      "container@nginx-proxy.service"
-    ];
-  };
-  #sops.secrets.gandi_dns_pat = {
-  #  sopsFile = ../../../../system/common/secrets.yaml;
-  #  restartUnits = [
-  #    "container@nginx-proxy.service"
-  #  ];
-  #};
-
-  ##
-  ## Gandi (gandi.net)
-  ##
-  ## Single host update
-  # protocol=gandi
-  # zone=example.com
-  # password=my-gandi-access-token
-  # use-personal-access-token=yes
-  # ttl=10800 # optional
-  # myhost.example.com
-  services.ddclient = {
-    enable = true;
-    protocol = "gandi";
-    zone = "technicalissues.us";
-    domains = [ home_domain ];
-    username = "unused";
-    extraConfig = ''
-      usev4=webv4
-      #usev6=webv6
-      #use-personal-access-token=yes
-      ttl=300
-    '';
-    passwordFile = gandi_api; };
-
-  containers.nginx-proxy = {
-    bindMounts."${gandi_api}".isReadOnly = true;
-    #bindMounts."${gandi_dns_pat}".isReadOnly = true;
-    autoStart = true;
-    timeoutStartSec = "5min";
-    privateNetwork = true;
-    hostBridge = "br1-23";
-    localAddress = "192.168.23.21/24";
-    config = { config, pkgs, lib, ... }: {
-      system.stateVersion = "23.11";
-
-      programs.traceroute.enable = true;
-
-      services.nginx = {
-        enable = true;
-        recommendedGzipSettings = true;
-        recommendedOptimisation = true;
-        recommendedProxySettings = true;
-        recommendedTlsSettings = true;
-        appendHttpConfig = ''
-          # Add HSTS header with preloading to HTTPS requests.
-          # Adding this header to HTTP requests is discouraged
-          map $scheme $hsts_header {
-              https   "max-age=31536000;";
-          }
-          add_header Strict-Transport-Security $hsts_header;
-        '';
-
-        virtualHosts = {
-          "${home_domain}" = {
-            serverAliases = [ "nix-tester.${home_domain}" ];
-            default = true;
-            listen = [
-              { port = http_port; addr = "0.0.0.0"; }
-              { port = https_port; addr = "0.0.0.0"; ssl = true; }
-            ];
-            enableACME = true;
-            acmeRoot = null;
-            addSSL = true;
-            forceSSL = false;
-            locations."/" = {
-              return = "200 '<h1>Hello world ;)</h1>'";
-              extraConfig = ''
-                add_header Content-Type text/html;
-              '';
-            };
-          };
-          "ab.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            locations."/".proxyWebsockets = true;
-            locations."/".proxyPass = "http://${backend_ip}:13378";
-          };
-          "atuin.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            locations."/".proxyPass = "http://${mini_watcher}:9999";
-          };
-          "nc.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            extraConfig = ''
-              client_max_body_size 0;
-              underscores_in_headers on;
-            '';
-            locations."/".proxyWebsockets = true;
-            locations."/".proxyPass = "http://${mini_watcher}:8081";
-            locations."/".extraConfig = ''
-              # these are added per https://www.nicemicro.com/tutorials/debian-snap-nextcloud.html
-              add_header Front-End-Https on;
-              proxy_headers_hash_max_size 512;
-              proxy_headers_hash_bucket_size 64;
-              proxy_buffering off;
-              proxy_max_temp_file_size 0;
-            '';
-          };
-          "onlyoffice.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            locations."/".proxyWebsockets = true;
-            locations."/".proxyPass = "http://${mini_watcher}:8888";
-          };
-          "readit.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            locations."/".proxyPass = "http://${backend_ip}:8090";
-          };
-          "tandoor.${home_domain}" = {
-            listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
-            enableACME = true;
-            acmeRoot = null;
-            forceSSL = true;
-            locations."/".proxyPass = "http://${backend_ip}:8080";
-          };
-        };
-      };
-
-      security.acme = {
-        acceptTerms = true;
-        defaults = {
-          email = "lets-encrypt@technicalissues.us";
-          credentialFiles = { "GANDIV5_API_KEY_FILE" = gandi_api; };
-          #credentialFiles = { "GANDIV5_PERSONAL_ACCESS_TOKEN_FILE" = gandi_dns_pat; };
-          dnsProvider = "gandiv5";
-          dnsResolver = "ns1.gandi.net";
-          # uncomment below for testing
-          #server = "https://acme-staging-v02.api.letsencrypt.org/directory";
-        };
-      };
-
-      networking = {
-        firewall = {
-          enable = true;
-          allowedTCPPorts = [ http_port https_port ];
-        };
-        defaultGateway = "192.168.23.1";
-        # Use systemd-resolved inside the container
-        # Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686
-        useHostResolvConf = lib.mkForce false;
-      };
-
-      services.resolved.enable = true;
-    };
-  };
-}

modules/hosts/nixos/nixnuc/default.nix

@@ -3,7 +3,6 @@
   https_port = 443;
   home_domain = "home.technicalissues.us";
   backend_ip = "127.0.0.1";
-  mini_watcher = "192.168.23.20";
   restic_backup_time = "02:00";
 in {
   imports = [
@@ -11,6 +10,7 @@ in {
     ./containers/audiobookshelf.nix
     ./containers/mountain-mesh-bot-discord.nix
     ./containers/psitransfer.nix
+    ./monitoring-stack.nix
     ../../common/linux/lets-encrypt.nix
     ../../common/linux/restic.nix
   ];
@@ -96,16 +96,11 @@ in {
          8888 # Atuin
          8090 # Wallabag in docker compose
          8945 # Pinchflat
-         9090 # Prometheus Server
-         9273 # Telegraf's Prometheus endpoint
         13378 # Audiobookshelf in oci-container
-        22000 # Syncthing transfers
       ];
       allowedUDPPorts = [
          1900 # Jellyfin service auto-discovery
          7359 # Jellyfin auto-discovery
-        21027 # Syncthing discovery
-        22000 # Syncthing transfers
       ];
     };
     # Or disable the firewall altogether.
@@ -135,7 +130,9 @@ in {
   };
   services.pulseaudio.enable = false;
 
-  programs.mtr.enable = true;
+  programs = {
+    mtr.enable = true;
+  };
 
   # List services that you want to enable:
   services = {
@@ -168,10 +165,10 @@ in {
       extraConfig = ''
         usev4=webv4
         #usev6=webv6
-        #use-personal-access-token=yes
+        use-personal-access-token=yes
         ttl=300
       '';
-      passwordFile = "${config.sops.secrets.gandi_api.path}";
+      passwordFile = "${config.sops.secrets.gandi_dns_pat.path}";
     };
     firefly-iii = {
       enable = true;
@@ -219,18 +216,6 @@ in {
       stateDir = "/orico/forgejo";
     };
     fwupd.enable = true;
-    grafana = {
-      enable = true;
-      settings = {
-        server = {
-          domain = "monitoring.${home_domain}";
-          http_addr = "0.0.0.0";
-          http_port = 3002;
-          root_url = "https://monitoring.${home_domain}/grafana/"; # Not needed if it is `https://your.domain/`
-          serve_from_sub_path = true;
-        };
-      };
-    };
     jellyfin = {
       enable = true;
       openFirewall = true;
@@ -376,6 +361,18 @@ in {
             client_max_body_size 0;
           '';
         };
+        "id.${home_domain}" = {
+          listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
+          enableACME = true;
+          acmeRoot = null;
+          forceSSL = true;
+          locations."/".proxyPass = "http://${backend_ip}:1411";
+          extraConfig = ''
+            proxy_busy_buffers_size   512k;
+            proxy_buffers           4 512k;
+            proxy_buffer_size         256k;
+          '';
+        };
         "immich.${home_domain}" = {
           listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
           enableACME = true;
@@ -445,40 +442,31 @@ in {
           enableACME = true;
           acmeRoot = null;
           forceSSL = true;
-          locations."/grafana/".proxyPass = "http://${backend_ip}:3002/grafana/";
+          locations = {
-        };
+            "/grafana/".proxyPass = "http://${backend_ip}:3002/grafana/";
-        "nc.${home_domain}" = {
+            "/remotewrite" = {
-          listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
+              basicAuthFile = config.sops.secrets.nginx_basic_auth.path;
-          enableACME = true;
+              proxyPass = "http://127.0.0.1:8428/api/v1/write";
-          acmeRoot = null;
+              proxyWebsockets = true;
-          forceSSL = true;
+            };
-          extraConfig = ''
+          };
-            client_max_body_size 0;
-            underscores_in_headers on;
-          '';
-          locations."/".proxyWebsockets = true;
-          locations."/".proxyPass = "http://${mini_watcher}:8081";
-          locations."/".extraConfig = ''
-            # these are added per https://www.nicemicro.com/tutorials/debian-snap-nextcloud.html
-            add_header Front-End-Https on;
-            proxy_headers_hash_max_size 512;
-            proxy_headers_hash_bucket_size 64;
-            proxy_buffering off;
-            proxy_max_temp_file_size 0;
-          '';
         };
         "nextcloud.${home_domain}" = {
           enableACME = true;
           acmeRoot = null;
           forceSSL = true;
         };
-        "onlyoffice.${home_domain}" = {
+        "nominatim.${home_domain}" = {
-          listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
           enableACME = true;
           acmeRoot = null;
           forceSSL = true;
-          locations."/".proxyWebsockets = true;
+          extraConfig = ''
-          locations."/".proxyPass = "http://${mini_watcher}:8888";
+            allow 127.0.0.1;
+            allow ::1;
+            allow 2600:1700:1712:880f:8eee:4ba4:75dc:f39c;
+            allow 100.64.0.0/10;
+            deny all;
+          '';
         };
         "readit.${home_domain}" = {
           listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
@@ -489,6 +477,17 @@ in {
         };
       };
     };
+    nominatim = {
+      enable = true;
+      hostName = "nominatim.home.technicalissues.us";
+      settings = {
+        NOMINATIM_PROJECT_DIR = "/var/lib/nominatim/project";
+      };
+      ui.config = ''
+        Nominatim_Config.Page_Title="Beantown's Nominatim";
+        Nominatim_Config.Nominatim_API_Endpoint='https://${config.services.nominatim.hostName}/';
+      '';
+    };
     pinchflat = {
       enable = true;
       group = "jellyfin";
@@ -496,55 +495,33 @@ in {
       selfhosted = true; # Only because this is not exsposed to the web
       user = "jellyfin";
     };
+    pocket-id = {
+      enable = true;
+      settings = {
+        APP_URL = "https://id.${home_domain}";
+        TRUST_PROXY = true;
+      };
+    };
     postgresql = {
       enable = true;
       package = pkgs.postgresql_16;
+      ensureUsers = [
+        {
+          # Required by Nominatim
+          name = "www-data";
+          ensureDBOwnership = false;
+        }
+      ];
     };
     postgresqlBackup = {
       enable = true;
       backupAll = true;
       startAt = "*-*-* 23:00:00";
     };
-    prometheus = {
-      enable = true;
-      checkConfig = "syntax-only";
-      globalConfig.scrape_interval = "10s"; # "1m"
-      scrapeConfigs = [
-        {
-          job_name = "hass";
-          scrape_interval = "30s";
-          metrics_path = "/api/prometheus";
-          static_configs = [{
-            targets = [ "192.168.22.22:8123" ];
-          }];
-          bearer_token_file = config.sops.secrets.home_assistant_token.path;
-        }
-        {
-          job_name = "telegraf";
-          scrape_interval = "5s";
-          static_configs = [{
-            targets = [ "localhost:9273" ];
-          }];
-        }
-        {
-          job_name = "uptimekuma";
-          scheme = "https";
-          scrape_interval = "30s";
-          static_configs = [{
-            targets = [ "utk.technicalissues.us" ];
-          }];
-          basic_auth = {
-            password_file = config.sops.secrets.uptimekuma_grafana_api_key.path;
-            username = "";
-          };
-        }
-      ];
-    };
     resolved.enable = true;
     restic.backups.daily = {
       paths = [
         config.services.forgejo.stateDir
-        config.services.grafana.dataDir
         config.services.mealie.settings.DATA_DIR
         config.services.nextcloud.home
         "${config.users.users.${username}.home}/compose-files/wallabag"
@@ -552,7 +529,6 @@ in {
         "/orico/jellyfin/data"
         "/orico/jellyfin/staging/downloaded-files"
         "/var/backup/postgresql"
-        "/var/lib/prometheus2"
       ];
       timerConfig = {
         OnCalendar = restic_backup_time;
@@ -560,12 +536,6 @@ in {
       };
     };
     smartd.enable = true;
-    syncthing = {
-      enable = true;
-      dataDir = "/orico/syncthing";
-      openDefaultPorts = true;
-      guiAddress = "0.0.0.0:8384";
-    };
     tailscale = {
       enable = true;
       authKeyFile = config.sops.secrets.tailscale_key.path;
@@ -578,83 +548,6 @@ in {
       ];
       useRoutingFeatures = "both";
     };
-    telegraf = {
-      enable = true;
-      extraConfig = {
-        agent = {
-          interval = "15s";
-          round_interval = true;
-          collection_jitter = "0s";
-          debug = false;
-        };
-        inputs = {
-          cpu = {
-            percpu = true;
-            totalcpu = true;
-            report_active = true;
-            collect_cpu_time = true;
-          };
-          disk = {
-            ignore_fs = [
-              "aufs"
-              "devfs"
-              "devtmpfs"
-              "iso9660"
-              "overlay"
-              "squashfs"
-              "tmpfs"
-            ];
-          };
-          diskio = {
-            skip_serial_number = true;
-          };
-          docker = {
-            endpoint = "unix:///run/docker.sock";
-            container_name_include  = [];
-            storage_objects = [
-              "container"
-              "volume"
-            ];
-            perdevice_include = [
-              "cpu"
-              "blkio"
-              "network"
-            ];
-            total_include = [
-              "cpu"
-              "blkio"
-              "network"
-            ];
-            timeout = "5s";
-            gather_services = false;
-          };
-          mem = {};
-          net = {
-            ignore_protocol_stats = true;
-          };
-          nginx = {
-            insecure_skip_verify = true;
-            urls = [ "https://127.0.0.1/server_status" ];
-          };
-          #smart = {};
-          system = {};
-          #systemd_units = {};
-          zfs = {
-            datasetMetrics = true;
-            poolMetrics = true;
-          };
-        };
-        outputs = {
-          prometheus_client = {
-            collectors_exclude = [
-              "gocollector"
-              "process"
-            ];
-            listen = ":9273";
-          };
-        };
-      };
-    };
     zfs.autoScrub.enable = true;
   };
 
@@ -687,10 +580,6 @@ in {
           "phpfpm-firefly-iii-data-importer.service"
         ];
       };
-      home_assistant_token = {
-        owner = config.users.users.prometheus.name;
-        restartUnits = ["prometheus.service"];
-      };
       immich_kiosk_basic_auth = {
         owner = config.users.users.nginx.name;
         restartUnits = ["nginx.service"];
@@ -708,14 +597,13 @@ in {
         restartUnits = ["mealie.service"];
       };
       nextcloud_admin_pass.owner = config.users.users.nextcloud.name;
+      nginx_basic_auth = {
+        owner = "nginx";
+        restartUnits = ["nginx.service"];
+      };
       tailscale_key = {
         restartUnits = [ "tailscaled-autoconnect.service" ];
       };
-      uptimekuma_grafana_api_key = {
-        owner = config.users.users.prometheus.name;
-        restartUnits = ["prometheus.service"];
-        sopsFile = ../../common/secrets.yaml;
-      };
     };
   };
 
@@ -743,8 +631,6 @@ in {
     ];
   };
 
-  users.users.telegraf.extraGroups = [ "docker" ];
-
   # Enable common container config files in /etc/containers
   virtualisation.containers.enable = true;
 

modules/hosts/nixos/nixnuc/grafana-files/alert-rules.yaml

@@ -0,0 +1,298 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: infrastructure_alerts
+      folder: Monitoring
+      interval: 1m
+      rules:
+        - uid: node_exporter_down
+          title: Node Exporter Down
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: VictoriaMetrics
+              model:
+                expr: up{job="node"}
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: A
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 1
+                        type: lt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      type: query
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: ""
+          panelId: 0
+          noDataState: NoData
+          execErrState: Error
+          for: 2m
+          annotations:
+            summary: Node exporter has been down for more than 2 minutes
+          labels:
+            severity: critical
+          isPaused: false
+        - uid: high_cpu_usage
+          title: High CPU Usage
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: VictoriaMetrics
+              model:
+                expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: A
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 80
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      type: query
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: ""
+          panelId: 0
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations:
+            summary: CPU usage is above 80% for more than 5 minutes
+          labels:
+            severity: warning
+          isPaused: false
+        - uid: high_memory_usage
+          title: High Memory Usage
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: VictoriaMetrics
+              model:
+                expr: 100 * (1 - ((node_memory_MemAvailable_bytes) / (node_memory_MemTotal_bytes)))
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: A
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 85
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      type: query
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: ""
+          panelId: 0
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations:
+            summary: Memory usage is above 85% for more than 5 minutes
+          labels:
+            severity: warning
+          isPaused: false
+        - uid: low_disk_space
+          title: Low Disk Space
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: VictoriaMetrics
+              model:
+                expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|fuse.lxcfs"} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|fuse.lxcfs"})
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: A
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 85
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      type: query
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: ""
+          panelId: 0
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations:
+            summary: Disk usage on root filesystem is above 85%
+          labels:
+            severity: warning
+          isPaused: false
+        - uid: cadvisor_down
+          title: cAdvisor Down
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: VictoriaMetrics
+              model:
+                expr: up{job="cadvisor"}
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: A
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: "-100"
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 1
+                        type: lt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      type: query
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: ""
+          panelId: 0
+          noDataState: NoData
+          execErrState: Error
+          for: 2m
+          annotations:
+            summary: cAdvisor (container metrics) has been down for more than 2 minutes
+          labels:
+            severity: warning
+          isPaused: false
+

modules/hosts/nixos/nixnuc/hardware-configuration.nix

@@ -28,11 +28,6 @@
       fsType = "zfs";
     };
 
-  fileSystems."/var/lib/prometheus2" =
-    { device = "orico/prometheus";
-      fsType = "zfs";
-    };
-
   swapDevices = [ ];
 
   # Enables DHCP on each ethernet and wireless interface. In case of scripted networking

modules/hosts/nixos/nixnuc/monitoring-stack.nix

@@ -0,0 +1,381 @@
+{ config, pkgs, ... }: let
+  home_domain = "home.technicalissues.us";
+in {
+  environment.systemPackages = with pkgs; [
+    # Keeping empty for manual testing if needed
+  ];
+
+  services = {
+    # ----------------------------
+    # PostgreSQL database
+    # ----------------------------
+    postgresql = {
+      enable = true;
+      ensureDatabases = [ "grafana" ];
+      ensureUsers = [
+        {
+          name = "grafana";
+          ensureDBOwnership = true;
+        }
+      ];
+    };
+
+    # ----------------------------
+    # VictoriaMetrics storage
+    # ----------------------------
+    victoriametrics = {
+      enable = true;
+      stateDir = "victoriametrics";  # Just the directory name, module adds /var/lib/ prefix
+      package = pkgs.victoriametrics;
+    };
+
+    # ----------------------------
+    # vmagent: scrape exporters
+    # ----------------------------
+    vmagent = {
+      enable = true;
+      package = pkgs.victoriametrics;
+
+      # Prometheus-style scrape configuration
+      prometheusConfig = {
+        global.scrape_interval = "15s";
+
+        scrape_configs = [
+          # Node exporter: CPU, memory, disk, diskio, network, system, ZFS
+          {
+            job_name = "node";
+            static_configs = [
+              {
+                targets = [
+                  "127.0.0.1:9100"      # nixnuc
+                  "192.168.22.22:9100"  # home assistant
+                  "umbrel:9100"
+                ];
+              }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__" "nodename"];
+                regex = "node_uname_info;0d869efa-prometheus-node-exporter";
+                target_label = "nodename";
+                replacement = "homeassistant";
+              }
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                regex = "127.0.0.1.*";
+                replacement = "${config.networking.hostName}";
+              }
+              {
+                target_label = "instance";
+                regex = "192.168.22.22.*";
+                replacement = "homeassistant";
+              }
+            ];
+          }
+
+          # cAdvisor: Docker containers
+          {
+            job_name = "cadvisor";
+            static_configs = [
+              { targets = ["127.0.0.1:8081"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+
+          # Nginx exporter
+          {
+            job_name = "nginx";
+            static_configs = [
+              { targets = ["127.0.0.1:9113"]; }
+            ];
+            metric_relabel_configs = [
+              {
+                source_labels = ["__name__"];
+                regex = "go_.*";
+                action = "drop";
+              }
+            ];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "${config.networking.hostName}";
+              }
+            ];
+          }
+
+          # Home Assistant metrics
+          {
+            job_name = "homeassistant"; # built in endpoint
+            scrape_interval = "30s";
+            metrics_path = "/api/prometheus";
+            static_configs = [
+              { targets = ["192.168.22.22:8123"]; }
+            ];
+            bearer_token_file = config.sops.secrets.home_assistant_token.path;
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = "homeassistant";
+              }
+            ];
+          }
+
+          # Uptime Kuma metrics
+          {
+            job_name = "uptimekuma";
+            scheme = "https";
+            scrape_interval = "30s";
+            static_configs = [
+              { targets = ["utk.technicalissues.us"]; }
+            ];
+            basic_auth = {
+              password_file = config.sops.secrets.uptimekuma_grafana_api_key.path;
+              username = "unused";
+            };
+            metric_relabel_configs = [
+              {
+                source_labels = ["monitor_hostname"];
+                regex = "^null$";
+                replacement = "";
+                target_label = "monitor_hostname";
+              }
+              {
+                source_labels = ["monitor_port"];
+                regex = "^null$";
+                replacement = "";
+                target_label = "monitor_port";
+              }
+              {
+                source_labels = ["monitor_url"];
+                regex = "https:\/\/";
+                replacement = "";
+                target_label = "monitor_url";
+              }
+            ];
+          }
+        ];
+      };
+
+      # Remote write to VictoriaMetrics
+      remoteWrite.url = "http://127.0.0.1:8428/api/v1/write";
+
+      extraArgs = [
+        # Pass other remote write flags the module does not expose natively:
+        "-remoteWrite.flushInterval=10s"
+        "-remoteWrite.maxDiskUsagePerURL=1GB"
+
+        # Prevent vmagent from failing the entire scrape if a target is down:
+        "-promscrape.suppressScrapeErrors"
+
+        # Enable some debugging info suggested by the interface on port 8429
+        "-promscrape.dropOriginalLabels=false"
+      ];
+    };
+
+    # ----------------------------
+    # Grafana with VictoriaMetrics datasource
+    # ----------------------------
+    grafana = {
+      enable = true;
+
+      # Install VictoriaMetrics plugin declaratively
+      declarativePlugins = [
+        pkgs.grafanaPlugins.victoriametrics-metrics-datasource
+      ];
+
+      provision = {
+        # Alert rules provisioning
+        # To add more rules: create them in Grafana UI, then export via:
+        # Alerting -> Alert rules -> Export rules (YAML format)
+        # Copy the exported rules into ./alert-rules.nix
+        alerting.rules.path = ./grafana-files/alert-rules.yaml;
+
+        datasources.settings.datasources = [
+          {
+            name   = "VictoriaMetrics";
+            type   = "victoriametrics-metrics-datasource";
+            access = "proxy";
+            url    = "http://127.0.0.1:8428";
+            isDefault = true;
+            uid = "VictoriaMetrics";  # Set explicit UID for use in alert rules
+          }
+        ];
+      };
+
+
+      settings = {
+        auth = {
+          # Set to true to disable (hide) the login form, useful if you use OAuth
+          disable_login_form = false;
+        };
+
+        "auth.generic_oauth" = {
+          name                       = "Pocket ID";
+          enabled                    = true;
+
+          # Use Grafana's file reference syntax for secrets
+          client_id                  = "$__file{${config.sops.secrets.grafana_oauth_client_id.path}}";
+          client_secret              = "$__file{${config.sops.secrets.grafana_oauth_client_secret.path}}";
+
+          auth_style                 = "AutoDetect";
+          scopes                     = "openid email profile groups";
+          auth_url                   = "${config.services.pocket-id.settings.APP_URL}/authorize";
+          token_url                  = "${config.services.pocket-id.settings.APP_URL}/api/oidc/token";
+          allow_sign_up              = true;
+          auto_login                 = true;
+          name_attribute_path        = "display_name";
+          login_attribute_path       = "preferred_username";
+          email_attribute_name       = "email:primary";
+          email_attribute_path       = "email";
+          role_attribute_path        = "contains(groups[*], 'grafana_super_admin') && 'GrafanaAdmin' || contains(groups[*], 'grafana_admin') && 'Admin' || contains(groups[*], 'grafana_editor') && 'Editor' || 'Viewer'";
+          role_attribute_strict      = false;
+          allow_assign_grafana_admin = true;
+          skip_org_role_sync         = false;
+          use_pkce                   = true;
+          use_refresh_token          = false;
+          tls_skip_verify_insecure   = false;
+        };
+
+        # Database configuration - use PostgreSQL with peer authentication
+        database = {
+          type = "postgres";
+          host = "/run/postgresql";  # Use Unix socket instead of TCP
+          name = "grafana";
+          user = "grafana";
+          # No password needed - using peer authentication via Unix socket
+        };
+
+        # Server configuration
+        server = {
+          domain             = "monitoring.${home_domain}";
+          http_addr          = "0.0.0.0";
+          http_port          = 3002;
+          root_url           = "https://monitoring.${home_domain}/grafana/";
+          serve_from_sub_path = true;
+        };
+
+        # Enable unified alerting (Grafana's built-in alerting)
+        "unified_alerting" = {
+          enabled = true;
+        };
+
+        # Disable legacy alerting
+        alerting.enabled = false;
+      };
+    };
+
+    # ----------------------------
+    # Exporters (using built-in NixOS modules)
+    # ----------------------------
+    
+    # Node exporter - using the built-in module
+    prometheus.exporters.node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9100;
+      enabledCollectors = [
+        "zfs"
+        "systemd"
+      ];
+      extraFlags = [
+        "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)"
+        "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$"
+      ];
+    };
+
+    # Nginx exporter - using the built-in module
+    prometheus.exporters.nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9113;
+      scrapeUri = "https://127.0.0.1/server_status";
+      sslVerify = false;
+    };
+
+    # cAdvisor for Docker containers
+    cadvisor = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 8081;
+      extraOptions = [
+        "--docker_only=true"
+        "--housekeeping_interval=30s"
+        "--disable_metrics=hugetlb"
+      ];
+    };
+  };
+
+  # ----------------------------
+  # Users and groups for service accounts
+  # ----------------------------
+  users.users.vmagent = {
+    isSystemUser = true;
+    group = "vmagent";
+  };
+
+  users.groups.vmagent = {};
+
+  # ----------------------------
+  # Systemd service dependencies
+  # ----------------------------
+  systemd.services.grafana = {
+    after = [ "postgresql.service" ];
+    requires = [ "postgresql.service" ];
+  };
+
+  # ----------------------------
+  # SOPS secrets configuration
+  # ----------------------------
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets = {
+      grafana_oauth_client_id = {
+        owner = "grafana";
+        restartUnits = ["grafana.service"];
+      };
+      grafana_oauth_client_secret = {
+        owner = "grafana";
+        restartUnits = ["grafana.service"];
+      };
+      home_assistant_token = {
+        owner = "vmagent";
+        restartUnits = ["vmagent.service"];
+      };
+      uptimekuma_grafana_api_key = {
+        owner = "vmagent";
+        restartUnits = ["vmagent.service"];
+        sopsFile = ../../common/secrets.yaml;
+      };
+    };
+  };
+
+  # -----------------------------
+  # Backups of all this
+  # -----------------------------
+  services.restic.backups.daily = {
+    paths = [
+      config.services.grafana.dataDir
+      config.services.victoriametrics.stateDir
+    ];
+  };
+}
+

modules/hosts/nixos/nixnuc/secrets.yaml

@@ -5,11 +5,14 @@ firefly_app_key: ENC[AES256_GCM,data:sNaqRgFOSmdSS0lCmEG8Nxy/3N7F/hQyS6iPnwau3sQ
 firefly_pat_data_import: ENC[AES256_GCM,data:mZ32gVaDZGEnpsu4HBHeIZ8UtkTQMrHNWp6iv1jOxsUNJlswWS0kU3IN/sVRWT734KcBGUnriWcXIFvZTx6KXhOjWdEYXyO+g9ws88CWCAguCssiPJI23XaDwMhaqjWRvauzPgjzGXdvg4pFs+myxui+MkLyqcv/ZOBoMt85Nn/M6xuMyyzzmalvTtovCgkja3LJDgxCK7AZoGkXK3X9v15ShHaAjM6mEYkSpONeprew+a7PfmPkM8ynMMOgdCQbRLf/34tLXnTer9qL/kz/Xt14PZsQ5+Fj+fxVwt08xv4FWi3VqUR9wL4Fs4G/yJQHUkDQXDcCk67+TgHxFnsYcxZw6k0ue8u1Ab1aDPyEhUD/UwyzTKyIjzCCIc8NBnst6o8kV62d6ei7dG4PWvNk8m2yNhZVlvoKEjmvZg0Bxv7xGniSimXpvTcMs18+mB0nOcYFUZJf6S0tolcClEDBEegwf0kD+p2F7jXsUjcJsaxqh9Xk215qbTi9iRVtQMWfPO8dCo4EC9vs7MFNmtJsGEp0WR2nwGFh7eIRbzug5+HC5OtUEGbmIUhLmTYnrohkJUKauMYALeYUS7j8GwFdfN0qT0N5r1d5OC/uOih2W5jWiMsOkNWKlaxZ/D1TOTYjLdcHCmIDlQHv9XVFUketsy6KPyg3XkmicKYq18xMi2pkWwGwDV0rd0aqHqRLTNfj3BoXI53eWSsZo8CKuUgWyXM4mIRy7yle2s52a2Idsc8/Zx42MmbWov3c5CNaQDjaseTSO8bv57U30GBY43UBJ4n+p6AIBnXQAN9d/YQ2Wvphe80ZdqyfHzTgj+6PEQKFXzs4KJDE83es/1+HVZdKehpUksenPeQ8UAxoIFtgFjdz6p9JBdXcuFhkWs6oAT96VQtkS6aHgOOUSTN2+H0Q/l6yvIvQB/QcI88r4BiU2vroGqzuVRCbgaXZqjUU06lS608gJOGqIe2a2bHbxwKU+SHEglTdDzRP4n02v8Ua5+BpcWVmhamEzd+8qhLEp/awYi9rd2Kc38T/vowsumLQAvfuZHUt8ySS1JEsyjrWsXQ/cfAiKMeTWkL7b2vEmvx30D8Acz68wDmXwxI/pWDKSe+jagegkmPWVnGQz7MATfApjIl4ZksDQH/O88WrYVF71YYfhA2Qpxr+2FHJ0F2kTPV6tv3QPcteJH4s8pVXvgpYZ050FL9/gJ+gB4rt23Lp0pGW+WEo6YenZ0plbWe4ixf8+U4he5eecGAlAYFEVYZnl6WKLLhX9/xMG9vNJODnUxF97csMPDYiGBhupT8dVk6+1ww=,iv:L0Ff7RYYOPqPeR81LJuTMZ5dsmeQrJtfO1e7Aei+tc4=,tag:wK5s7gRQNpk2aOnsIhtr2A==,type:str]
 firefly_simplefin_token: ENC[AES256_GCM,data:xJGII+ChIinSpJt+DNU8k8xtgxZkQPmr+02MpO39SylPI6qLHUiXib8xHGrxSSoux19dVdUVXsGO3jRsdIfrX4R6vt/zrIi7yK2gSTZaaoYI1nRg125aino1tIu1Xc+btbro26gDUL8ExlGOg+xvCdPrsmEPbuxVEKm1XxSOOjfvHFjhaMr8XV8/ewqcf5fYlUXEOtNyb6TCrNnp1eSp8+/Uii9GU5O8dlCt1Rof2wNeGQD4lV+lRZqOD0QX2aL5fyXxNUm5W8qFXLiscS8XCl0x2MM9X/mSQ6PC9MlY4WOHXw9/1S3jiVpUyM0U7wJO,iv:+/EwhdcfEwDm+1nhMuIFMiRvT/wbH+vIsmiOLP8bX8g=,tag:RjBomarf49a3xT2TPSTV+A==,type:str]
 firefly_vanity_url: ENC[AES256_GCM,data:ypIv2VA/B8RStW5Ifj5J20n/MI+jevd0vYrJPchERpt+txPJ0fo=,iv:16/hQyr/TJgLlNjJSUirEpth7jSHP/ooXQ/Vr97wBiA=,tag:p4WiNKm1zkizQCOFVKcW0Q==,type:str]
+grafana_oauth_client_id: ENC[AES256_GCM,data:AyqM4bV/9yiWyi0/wqu/SUt5cXYgpsvDcsOgCWENcLeFgGbV,iv:jbUSZQwGOc9V0RON9nAvI86HjGCDuh4wrnpgWcWNG0I=,tag:V6Xe1/r0srITkPDQZ2G+wA==,type:str]
+grafana_oauth_client_secret: ENC[AES256_GCM,data:As+m4hLWXy2uMq3KHWPtKzsRY4faYjGUrCdCSzP+TuI=,iv:3UAl8vuf7gUc5voyA2yuY8iiBaoI0iF3U7AFv0siskE=,tag:oD3B5SCNkIfqn88gBxy3Fw==,type:str]
 home_assistant_token: ENC[AES256_GCM,data:fNpoH60rXAsoVx/NBoDobDw/e6IoeoZfef1uDR7HbmnHNI101b+kQKkB8wBXEDLf7MlqlXVc1ExlgYFUo7+h2msx4WZUCGzuHtp1cMP0O9s2PZoQU8KQM2Frd69vOUcOT5y5ShJZPRrf6H2UKrm//3jE9zDOxxy3cQj6Q+jQLXX4AfZ12JAKzSiee9URWdU8eCHbnquUl1RNHF7zZQ8Mr9m41sBOrpBXt7ErsrhxxRhwlk7qprLt,iv:1j/QmOkLYd7nA+wXS49drBHU09HzMP3XxPbPdaErV6E=,tag:ftyOeNaN5PwNebeMPNAmTg==,type:str]
 immich_kiosk_basic_auth: ENC[AES256_GCM,data:R/vaYXe0SRwSUg8zC367vA/OYhjIcgfbHxR3OhLt1YYUhPNXVuUGmn199zLD,iv:GBEjJKvbwHAS75negE2whTlI4wS8K2nQYNm3015DFoY=,tag:zs+bJEr9JlIrJtikHycN4g==,type:str]
 mealie: ENC[AES256_GCM,data:fZFBWlh/nbxK0GA6+fb1FK6aFfbiV/GsBYKYRPgavcsB9h4HwRSZN94gPQ3AfGnk1q08ORPBYShNUIaVdsf4+t3taDC7bNTwPqFGRxoeKOU15BPj6Fpw7SHlplApUg7RDmuTWZuXDH0F49N+09GU/Lc26ewHMAzL8CtGg/H7s3CQfFEqCYYcXJeUcnNDu6PRnKFIAppflSScREp29CUf/IfTYQXidQC25upZgcvGJ+dCTDyNFftlhQiFQh8nFRFVJfKk4iBqnig3fOYsTD32ohAotfDGsSUWb9yjoYQNnNn0V14/,iv:WRFiBtavTP3iP5YSaCMkd7ag8ccx1BO76Vlw6axPkJU=,tag:4BhAxbF8JUB0MV12yx86uQ==,type:str]
 mtnmesh_bot_dot_env: ENC[AES256_GCM,data:jKz0voG/a7Eq+zHI2fsejTpu5H+ic5ZE8VmtOQDQEOTvYT8GWr2oCGwh68ql7g+nU5hlNN7uLl9LreW59nTgmvWNCVjVNKchxJeydktMUoq2FlgyoC2I354bi9gFEL0X5JYRR7jG9iwy4WGyEFfhhUJn1MSwOurRZPw/tdWTUKQVwN5oqjMxkXrZhFLDzq6BrTs0hBpzRYTe,iv:ZBxHkW8VyU+8v4GiDJMEaeqm3Fdtuwm7M/7YXkfkMnw=,tag:S8hTOSicQNIAj/Gdxzw5+A==,type:str]
 nextcloud_admin_pass: ENC[AES256_GCM,data:KztB3Tkqlt73PEO41lthGYElrbwVdfqQgT6f,iv:kRwXqGJO4AUOMq+uYzndGhscaJiyvG4ANKabHHd78YM=,tag:dP3PgKafDTv8x7huKJGDqA==,type:str]
+nginx_basic_auth: ENC[AES256_GCM,data:9qy8ccV9yWOrQtagf4D4fTp4v8GuY5ORxsE8hUv/vKngOhlG1ikvuIJacIfTfZj9JCoVWnQ=,iv:UGh/KGqT93t91rXu0JN1Hi4/Hb0ICnGiKkymm/5VLU0=,tag:8X6+bKk9zqggRXQhXrhJAg==,type:str]
 psitransfer_dot_env: ENC[AES256_GCM,data:bhvU0AOCjecZ62BtLw4H1DdkLeatI+uUl6L7UkdDRkBF3sayO45Z1eR4q60tflXucyTGhT8WgKFz53I+C2dn265wzojIRc3Xr4TBLyWpfJ7/dct40SckgUiRvOnrefiriWQ=,iv:DGMhDkzgeupzzTJnCdVWDPUSo2wxI3MAypKQwVfHExE=,tag:KbteGqrkqgj2XB1lvlk/yQ==,type:str]
 pinchflat_dot_env: ENC[AES256_GCM,data:8DLiFXThG5PGJ0ymW5bMVy5A8dM=,iv:BGkVvxaNwFIMSaA3F6h4ZsgkC9tm1lohA0lg2pgZhpw=,tag:qQNrluP5exdKG3NXgQHM8g==,type:str]
 tandoor_db_pass: ENC[AES256_GCM,data:X0unx5jquLsUXadbF6xLjjeGY+f8Ec4kdc15JQ==,iv:XptlJHfAkF+3jbgJTqxhVReYjuVVdk3NzfPepP78DRI=,tag:3RG5P9QGCJ/fjdxWpY1xWA==,type:str]
@@ -25,7 +28,7 @@ sops:
             bHZlNTZDV2NYU1hQQy9mem80SFF6TFkKfmjkJBfTdh0vTtGaVx1t3tHJvSsAwdYD
             PF025X9U+yG2oIopwXEVBkxcD70eyuJn3OqH0xoVLBkbhNM9i8LHrA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-11-10T23:26:34Z"
+    lastmodified: "2026-02-01T03:12:35Z"
-    mac: ENC[AES256_GCM,data:pwgPKBBp2oNu2RRZDprbpVmWxCbsReS3l5/alUfJ7KrMWU6tWiKXNbE2HY/40HYAGyJQ4Jal+u4tKEnPReVubEIe8NwuMlXaDh4ynEN1DId9FaRuCjKatkySWPOGO6DFd4Ajr1Xt1XtWVRqhlCAf5nE+nG1ZBQbloWgOEQZ9m6s=,iv:D0l5yu4lFwrboHbTN26ECPBM6MluPHGR2x6JAwbAUoE=,tag:ctlUVc2CFglsYuTYyCuuYQ==,type:str]
+    mac: ENC[AES256_GCM,data:2PCSk5RQfgsDkQwlujmrBw4yDOIypKBeW/MAF339OR2o77Dz4+YHbUjxoPHt84bpZDMNeUDAifQUoBrKqq66gBJU7CcF/A/dRGCw5xxkdGGEqIjOX+SpC4I+j0zfJ34Pc1BvmTtY32Ivb9njqKZtTj21KJGMB/NDdkgYrDkqY+g=,iv:TSh4Xlmu840HVPBRw+2D2NoDURkEusjwhUEVoL0YWvs=,tag:4K6sHya1LEOziB4zBo0QIg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.11.0