diff --git a/modules/hosts/common/secrets.yaml b/modules/hosts/common/secrets.yaml index 98375e1..05dfce3 100644 --- a/modules/hosts/common/secrets.yaml +++ b/modules/hosts/common/secrets.yaml @@ -5,7 +5,8 @@ hetzner_lego_env: ENC[AES256_GCM,data:xRADnkMC/mTq8/oRpZ+NYTStB9qX2N6V0GNIpGsXNe restic_env: ENC[AES256_GCM,data:FCYR8tkClRwfcjUotcr28D6uRz7sNihn50nw38CaYnqOD/U9+5kU0iAPSvqAbeuw+xUoKKKAPAfMHI12dPTYt17Wz1N7i4a+MRkiIR9pjyv5KZTK59G+,iv:jStc8GMbZUQUgooZiRdImSZskdckYN1cRm2gsKbUyYY=,tag:HpQQIj1j7fjCmxkSeY/k4g==,type:str] restic_repo: ENC[AES256_GCM,data:kCoNYVKwB87W4h5doa3IXj4n,iv:jKEw/Hki/tp3RSTsRB4dlg593I5B4pCLBav84ADCh70=,tag:+GFF5vHOVw0r/G8BbhcCjw==,type:str] restic_password: ENC[AES256_GCM,data:PfQsxJul1Qpt3WQoUEI941l+yng3lVjhDd8=,iv:U5KjhcVqyksN2ay19RBjNhYIB31tUbfNRIqCEx/+Wbc=,tag:jsoU+B1mjAprPK+M5I0pAQ==,type:str] -uptimekuma_grafana_api_key: ENC[AES256_GCM,data:irDlzCrDO5n/pMRHagueBTrXmm3herjOSioAz9E5O4a9dJD4ALhp5LPpmTs=,iv:oJhm8eZWSEvqF3Whv/UrS6sgIRdrZ0eqWF98vdwD0V8=,tag:HvOBFo4CJ9NLSz9FwMCP2w==,type:str] +vmagent_push_pw: ENC[AES256_GCM,data:Ey82+FQWOdTn8iTcCbn73A2pYIh8dfD6Dw==,iv:0xEtakEOzZd2wbg407aA0rLeZqPo5NpqWdTSD3VQ+yw=,tag:wINlAjbCYnw9L4odxKJFAg==,type:str] +uptimekuma_grafana_api_key: ENC[AES256_GCM,data:cXdbdiEa4dqigcojFgE8Wf4esjbF7wmrx5BuEQltfwM5fNJQG6LP/8ZcopI=,iv:dUftPkEyy1DMq5fJTQiJlDab2UMAbwJt+81p35eBrAU=,tag:Nh+X6lTKB5EboADaSVK00g==,type:str] wifi_creds: ENC[AES256_GCM,data:9lgTtI8YHyCHrvqss4W7coLnqfOAoQzrCQne6dLv0x66pt7jLo4Y6YSd3TklRTurS9usvNk3sg==,iv:6g86hOmpnOxf4p4C+wPit7EP0DD+xb+cINiWRJnTRDM=,tag:ZW336IhXtrf5l5n/RJecoQ==,type:str] sops: age: @@ -108,7 +109,7 @@ sops: ODFjcWxtRjkweGJvdzdWSEphMHRCdm8Kx0amHgaZZR26c+VRVTyBEnm+w5c5nA7R txHj1U349LbfEsovTqZAL1o2WuX+gmXSj1aeXPKW+S0bIagC6dDacA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-01-25T02:27:36Z" - mac: ENC[AES256_GCM,data:83QyyqQqy9dkW1PIjo6fFDV1oLv3GCrfV9xiq5pZJwW9uhvNTi1LvR4bX55foK7lPBRmtW9xCRIGWCm1nORTJH1ae3cdRuZ+moUictpn0AuqJY/E/4+Nlr7TMdJUj3NI9bfGkJ+BoizGvPg+43ubobnl/+PH7/3Tu0omXkhUB9A=,iv:tcBMolA2hcVKfUhPnK6P9H1Xl1n5jMH+Qo0vEe5CgkE=,tag:L9gpXtvYIYN/gRl7rBPqtA==,type:str] + lastmodified: "2026-02-01T03:15:26Z" + mac: ENC[AES256_GCM,data:gGk69AnZNAlA5fxViXZQMpGStn8v2L/IyIUAshyrlAKu8dcTDQfCDfC2kTlo0Q/gwhqchSKYFFKWWze7EUwUWmlYjJu5MEEK4aQ0HEObff27AKQloOln1X3jiIR7bnsKoakbAKJWYpGf6PWClbH64mjoJhBroG5amAwE25bevDE=,iv:bO2kFuGcq/1j2uo2y5/UwOHNQ5A8orStI190NMdAr1E=,tag:7xwClQ28I695qDICc0GS+A==,type:str] unencrypted_suffix: _unencrypted version: 3.11.0 diff --git a/modules/hosts/nixos/hetznix01/post-install/default.nix b/modules/hosts/nixos/hetznix01/post-install/default.nix index 8741d3f..1e49b0d 100644 --- a/modules/hosts/nixos/hetznix01/post-install/default.nix +++ b/modules/hosts/nixos/hetznix01/post-install/default.nix @@ -7,6 +7,7 @@ in { ../../../common/linux/restic.nix ./containers/emqx.nix ./matrix-synapse.nix + ./monitoring.nix #./mosquitto.nix ./nginx.nix ]; diff --git a/modules/hosts/nixos/hetznix01/post-install/monitoring.nix b/modules/hosts/nixos/hetznix01/post-install/monitoring.nix new file mode 100644 index 0000000..c9af29a --- /dev/null +++ b/modules/hosts/nixos/hetznix01/post-install/monitoring.nix @@ -0,0 +1,129 @@ +{ config, pkgs, ... }: let + metrics_server = "https://monitoring.home.technicalissues.us/remotewrite"; +in { + services = { + vmagent = { + enable = true; + package = pkgs.victoriametrics; + + # Prometheus-style scrape configuration + prometheusConfig = { + global.scrape_interval = "15s"; + + scrape_configs = [ + { + job_name = "node"; + static_configs = [ + { targets = ["127.0.0.1:9100"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + regex = "127.0.0.1.*"; + replacement = "${config.networking.hostName}"; + } + ]; + } + + # Nginx exporter + { + job_name = "nginx"; + static_configs = [ + { targets = ["127.0.0.1:9113"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + replacement = "${config.networking.hostName}"; + } + ]; + } + ]; + }; + + # Remote write to VictoriaMetrics + remoteWrite = { + basicAuthUsername = "metricsshipper"; + basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path; + url = metrics_server; + }; + + extraArgs = [ + # Pass other remote write flags the module does not expose natively: + "-remoteWrite.flushInterval=10s" + "-remoteWrite.maxDiskUsagePerURL=1GB" + + # Prevent vmagent from failing the entire scrape if a target is down: + "-promscrape.suppressScrapeErrors" + + # Enable some debugging info suggested by the interface on port 8429 + "-promscrape.dropOriginalLabels=false" + ]; + }; + + # ---------------------------- + # Exporters (using built-in NixOS modules) + # ---------------------------- + + # Node exporter - using the built-in module + prometheus.exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9100; + enabledCollectors = [ + "systemd" + ]; + extraFlags = [ + "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)" + "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$" + ]; + }; + + # Nginx exporter - using the built-in module + prometheus.exporters.nginx = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9113; + scrapeUri = "https://127.0.0.1/server_status"; + sslVerify = false; + }; + }; + + # ---------------------------- + # Users and groups for service accounts + # ---------------------------- + users.users.vmagent = { + isSystemUser = true; + group = "vmagent"; + }; + + users.groups.vmagent = {}; + + # ---------------------------- + # SOPS secrets configuration + # ---------------------------- + sops = { + secrets = { + vmagent_push_pw = { + owner = "vmagent"; + restartUnits = ["vmagent.service"]; + sopsFile = ../../../common/secrets.yaml; + }; + }; + }; +} + diff --git a/modules/hosts/nixos/hetznix02/post-install/default.nix b/modules/hosts/nixos/hetznix02/post-install/default.nix index 4545616..2500481 100644 --- a/modules/hosts/nixos/hetznix02/post-install/default.nix +++ b/modules/hosts/nixos/hetznix02/post-install/default.nix @@ -1,6 +1,7 @@ { config, username, ... }: { imports = [ ../../../common/linux/lets-encrypt.nix + ./monitoring.nix ./nginx.nix ]; diff --git a/modules/hosts/nixos/hetznix02/post-install/monitoring.nix b/modules/hosts/nixos/hetznix02/post-install/monitoring.nix new file mode 100644 index 0000000..c9af29a --- /dev/null +++ b/modules/hosts/nixos/hetznix02/post-install/monitoring.nix @@ -0,0 +1,129 @@ +{ config, pkgs, ... }: let + metrics_server = "https://monitoring.home.technicalissues.us/remotewrite"; +in { + services = { + vmagent = { + enable = true; + package = pkgs.victoriametrics; + + # Prometheus-style scrape configuration + prometheusConfig = { + global.scrape_interval = "15s"; + + scrape_configs = [ + { + job_name = "node"; + static_configs = [ + { targets = ["127.0.0.1:9100"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + regex = "127.0.0.1.*"; + replacement = "${config.networking.hostName}"; + } + ]; + } + + # Nginx exporter + { + job_name = "nginx"; + static_configs = [ + { targets = ["127.0.0.1:9113"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + replacement = "${config.networking.hostName}"; + } + ]; + } + ]; + }; + + # Remote write to VictoriaMetrics + remoteWrite = { + basicAuthUsername = "metricsshipper"; + basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path; + url = metrics_server; + }; + + extraArgs = [ + # Pass other remote write flags the module does not expose natively: + "-remoteWrite.flushInterval=10s" + "-remoteWrite.maxDiskUsagePerURL=1GB" + + # Prevent vmagent from failing the entire scrape if a target is down: + "-promscrape.suppressScrapeErrors" + + # Enable some debugging info suggested by the interface on port 8429 + "-promscrape.dropOriginalLabels=false" + ]; + }; + + # ---------------------------- + # Exporters (using built-in NixOS modules) + # ---------------------------- + + # Node exporter - using the built-in module + prometheus.exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9100; + enabledCollectors = [ + "systemd" + ]; + extraFlags = [ + "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)" + "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$" + ]; + }; + + # Nginx exporter - using the built-in module + prometheus.exporters.nginx = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9113; + scrapeUri = "https://127.0.0.1/server_status"; + sslVerify = false; + }; + }; + + # ---------------------------- + # Users and groups for service accounts + # ---------------------------- + users.users.vmagent = { + isSystemUser = true; + group = "vmagent"; + }; + + users.groups.vmagent = {}; + + # ---------------------------- + # SOPS secrets configuration + # ---------------------------- + sops = { + secrets = { + vmagent_push_pw = { + owner = "vmagent"; + restartUnits = ["vmagent.service"]; + sopsFile = ../../../common/secrets.yaml; + }; + }; + }; +} + diff --git a/modules/hosts/nixos/kiosk-entryway/default.nix b/modules/hosts/nixos/kiosk-entryway/default.nix index f3cb9cf..a28e1eb 100644 --- a/modules/hosts/nixos/kiosk-entryway/default.nix +++ b/modules/hosts/nixos/kiosk-entryway/default.nix @@ -2,6 +2,7 @@ imports = [ ./disk-config.nix ./hardware-configuration.nix + ./monitoring.nix ]; system.stateVersion = "24.11"; diff --git a/modules/hosts/nixos/kiosk-entryway/monitoring.nix b/modules/hosts/nixos/kiosk-entryway/monitoring.nix new file mode 100644 index 0000000..c9af29a --- /dev/null +++ b/modules/hosts/nixos/kiosk-entryway/monitoring.nix @@ -0,0 +1,129 @@ +{ config, pkgs, ... }: let + metrics_server = "https://monitoring.home.technicalissues.us/remotewrite"; +in { + services = { + vmagent = { + enable = true; + package = pkgs.victoriametrics; + + # Prometheus-style scrape configuration + prometheusConfig = { + global.scrape_interval = "15s"; + + scrape_configs = [ + { + job_name = "node"; + static_configs = [ + { targets = ["127.0.0.1:9100"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + regex = "127.0.0.1.*"; + replacement = "${config.networking.hostName}"; + } + ]; + } + + # Nginx exporter + { + job_name = "nginx"; + static_configs = [ + { targets = ["127.0.0.1:9113"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + replacement = "${config.networking.hostName}"; + } + ]; + } + ]; + }; + + # Remote write to VictoriaMetrics + remoteWrite = { + basicAuthUsername = "metricsshipper"; + basicAuthPasswordFile = config.sops.secrets.vmagent_push_pw.path; + url = metrics_server; + }; + + extraArgs = [ + # Pass other remote write flags the module does not expose natively: + "-remoteWrite.flushInterval=10s" + "-remoteWrite.maxDiskUsagePerURL=1GB" + + # Prevent vmagent from failing the entire scrape if a target is down: + "-promscrape.suppressScrapeErrors" + + # Enable some debugging info suggested by the interface on port 8429 + "-promscrape.dropOriginalLabels=false" + ]; + }; + + # ---------------------------- + # Exporters (using built-in NixOS modules) + # ---------------------------- + + # Node exporter - using the built-in module + prometheus.exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9100; + enabledCollectors = [ + "systemd" + ]; + extraFlags = [ + "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)" + "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$" + ]; + }; + + # Nginx exporter - using the built-in module + prometheus.exporters.nginx = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9113; + scrapeUri = "https://127.0.0.1/server_status"; + sslVerify = false; + }; + }; + + # ---------------------------- + # Users and groups for service accounts + # ---------------------------- + users.users.vmagent = { + isSystemUser = true; + group = "vmagent"; + }; + + users.groups.vmagent = {}; + + # ---------------------------- + # SOPS secrets configuration + # ---------------------------- + sops = { + secrets = { + vmagent_push_pw = { + owner = "vmagent"; + restartUnits = ["vmagent.service"]; + sopsFile = ../../../common/secrets.yaml; + }; + }; + }; +} + diff --git a/modules/hosts/nixos/nixnuc/containers/nginx-proxy.nix b/modules/hosts/nixos/nixnuc/containers/nginx-proxy.nix deleted file mode 100644 index 0a78727..0000000 --- a/modules/hosts/nixos/nixnuc/containers/nginx-proxy.nix +++ /dev/null @@ -1,181 +0,0 @@ -{ config, ... }: let - http_port = 80; - https_port = 443; - gandi_api = "${config.sops.secrets.gandi_api.path}"; - #gandi_dns_pat = "${config.sops.secrets.gandi_dns_pat.path}"; - home_domain = "home.technicalissues.us"; - backend_ip = "192.168.20.190"; - mini_watcher = "192.168.23.20"; -in { - sops.secrets.gandi_api = { - sopsFile = ../../../../system/common/secrets.yaml; - restartUnits = [ - "container@nginx-proxy.service" - ]; - }; - #sops.secrets.gandi_dns_pat = { - # sopsFile = ../../../../system/common/secrets.yaml; - # restartUnits = [ - # "container@nginx-proxy.service" - # ]; - #}; - - ## - ## Gandi (gandi.net) - ## - ## Single host update - # protocol=gandi - # zone=example.com - # password=my-gandi-access-token - # use-personal-access-token=yes - # ttl=10800 # optional - # myhost.example.com - services.ddclient = { - enable = true; - protocol = "gandi"; - zone = "technicalissues.us"; - domains = [ home_domain ]; - username = "unused"; - extraConfig = '' - usev4=webv4 - #usev6=webv6 - #use-personal-access-token=yes - ttl=300 - ''; - passwordFile = gandi_api; }; - - containers.nginx-proxy = { - bindMounts."${gandi_api}".isReadOnly = true; - #bindMounts."${gandi_dns_pat}".isReadOnly = true; - autoStart = true; - timeoutStartSec = "5min"; - privateNetwork = true; - hostBridge = "br1-23"; - localAddress = "192.168.23.21/24"; - config = { config, pkgs, lib, ... }: { - system.stateVersion = "23.11"; - - programs.traceroute.enable = true; - - services.nginx = { - enable = true; - recommendedGzipSettings = true; - recommendedOptimisation = true; - recommendedProxySettings = true; - recommendedTlsSettings = true; - appendHttpConfig = '' - # Add HSTS header with preloading to HTTPS requests. - # Adding this header to HTTP requests is discouraged - map $scheme $hsts_header { - https "max-age=31536000;"; - } - add_header Strict-Transport-Security $hsts_header; - ''; - - virtualHosts = { - "${home_domain}" = { - serverAliases = [ "nix-tester.${home_domain}" ]; - default = true; - listen = [ - { port = http_port; addr = "0.0.0.0"; } - { port = https_port; addr = "0.0.0.0"; ssl = true; } - ]; - enableACME = true; - acmeRoot = null; - addSSL = true; - forceSSL = false; - locations."/" = { - return = "200 '

Hello world ;)

'"; - extraConfig = '' - add_header Content-Type text/html; - ''; - }; - }; - "ab.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyWebsockets = true; - locations."/".proxyPass = "http://${backend_ip}:13378"; - }; - "atuin.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyPass = "http://${mini_watcher}:9999"; - }; - "nc.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - extraConfig = '' - client_max_body_size 0; - underscores_in_headers on; - ''; - locations."/".proxyWebsockets = true; - locations."/".proxyPass = "http://${mini_watcher}:8081"; - locations."/".extraConfig = '' - # these are added per https://www.nicemicro.com/tutorials/debian-snap-nextcloud.html - add_header Front-End-Https on; - proxy_headers_hash_max_size 512; - proxy_headers_hash_bucket_size 64; - proxy_buffering off; - proxy_max_temp_file_size 0; - ''; - }; - "onlyoffice.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyWebsockets = true; - locations."/".proxyPass = "http://${mini_watcher}:8888"; - }; - "readit.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyPass = "http://${backend_ip}:8090"; - }; - "tandoor.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyPass = "http://${backend_ip}:8080"; - }; - }; - }; - - security.acme = { - acceptTerms = true; - defaults = { - email = "lets-encrypt@technicalissues.us"; - credentialFiles = { "GANDIV5_API_KEY_FILE" = gandi_api; }; - #credentialFiles = { "GANDIV5_PERSONAL_ACCESS_TOKEN_FILE" = gandi_dns_pat; }; - dnsProvider = "gandiv5"; - dnsResolver = "ns1.gandi.net"; - # uncomment below for testing - #server = "https://acme-staging-v02.api.letsencrypt.org/directory"; - }; - }; - - networking = { - firewall = { - enable = true; - allowedTCPPorts = [ http_port https_port ]; - }; - defaultGateway = "192.168.23.1"; - # Use systemd-resolved inside the container - # Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686 - useHostResolvConf = lib.mkForce false; - }; - - services.resolved.enable = true; - }; - }; -} diff --git a/modules/hosts/nixos/nixnuc/default.nix b/modules/hosts/nixos/nixnuc/default.nix index 3921f53..be8ff03 100644 --- a/modules/hosts/nixos/nixnuc/default.nix +++ b/modules/hosts/nixos/nixnuc/default.nix @@ -3,7 +3,6 @@ https_port = 443; home_domain = "home.technicalissues.us"; backend_ip = "127.0.0.1"; - mini_watcher = "192.168.23.20"; restic_backup_time = "02:00"; in { imports = [ @@ -11,6 +10,7 @@ in { ./containers/audiobookshelf.nix ./containers/mountain-mesh-bot-discord.nix ./containers/psitransfer.nix + ./monitoring-stack.nix ../../common/linux/lets-encrypt.nix ../../common/linux/restic.nix ]; @@ -96,16 +96,11 @@ in { 8888 # Atuin 8090 # Wallabag in docker compose 8945 # Pinchflat - 9090 # Prometheus Server - 9273 # Telegraf's Prometheus endpoint 13378 # Audiobookshelf in oci-container - 22000 # Syncthing transfers ]; allowedUDPPorts = [ 1900 # Jellyfin service auto-discovery 7359 # Jellyfin auto-discovery - 21027 # Syncthing discovery - 22000 # Syncthing transfers ]; }; # Or disable the firewall altogether. @@ -135,7 +130,9 @@ in { }; services.pulseaudio.enable = false; - programs.mtr.enable = true; + programs = { + mtr.enable = true; + }; # List services that you want to enable: services = { @@ -219,22 +216,6 @@ in { stateDir = "/orico/forgejo"; }; fwupd.enable = true; - grafana = { - enable = true; - settings = { - auth = { - disable_login_form = true; - oauth_auto_login = true; - }; - server = { - domain = "monitoring.${home_domain}"; - http_addr = "0.0.0.0"; - http_port = 3002; - root_url = "https://monitoring.${home_domain}/grafana/"; # Not needed if it is `https://your.domain/` - serve_from_sub_path = true; - }; - }; - }; jellyfin = { enable = true; openFirewall = true; @@ -461,27 +442,14 @@ in { enableACME = true; acmeRoot = null; forceSSL = true; - locations."/grafana/".proxyPass = "http://${backend_ip}:3002/grafana/"; - }; - "nc.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - extraConfig = '' - client_max_body_size 0; - underscores_in_headers on; - ''; - locations."/".proxyWebsockets = true; - locations."/".proxyPass = "http://${mini_watcher}:8081"; - locations."/".extraConfig = '' - # these are added per https://www.nicemicro.com/tutorials/debian-snap-nextcloud.html - add_header Front-End-Https on; - proxy_headers_hash_max_size 512; - proxy_headers_hash_bucket_size 64; - proxy_buffering off; - proxy_max_temp_file_size 0; - ''; + locations = { + "/grafana/".proxyPass = "http://${backend_ip}:3002/grafana/"; + "/remotewrite" = { + basicAuthFile = config.sops.secrets.nginx_basic_auth.path; + proxyPass = "http://127.0.0.1:8428/api/v1/write"; + proxyWebsockets = true; + }; + }; }; "nextcloud.${home_domain}" = { enableACME = true; @@ -500,14 +468,6 @@ in { deny all; ''; }; - "onlyoffice.${home_domain}" = { - listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; - enableACME = true; - acmeRoot = null; - forceSSL = true; - locations."/".proxyWebsockets = true; - locations."/".proxyPass = "http://${mini_watcher}:8888"; - }; "readit.${home_domain}" = { listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; enableACME = true; @@ -520,6 +480,9 @@ in { nominatim = { enable = true; hostName = "nominatim.home.technicalissues.us"; + settings = { + NOMINATIM_PROJECT_DIR = "/var/lib/nominatim/project"; + }; ui.config = '' Nominatim_Config.Page_Title="Beantown's Nominatim"; Nominatim_Config.Nominatim_API_Endpoint='https://${config.services.nominatim.hostName}/'; @@ -542,52 +505,23 @@ in { postgresql = { enable = true; package = pkgs.postgresql_16; + ensureUsers = [ + { + # Required by Nominatim + name = "www-data"; + ensureDBOwnership = false; + } + ]; }; postgresqlBackup = { enable = true; backupAll = true; startAt = "*-*-* 23:00:00"; }; - prometheus = { - enable = true; - checkConfig = "syntax-only"; - globalConfig.scrape_interval = "10s"; # "1m" - scrapeConfigs = [ - { - job_name = "hass"; - scrape_interval = "30s"; - metrics_path = "/api/prometheus"; - static_configs = [{ - targets = [ "192.168.22.22:8123" ]; - }]; - bearer_token_file = config.sops.secrets.home_assistant_token.path; - } - { - job_name = "telegraf"; - scrape_interval = "5s"; - static_configs = [{ - targets = [ "localhost:9273" ]; - }]; - } - { - job_name = "uptimekuma"; - scheme = "https"; - scrape_interval = "30s"; - static_configs = [{ - targets = [ "utk.technicalissues.us" ]; - }]; - basic_auth = { - password_file = config.sops.secrets.uptimekuma_grafana_api_key.path; - username = ""; - }; - } - ]; - }; resolved.enable = true; restic.backups.daily = { paths = [ config.services.forgejo.stateDir - config.services.grafana.dataDir config.services.mealie.settings.DATA_DIR config.services.nextcloud.home "${config.users.users.${username}.home}/compose-files/wallabag" @@ -595,7 +529,6 @@ in { "/orico/jellyfin/data" "/orico/jellyfin/staging/downloaded-files" "/var/backup/postgresql" - "/var/lib/prometheus2" ]; timerConfig = { OnCalendar = restic_backup_time; @@ -615,83 +548,6 @@ in { ]; useRoutingFeatures = "both"; }; - telegraf = { - enable = true; - extraConfig = { - agent = { - interval = "15s"; - round_interval = true; - collection_jitter = "0s"; - debug = false; - }; - inputs = { - cpu = { - percpu = true; - totalcpu = true; - report_active = true; - collect_cpu_time = true; - }; - disk = { - ignore_fs = [ - "aufs" - "devfs" - "devtmpfs" - "iso9660" - "overlay" - "squashfs" - "tmpfs" - ]; - }; - diskio = { - skip_serial_number = true; - }; - docker = { - endpoint = "unix:///run/docker.sock"; - container_name_include = []; - storage_objects = [ - "container" - "volume" - ]; - perdevice_include = [ - "cpu" - "blkio" - "network" - ]; - total_include = [ - "cpu" - "blkio" - "network" - ]; - timeout = "5s"; - gather_services = false; - }; - mem = {}; - net = { - ignore_protocol_stats = true; - }; - nginx = { - insecure_skip_verify = true; - urls = [ "https://127.0.0.1/server_status" ]; - }; - #smart = {}; - system = {}; - #systemd_units = {}; - zfs = { - datasetMetrics = true; - poolMetrics = true; - }; - }; - outputs = { - prometheus_client = { - collectors_exclude = [ - "gocollector" - "process" - ]; - listen = ":9273"; - }; - }; - }; - }; zfs.autoScrub.enable = true; }; @@ -724,10 +580,6 @@ in { "phpfpm-firefly-iii-data-importer.service" ]; }; - home_assistant_token = { - owner = config.users.users.prometheus.name; - restartUnits = ["prometheus.service"]; - }; immich_kiosk_basic_auth = { owner = config.users.users.nginx.name; restartUnits = ["nginx.service"]; @@ -745,14 +597,13 @@ in { restartUnits = ["mealie.service"]; }; nextcloud_admin_pass.owner = config.users.users.nextcloud.name; + nginx_basic_auth = { + owner = "nginx"; + restartUnits = ["nginx.service"]; + }; tailscale_key = { restartUnits = [ "tailscaled-autoconnect.service" ]; }; - uptimekuma_grafana_api_key = { - owner = config.users.users.prometheus.name; - restartUnits = ["prometheus.service"]; - sopsFile = ../../common/secrets.yaml; - }; }; }; @@ -780,8 +631,6 @@ in { ]; }; - users.users.telegraf.extraGroups = [ "docker" ]; - # Enable common container config files in /etc/containers virtualisation.containers.enable = true; diff --git a/modules/hosts/nixos/nixnuc/grafana-files/alert-rules.yaml b/modules/hosts/nixos/nixnuc/grafana-files/alert-rules.yaml new file mode 100644 index 0000000..6008fe7 --- /dev/null +++ b/modules/hosts/nixos/nixnuc/grafana-files/alert-rules.yaml @@ -0,0 +1,298 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: infrastructure_alerts + folder: Monitoring + interval: 1m + rules: + - uid: node_exporter_down + title: Node Exporter Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: VictoriaMetrics + model: + expr: up{job="node"} + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + type: query + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: "" + panelId: 0 + noDataState: NoData + execErrState: Error + for: 2m + annotations: + summary: Node exporter has been down for more than 2 minutes + labels: + severity: critical + isPaused: false + - uid: high_cpu_usage + title: High CPU Usage + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: VictoriaMetrics + model: + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + conditions: + - evaluator: + params: + - 80 + type: gt + operator: + type: and + query: + params: + - C + type: query + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: "" + panelId: 0 + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: CPU usage is above 80% for more than 5 minutes + labels: + severity: warning + isPaused: false + - uid: high_memory_usage + title: High Memory Usage + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: VictoriaMetrics + model: + expr: 100 * (1 - ((node_memory_MemAvailable_bytes) / (node_memory_MemTotal_bytes))) + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + conditions: + - evaluator: + params: + - 85 + type: gt + operator: + type: and + query: + params: + - C + type: query + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: "" + panelId: 0 + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: Memory usage is above 85% for more than 5 minutes + labels: + severity: warning + isPaused: false + - uid: low_disk_space + title: Low Disk Space + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: VictoriaMetrics + model: + expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|fuse.lxcfs"} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|fuse.lxcfs"}) + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + conditions: + - evaluator: + params: + - 85 + type: gt + operator: + type: and + query: + params: + - C + type: query + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: "" + panelId: 0 + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: Disk usage on root filesystem is above 85% + labels: + severity: warning + isPaused: false + - uid: cadvisor_down + title: cAdvisor Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: VictoriaMetrics + model: + expr: up{job="cadvisor"} + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: "-100" + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + type: query + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: "" + panelId: 0 + noDataState: NoData + execErrState: Error + for: 2m + annotations: + summary: cAdvisor (container metrics) has been down for more than 2 minutes + labels: + severity: warning + isPaused: false + diff --git a/modules/hosts/nixos/nixnuc/hardware-configuration.nix b/modules/hosts/nixos/nixnuc/hardware-configuration.nix index 4af6582..17b0c38 100644 --- a/modules/hosts/nixos/nixnuc/hardware-configuration.nix +++ b/modules/hosts/nixos/nixnuc/hardware-configuration.nix @@ -28,11 +28,6 @@ fsType = "zfs"; }; - fileSystems."/var/lib/prometheus2" = - { device = "orico/prometheus"; - fsType = "zfs"; - }; - swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/modules/hosts/nixos/nixnuc/monitoring-stack.nix b/modules/hosts/nixos/nixnuc/monitoring-stack.nix new file mode 100644 index 0000000..e486512 --- /dev/null +++ b/modules/hosts/nixos/nixnuc/monitoring-stack.nix @@ -0,0 +1,381 @@ +{ config, pkgs, ... }: let + home_domain = "home.technicalissues.us"; +in { + environment.systemPackages = with pkgs; [ + # Keeping empty for manual testing if needed + ]; + + services = { + # ---------------------------- + # PostgreSQL database + # ---------------------------- + postgresql = { + enable = true; + ensureDatabases = [ "grafana" ]; + ensureUsers = [ + { + name = "grafana"; + ensureDBOwnership = true; + } + ]; + }; + + # ---------------------------- + # VictoriaMetrics storage + # ---------------------------- + victoriametrics = { + enable = true; + stateDir = "victoriametrics"; # Just the directory name, module adds /var/lib/ prefix + package = pkgs.victoriametrics; + }; + + # ---------------------------- + # vmagent: scrape exporters + # ---------------------------- + vmagent = { + enable = true; + package = pkgs.victoriametrics; + + # Prometheus-style scrape configuration + prometheusConfig = { + global.scrape_interval = "15s"; + + scrape_configs = [ + # Node exporter: CPU, memory, disk, diskio, network, system, ZFS + { + job_name = "node"; + static_configs = [ + { + targets = [ + "127.0.0.1:9100" # nixnuc + "192.168.22.22:9100" # home assistant + "umbrel:9100" + ]; + } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__" "nodename"]; + regex = "node_uname_info;0d869efa-prometheus-node-exporter"; + target_label = "nodename"; + replacement = "homeassistant"; + } + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + regex = "127.0.0.1.*"; + replacement = "${config.networking.hostName}"; + } + { + target_label = "instance"; + regex = "192.168.22.22.*"; + replacement = "homeassistant"; + } + ]; + } + + # cAdvisor: Docker containers + { + job_name = "cadvisor"; + static_configs = [ + { targets = ["127.0.0.1:8081"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + replacement = "${config.networking.hostName}"; + } + ]; + } + + # Nginx exporter + { + job_name = "nginx"; + static_configs = [ + { targets = ["127.0.0.1:9113"]; } + ]; + metric_relabel_configs = [ + { + source_labels = ["__name__"]; + regex = "go_.*"; + action = "drop"; + } + ]; + relabel_configs = [ + { + target_label = "instance"; + replacement = "${config.networking.hostName}"; + } + ]; + } + + # Home Assistant metrics + { + job_name = "homeassistant"; # built in endpoint + scrape_interval = "30s"; + metrics_path = "/api/prometheus"; + static_configs = [ + { targets = ["192.168.22.22:8123"]; } + ]; + bearer_token_file = config.sops.secrets.home_assistant_token.path; + relabel_configs = [ + { + target_label = "instance"; + replacement = "homeassistant"; + } + ]; + } + + # Uptime Kuma metrics + { + job_name = "uptimekuma"; + scheme = "https"; + scrape_interval = "30s"; + static_configs = [ + { targets = ["utk.technicalissues.us"]; } + ]; + basic_auth = { + password_file = config.sops.secrets.uptimekuma_grafana_api_key.path; + username = "unused"; + }; + metric_relabel_configs = [ + { + source_labels = ["monitor_hostname"]; + regex = "^null$"; + replacement = ""; + target_label = "monitor_hostname"; + } + { + source_labels = ["monitor_port"]; + regex = "^null$"; + replacement = ""; + target_label = "monitor_port"; + } + { + source_labels = ["monitor_url"]; + regex = "https:\/\/"; + replacement = ""; + target_label = "monitor_url"; + } + ]; + } + ]; + }; + + # Remote write to VictoriaMetrics + remoteWrite.url = "http://127.0.0.1:8428/api/v1/write"; + + extraArgs = [ + # Pass other remote write flags the module does not expose natively: + "-remoteWrite.flushInterval=10s" + "-remoteWrite.maxDiskUsagePerURL=1GB" + + # Prevent vmagent from failing the entire scrape if a target is down: + "-promscrape.suppressScrapeErrors" + + # Enable some debugging info suggested by the interface on port 8429 + "-promscrape.dropOriginalLabels=false" + ]; + }; + + # ---------------------------- + # Grafana with VictoriaMetrics datasource + # ---------------------------- + grafana = { + enable = true; + + # Install VictoriaMetrics plugin declaratively + declarativePlugins = [ + pkgs.grafanaPlugins.victoriametrics-metrics-datasource + ]; + + provision = { + # Alert rules provisioning + # To add more rules: create them in Grafana UI, then export via: + # Alerting -> Alert rules -> Export rules (YAML format) + # Copy the exported rules into ./alert-rules.nix + alerting.rules.path = ./grafana-files/alert-rules.yaml; + + datasources.settings.datasources = [ + { + name = "VictoriaMetrics"; + type = "victoriametrics-metrics-datasource"; + access = "proxy"; + url = "http://127.0.0.1:8428"; + isDefault = true; + uid = "VictoriaMetrics"; # Set explicit UID for use in alert rules + } + ]; + }; + + + settings = { + auth = { + # Set to true to disable (hide) the login form, useful if you use OAuth + disable_login_form = false; + }; + + "auth.generic_oauth" = { + name = "Pocket ID"; + enabled = true; + + # Use Grafana's file reference syntax for secrets + client_id = "$__file{${config.sops.secrets.grafana_oauth_client_id.path}}"; + client_secret = "$__file{${config.sops.secrets.grafana_oauth_client_secret.path}}"; + + auth_style = "AutoDetect"; + scopes = "openid email profile groups"; + auth_url = "${config.services.pocket-id.settings.APP_URL}/authorize"; + token_url = "${config.services.pocket-id.settings.APP_URL}/api/oidc/token"; + allow_sign_up = true; + auto_login = true; + name_attribute_path = "display_name"; + login_attribute_path = "preferred_username"; + email_attribute_name = "email:primary"; + email_attribute_path = "email"; + role_attribute_path = "contains(groups[*], 'grafana_super_admin') && 'GrafanaAdmin' || contains(groups[*], 'grafana_admin') && 'Admin' || contains(groups[*], 'grafana_editor') && 'Editor' || 'Viewer'"; + role_attribute_strict = false; + allow_assign_grafana_admin = true; + skip_org_role_sync = false; + use_pkce = true; + use_refresh_token = false; + tls_skip_verify_insecure = false; + }; + + # Database configuration - use PostgreSQL with peer authentication + database = { + type = "postgres"; + host = "/run/postgresql"; # Use Unix socket instead of TCP + name = "grafana"; + user = "grafana"; + # No password needed - using peer authentication via Unix socket + }; + + # Server configuration + server = { + domain = "monitoring.${home_domain}"; + http_addr = "0.0.0.0"; + http_port = 3002; + root_url = "https://monitoring.${home_domain}/grafana/"; + serve_from_sub_path = true; + }; + + # Enable unified alerting (Grafana's built-in alerting) + "unified_alerting" = { + enabled = true; + }; + + # Disable legacy alerting + alerting.enabled = false; + }; + }; + + # ---------------------------- + # Exporters (using built-in NixOS modules) + # ---------------------------- + + # Node exporter - using the built-in module + prometheus.exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9100; + enabledCollectors = [ + "zfs" + "systemd" + ]; + extraFlags = [ + "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|run|tmp|var/lib/docker/.+)($|/)" + "--collector.diskstats.device-exclude=^(loop|ram|fd|sr|dm-|nvme[0-9]n[0-9]p[0-9]+_crypt)$" + ]; + }; + + # Nginx exporter - using the built-in module + prometheus.exporters.nginx = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9113; + scrapeUri = "https://127.0.0.1/server_status"; + sslVerify = false; + }; + + # cAdvisor for Docker containers + cadvisor = { + enable = true; + listenAddress = "127.0.0.1"; + port = 8081; + extraOptions = [ + "--docker_only=true" + "--housekeeping_interval=30s" + "--disable_metrics=hugetlb" + ]; + }; + }; + + # ---------------------------- + # Users and groups for service accounts + # ---------------------------- + users.users.vmagent = { + isSystemUser = true; + group = "vmagent"; + }; + + users.groups.vmagent = {}; + + # ---------------------------- + # Systemd service dependencies + # ---------------------------- + systemd.services.grafana = { + after = [ "postgresql.service" ]; + requires = [ "postgresql.service" ]; + }; + + # ---------------------------- + # SOPS secrets configuration + # ---------------------------- + sops = { + defaultSopsFile = ./secrets.yaml; + secrets = { + grafana_oauth_client_id = { + owner = "grafana"; + restartUnits = ["grafana.service"]; + }; + grafana_oauth_client_secret = { + owner = "grafana"; + restartUnits = ["grafana.service"]; + }; + home_assistant_token = { + owner = "vmagent"; + restartUnits = ["vmagent.service"]; + }; + uptimekuma_grafana_api_key = { + owner = "vmagent"; + restartUnits = ["vmagent.service"]; + sopsFile = ../../common/secrets.yaml; + }; + }; + }; + + # ----------------------------- + # Backups of all this + # ----------------------------- + services.restic.backups.daily = { + paths = [ + config.services.grafana.dataDir + config.services.victoriametrics.stateDir + ]; + }; +} + diff --git a/modules/hosts/nixos/nixnuc/secrets.yaml b/modules/hosts/nixos/nixnuc/secrets.yaml index 7f8ab72..407fbf3 100644 --- a/modules/hosts/nixos/nixnuc/secrets.yaml +++ b/modules/hosts/nixos/nixnuc/secrets.yaml @@ -5,11 +5,14 @@ firefly_app_key: ENC[AES256_GCM,data:sNaqRgFOSmdSS0lCmEG8Nxy/3N7F/hQyS6iPnwau3sQ firefly_pat_data_import: ENC[AES256_GCM,data:mZ32gVaDZGEnpsu4HBHeIZ8UtkTQMrHNWp6iv1jOxsUNJlswWS0kU3IN/sVRWT734KcBGUnriWcXIFvZTx6KXhOjWdEYXyO+g9ws88CWCAguCssiPJI23XaDwMhaqjWRvauzPgjzGXdvg4pFs+myxui+MkLyqcv/ZOBoMt85Nn/M6xuMyyzzmalvTtovCgkja3LJDgxCK7AZoGkXK3X9v15ShHaAjM6mEYkSpONeprew+a7PfmPkM8ynMMOgdCQbRLf/34tLXnTer9qL/kz/Xt14PZsQ5+Fj+fxVwt08xv4FWi3VqUR9wL4Fs4G/yJQHUkDQXDcCk67+TgHxFnsYcxZw6k0ue8u1Ab1aDPyEhUD/UwyzTKyIjzCCIc8NBnst6o8kV62d6ei7dG4PWvNk8m2yNhZVlvoKEjmvZg0Bxv7xGniSimXpvTcMs18+mB0nOcYFUZJf6S0tolcClEDBEegwf0kD+p2F7jXsUjcJsaxqh9Xk215qbTi9iRVtQMWfPO8dCo4EC9vs7MFNmtJsGEp0WR2nwGFh7eIRbzug5+HC5OtUEGbmIUhLmTYnrohkJUKauMYALeYUS7j8GwFdfN0qT0N5r1d5OC/uOih2W5jWiMsOkNWKlaxZ/D1TOTYjLdcHCmIDlQHv9XVFUketsy6KPyg3XkmicKYq18xMi2pkWwGwDV0rd0aqHqRLTNfj3BoXI53eWSsZo8CKuUgWyXM4mIRy7yle2s52a2Idsc8/Zx42MmbWov3c5CNaQDjaseTSO8bv57U30GBY43UBJ4n+p6AIBnXQAN9d/YQ2Wvphe80ZdqyfHzTgj+6PEQKFXzs4KJDE83es/1+HVZdKehpUksenPeQ8UAxoIFtgFjdz6p9JBdXcuFhkWs6oAT96VQtkS6aHgOOUSTN2+H0Q/l6yvIvQB/QcI88r4BiU2vroGqzuVRCbgaXZqjUU06lS608gJOGqIe2a2bHbxwKU+SHEglTdDzRP4n02v8Ua5+BpcWVmhamEzd+8qhLEp/awYi9rd2Kc38T/vowsumLQAvfuZHUt8ySS1JEsyjrWsXQ/cfAiKMeTWkL7b2vEmvx30D8Acz68wDmXwxI/pWDKSe+jagegkmPWVnGQz7MATfApjIl4ZksDQH/O88WrYVF71YYfhA2Qpxr+2FHJ0F2kTPV6tv3QPcteJH4s8pVXvgpYZ050FL9/gJ+gB4rt23Lp0pGW+WEo6YenZ0plbWe4ixf8+U4he5eecGAlAYFEVYZnl6WKLLhX9/xMG9vNJODnUxF97csMPDYiGBhupT8dVk6+1ww=,iv:L0Ff7RYYOPqPeR81LJuTMZ5dsmeQrJtfO1e7Aei+tc4=,tag:wK5s7gRQNpk2aOnsIhtr2A==,type:str] firefly_simplefin_token: ENC[AES256_GCM,data:xJGII+ChIinSpJt+DNU8k8xtgxZkQPmr+02MpO39SylPI6qLHUiXib8xHGrxSSoux19dVdUVXsGO3jRsdIfrX4R6vt/zrIi7yK2gSTZaaoYI1nRg125aino1tIu1Xc+btbro26gDUL8ExlGOg+xvCdPrsmEPbuxVEKm1XxSOOjfvHFjhaMr8XV8/ewqcf5fYlUXEOtNyb6TCrNnp1eSp8+/Uii9GU5O8dlCt1Rof2wNeGQD4lV+lRZqOD0QX2aL5fyXxNUm5W8qFXLiscS8XCl0x2MM9X/mSQ6PC9MlY4WOHXw9/1S3jiVpUyM0U7wJO,iv:+/EwhdcfEwDm+1nhMuIFMiRvT/wbH+vIsmiOLP8bX8g=,tag:RjBomarf49a3xT2TPSTV+A==,type:str] firefly_vanity_url: ENC[AES256_GCM,data:ypIv2VA/B8RStW5Ifj5J20n/MI+jevd0vYrJPchERpt+txPJ0fo=,iv:16/hQyr/TJgLlNjJSUirEpth7jSHP/ooXQ/Vr97wBiA=,tag:p4WiNKm1zkizQCOFVKcW0Q==,type:str] +grafana_oauth_client_id: ENC[AES256_GCM,data:AyqM4bV/9yiWyi0/wqu/SUt5cXYgpsvDcsOgCWENcLeFgGbV,iv:jbUSZQwGOc9V0RON9nAvI86HjGCDuh4wrnpgWcWNG0I=,tag:V6Xe1/r0srITkPDQZ2G+wA==,type:str] +grafana_oauth_client_secret: ENC[AES256_GCM,data:As+m4hLWXy2uMq3KHWPtKzsRY4faYjGUrCdCSzP+TuI=,iv:3UAl8vuf7gUc5voyA2yuY8iiBaoI0iF3U7AFv0siskE=,tag:oD3B5SCNkIfqn88gBxy3Fw==,type:str] home_assistant_token: ENC[AES256_GCM,data:fNpoH60rXAsoVx/NBoDobDw/e6IoeoZfef1uDR7HbmnHNI101b+kQKkB8wBXEDLf7MlqlXVc1ExlgYFUo7+h2msx4WZUCGzuHtp1cMP0O9s2PZoQU8KQM2Frd69vOUcOT5y5ShJZPRrf6H2UKrm//3jE9zDOxxy3cQj6Q+jQLXX4AfZ12JAKzSiee9URWdU8eCHbnquUl1RNHF7zZQ8Mr9m41sBOrpBXt7ErsrhxxRhwlk7qprLt,iv:1j/QmOkLYd7nA+wXS49drBHU09HzMP3XxPbPdaErV6E=,tag:ftyOeNaN5PwNebeMPNAmTg==,type:str] immich_kiosk_basic_auth: ENC[AES256_GCM,data:R/vaYXe0SRwSUg8zC367vA/OYhjIcgfbHxR3OhLt1YYUhPNXVuUGmn199zLD,iv:GBEjJKvbwHAS75negE2whTlI4wS8K2nQYNm3015DFoY=,tag:zs+bJEr9JlIrJtikHycN4g==,type:str] mealie: ENC[AES256_GCM,data:fZFBWlh/nbxK0GA6+fb1FK6aFfbiV/GsBYKYRPgavcsB9h4HwRSZN94gPQ3AfGnk1q08ORPBYShNUIaVdsf4+t3taDC7bNTwPqFGRxoeKOU15BPj6Fpw7SHlplApUg7RDmuTWZuXDH0F49N+09GU/Lc26ewHMAzL8CtGg/H7s3CQfFEqCYYcXJeUcnNDu6PRnKFIAppflSScREp29CUf/IfTYQXidQC25upZgcvGJ+dCTDyNFftlhQiFQh8nFRFVJfKk4iBqnig3fOYsTD32ohAotfDGsSUWb9yjoYQNnNn0V14/,iv:WRFiBtavTP3iP5YSaCMkd7ag8ccx1BO76Vlw6axPkJU=,tag:4BhAxbF8JUB0MV12yx86uQ==,type:str] mtnmesh_bot_dot_env: ENC[AES256_GCM,data:jKz0voG/a7Eq+zHI2fsejTpu5H+ic5ZE8VmtOQDQEOTvYT8GWr2oCGwh68ql7g+nU5hlNN7uLl9LreW59nTgmvWNCVjVNKchxJeydktMUoq2FlgyoC2I354bi9gFEL0X5JYRR7jG9iwy4WGyEFfhhUJn1MSwOurRZPw/tdWTUKQVwN5oqjMxkXrZhFLDzq6BrTs0hBpzRYTe,iv:ZBxHkW8VyU+8v4GiDJMEaeqm3Fdtuwm7M/7YXkfkMnw=,tag:S8hTOSicQNIAj/Gdxzw5+A==,type:str] nextcloud_admin_pass: ENC[AES256_GCM,data:KztB3Tkqlt73PEO41lthGYElrbwVdfqQgT6f,iv:kRwXqGJO4AUOMq+uYzndGhscaJiyvG4ANKabHHd78YM=,tag:dP3PgKafDTv8x7huKJGDqA==,type:str] +nginx_basic_auth: ENC[AES256_GCM,data:9qy8ccV9yWOrQtagf4D4fTp4v8GuY5ORxsE8hUv/vKngOhlG1ikvuIJacIfTfZj9JCoVWnQ=,iv:UGh/KGqT93t91rXu0JN1Hi4/Hb0ICnGiKkymm/5VLU0=,tag:8X6+bKk9zqggRXQhXrhJAg==,type:str] psitransfer_dot_env: ENC[AES256_GCM,data:bhvU0AOCjecZ62BtLw4H1DdkLeatI+uUl6L7UkdDRkBF3sayO45Z1eR4q60tflXucyTGhT8WgKFz53I+C2dn265wzojIRc3Xr4TBLyWpfJ7/dct40SckgUiRvOnrefiriWQ=,iv:DGMhDkzgeupzzTJnCdVWDPUSo2wxI3MAypKQwVfHExE=,tag:KbteGqrkqgj2XB1lvlk/yQ==,type:str] pinchflat_dot_env: ENC[AES256_GCM,data:8DLiFXThG5PGJ0ymW5bMVy5A8dM=,iv:BGkVvxaNwFIMSaA3F6h4ZsgkC9tm1lohA0lg2pgZhpw=,tag:qQNrluP5exdKG3NXgQHM8g==,type:str] tandoor_db_pass: ENC[AES256_GCM,data:X0unx5jquLsUXadbF6xLjjeGY+f8Ec4kdc15JQ==,iv:XptlJHfAkF+3jbgJTqxhVReYjuVVdk3NzfPepP78DRI=,tag:3RG5P9QGCJ/fjdxWpY1xWA==,type:str] @@ -25,7 +28,7 @@ sops: bHZlNTZDV2NYU1hQQy9mem80SFF6TFkKfmjkJBfTdh0vTtGaVx1t3tHJvSsAwdYD PF025X9U+yG2oIopwXEVBkxcD70eyuJn3OqH0xoVLBkbhNM9i8LHrA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-11-10T23:26:34Z" - mac: ENC[AES256_GCM,data:pwgPKBBp2oNu2RRZDprbpVmWxCbsReS3l5/alUfJ7KrMWU6tWiKXNbE2HY/40HYAGyJQ4Jal+u4tKEnPReVubEIe8NwuMlXaDh4ynEN1DId9FaRuCjKatkySWPOGO6DFd4Ajr1Xt1XtWVRqhlCAf5nE+nG1ZBQbloWgOEQZ9m6s=,iv:D0l5yu4lFwrboHbTN26ECPBM6MluPHGR2x6JAwbAUoE=,tag:ctlUVc2CFglsYuTYyCuuYQ==,type:str] + lastmodified: "2026-02-01T03:12:35Z" + mac: ENC[AES256_GCM,data:2PCSk5RQfgsDkQwlujmrBw4yDOIypKBeW/MAF339OR2o77Dz4+YHbUjxoPHt84bpZDMNeUDAifQUoBrKqq66gBJU7CcF/A/dRGCw5xxkdGGEqIjOX+SpC4I+j0zfJ34Pc1BvmTtY32Ivb9njqKZtTj21KJGMB/NDdkgYrDkqY+g=,iv:TSh4Xlmu840HVPBRw+2D2NoDURkEusjwhUEVoL0YWvs=,tag:4K6sHya1LEOziB4zBo0QIg==,type:str] unencrypted_suffix: _unencrypted version: 3.11.0