Monitoring time with Grafana & Prometheus

modules/hosts/common/secrets.yaml

@@ -3,6 +3,7 @@ gandi_api: ENC[AES256_GCM,data:YsdDMk75miIKO4LkCZjfwJw6gxfrmsTL,iv:BOPRxB661sPJn
 restic_env: ENC[AES256_GCM,data:FCYR8tkClRwfcjUotcr28D6uRz7sNihn50nw38CaYnqOD/U9+5kU0iAPSvqAbeuw+xUoKKKAPAfMHI12dPTYt17Wz1N7i4a+MRkiIR9pjyv5KZTK59G+,iv:jStc8GMbZUQUgooZiRdImSZskdckYN1cRm2gsKbUyYY=,tag:HpQQIj1j7fjCmxkSeY/k4g==,type:str]
 restic_repo: ENC[AES256_GCM,data:kCoNYVKwB87W4h5doa3IXj4n,iv:jKEw/Hki/tp3RSTsRB4dlg593I5B4pCLBav84ADCh70=,tag:+GFF5vHOVw0r/G8BbhcCjw==,type:str]
 restic_password: ENC[AES256_GCM,data:PfQsxJul1Qpt3WQoUEI941l+yng3lVjhDd8=,iv:U5KjhcVqyksN2ay19RBjNhYIB31tUbfNRIqCEx/+Wbc=,tag:jsoU+B1mjAprPK+M5I0pAQ==,type:str]
+uptimekuma_grafana_api_key: ENC[AES256_GCM,data:irDlzCrDO5n/pMRHagueBTrXmm3herjOSioAz9E5O4a9dJD4ALhp5LPpmTs=,iv:oJhm8eZWSEvqF3Whv/UrS6sgIRdrZ0eqWF98vdwD0V8=,tag:HvOBFo4CJ9NLSz9FwMCP2w==,type:str]
 wifi_creds: ENC[AES256_GCM,data:9lgTtI8YHyCHrvqss4W7coLnqfOAoQzrCQne6dLv0x66pt7jLo4Y6YSd3TklRTurS9usvNk3sg==,iv:6g86hOmpnOxf4p4C+wPit7EP0DD+xb+cINiWRJnTRDM=,tag:ZW336IhXtrf5l5n/RJecoQ==,type:str]
 sops:
     kms: []
@@ -100,8 +101,8 @@ sops:
             Y3I0d2xaYlRwVU9CdE44UW9vZ1NJeFUK5DQu30MuGjMq5YRSTh2II2uNvWm2XF9B
             YDcK/E1xKGIA/tKk/DDmpbUZMTIzh+tmYcN72EQQqlT/9a2HyINChg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-01-11T02:27:36Z"
+    lastmodified: "2025-01-14T03:30:50Z"
-    mac: ENC[AES256_GCM,data:Eezf5E3vh10bZTQCxGMZxioUoJqoc9rNBdMu+Wske1SC34Z8GzWbxy3s1T4RPB8I84woIigSgiwj03bZ/F5dchUqNtZwZDObpB44Ru9SEXMB/zEgM8g8LLha/Dgj1MFNbsVR9j2VhacDVv5XJKkjzr/TY5tKedi2mTSHUkpjAGI=,iv:U6YQWh0ISZ0LV6TrwA9QEGal/+lNyTV+tA6yCfNjd6A=,tag:6AUhKo1PniUxW98Yrq3Ulg==,type:str]
+    mac: ENC[AES256_GCM,data:lOrSir70ZiZKjajRLUN83FoQQ0+hwLznbul6Z8hVjbxBvXdwvaMfi/BmfG/+wOMFjShU+fEStAjryoKCcaB3RJod2MyncvE4+fY2lmq7U/T1GHEknQ5xm42J6+Dd79P48mDsJ9kUQXO1wpp9CEVkW5hTfzPGYV2tRWY3a9hgz3Y=,iv:+3hE34n4f2zy17TeoDF/lWvFaX2Rd7ZsojlpZq9R4fE=,tag:Bcs3CAKIk+mTwy0dGuzVMg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.2

modules/hosts/nixos/nixnuc/default.nix

@@ -37,8 +37,10 @@ in {
     jellyfin-web
     net-snmp
     nginx
+    nvme-cli
     podman-compose
     podman-tui # status of containers in the terminal
+    smartmontools
     yt-dlp
   ];
 
@@ -80,11 +82,14 @@ in {
           443 # https to local Nginx
          3000 # PsiTransfer in oci-container
          3001 # immich-kiosk in compose
+         3002 # grafana
          3030 # Forgejo
          8001 # Tube Archivist
          8384 # Syncthing gui
          8888 # Atuin
          8090 # Wallabag in docker compose
+         9090 # Prometheus Server
+         9273 # Telegraf's Prometheus endpoint
         13378 # Audiobookshelf in oci-container
         22000 # Syncthing transfers
       ];
@@ -194,6 +199,18 @@ in {
       stateDir = "/orico/forgejo";
     };
     fwupd.enable = true;
+    grafana = {
+      enable = true;
+      settings = {
+        server = {
+          domain = "monitoring.${home_domain}";
+          http_addr = "0.0.0.0";
+          http_port = 3002;
+          root_url = "https://monitoring.${home_domain}/grafana/"; # Not needed if it is `https://your.domain/`
+          serve_from_sub_path = true;
+        };
+      };
+    };
     jellyfin = {
       enable = true;
       openFirewall = true;
@@ -300,6 +317,13 @@ in {
               add_header Content-Type text/html;
             '';
           };
+          locations."/server_status" = {
+            extraConfig = ''
+              stub_status;
+              allow 127.0.0.1;
+              deny all;
+            '';
+          };
         };
         "ab.${home_domain}" = {
           listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
@@ -393,6 +417,13 @@ in {
             client_max_body_size 10M;
           '';
         };
+        "monitoring.${home_domain}" = {
+          listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
+          enableACME = true;
+          acmeRoot = null;
+          forceSSL = true;
+          locations."/grafana/".proxyPass = "http://${backend_ip}:3002";
+        };
         "nc.${home_domain}" = {
           listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }];
           enableACME = true;
@@ -444,9 +475,45 @@ in {
       backupAll = true;
       startAt = "*-*-* 23:00:00";
     };
+    prometheus = {
+      enable = true;
+      checkConfig = "syntax-only";
+      globalConfig.scrape_interval = "10s"; # "1m"
+      scrapeConfigs = [
+        {
+          job_name = "hass";
+          scrape_interval = "30s";
+          metrics_path = "/api/prometheus";
+          static_configs = [{
+            targets = [ "192.168.22.22:8123" ];
+          }];
+          bearer_token_file = config.sops.secrets.home_assistant_token.path;
+        }
+        {
+          job_name = "telegraf";
+          scrape_interval = "5s";
+          static_configs = [{
+            targets = [ "localhost:9273" ];
+          }];
+        }
+        {
+          job_name = "uptimekuma";
+          scheme = "https";
+          scrape_interval = "30s";
+          static_configs = [{
+            targets = [ "utk.technicalissues.us" ];
+          }];
+          basic_auth = {
+            password_file = config.sops.secrets.uptimekuma_grafana_api_key.path;
+            username = "";
+          };
+        }
+      ];
+    };
     resolved.enable = true;
     restic.backups.daily.paths = [
       config.services.forgejo.stateDir
+      config.services.grafana.dataDir
       config.services.mealie.settings.DATA_DIR
       config.services.nextcloud.home
       "${config.users.users.${username}.home}/compose-files/wallabag"
@@ -454,6 +521,7 @@ in {
       "/orico/jellyfin/data"
       "/orico/jellyfin/staging/downloaded-files"
       "/var/backup/postgresql"
+      "/var/lib/prometheus2"
     ];
     syncthing = {
       enable = true;
@@ -461,6 +529,83 @@ in {
       openDefaultPorts = true;
       guiAddress = "0.0.0.0:8384";
     };
+    telegraf = {
+      enable = true;
+      extraConfig = {
+        agent = {
+          interval = "15s";
+          round_interval = true;
+          collection_jitter = "0s";
+          debug = false;
+        };
+        inputs = {
+          cpu = {
+            percpu = true;
+            totalcpu = true;
+            report_active = true;
+            collect_cpu_time = true;
+          };
+          disk = {
+            ignore_fs = [
+              "aufs"
+              "devfs"
+              "devtmpfs"
+              "iso9660"
+              "overlay"
+              "squashfs"
+              "tmpfs"
+            ];
+          };
+          diskio = {
+            skip_serial_number = true;
+          };
+          docker = {
+            endpoint = "unix:///run/docker.sock";
+            container_name_include  = [];
+            storage_objects = [
+              "container"
+              "volume"
+            ];
+            perdevice_include = [
+              "cpu"
+              "blkio"
+              "network"
+            ];
+            total_include = [
+              "cpu"
+              "blkio"
+              "network"
+            ];
+            timeout = "5s";
+            gather_services = false;
+          };
+          mem = {};
+          net = {
+            ignore_protocol_stats = true;
+          };
+          nginx = {
+            insecure_skip_verify = true;
+            urls = [ "https://127.0.0.1/server_status" ];
+          };
+          #smart = {};
+          system = {};
+          #systemd_units = {};
+          zfs = {
+            datasetMetrics = true;
+            poolMetrics = true;
+          };
+        };
+        outputs = {
+          prometheus_client = {
+            collectors_exclude = [
+              "gocollector"
+              "process"
+            ];
+            listen = ":9273";
+          };
+        };
+      };
+    };
     zfs.autoScrub.enable = true;
   };
 
@@ -468,6 +613,10 @@ in {
     age.keyFile = "${config.users.users.${username}.home}/.config/sops/age/keys.txt";
     defaultSopsFile = ./secrets.yaml;
     secrets = {
+      home_assistant_token = {
+        owner = config.users.users.prometheus.name;
+        restartUnits = ["prometheus.service"];
+      };
       immich_kiosk_basic_auth = {
         owner = config.users.users.nginx.name;
         restartUnits = ["nginx.service"];
@@ -482,6 +631,11 @@ in {
       };
       mealie.mode = "0444";
       nextcloud_admin_pass.owner = config.users.users.nextcloud.name;
+      uptimekuma_grafana_api_key = {
+        owner = config.users.users.prometheus.name;
+        restartUnits = ["prometheus.service"];
+        sopsFile = ../../common/secrets.yaml;
+      };
     };
   };
 
@@ -507,6 +661,8 @@ in {
     ];
   };
 
+  users.users.telegraf.extraGroups = [ "docker" ];
+
   # Enable common container config files in /etc/containers
   virtualisation.containers.enable = true;
 

modules/hosts/nixos/nixnuc/hardware-configuration.nix

@@ -28,6 +28,11 @@
       fsType = "zfs";
     };
 
+  fileSystems."/var/lib/prometheus2" =
+    { device = "orico/prometheus";
+      fsType = "zfs";
+    };
+
   swapDevices = [ ];
 
   # Enables DHCP on each ethernet and wireless interface. In case of scripted networking

modules/hosts/nixos/nixnuc/secrets.yaml

@@ -1,6 +1,7 @@
 tailscale_key: ENC[AES256_GCM,data:aB3KUD4QYm+ZDrjjLcU3gQ8kneVGkVYBsrkVcioOhxunal2FekLDrpKxJwNXuiwx2M5vipnGAEPO,iv:e+tPPfVYkv4U0KRGwspWb1O3ZQom/WFFGm9H9cd/KKE=,tag:ZG5z1C18bj1L7DcGzunQ0w==,type:str]
 local_git_config: ENC[AES256_GCM,data:Nqwog5C4wnRzNoS4oqaYQ4J1DIj7fUL1y/nXESquR0N7KQ+ebhvuJnM=,iv:Q6o45LZStS3k8iO7s2P6u7OrKFu5alplshZuGgeRKmk=,tag:NcLJrI9AK4eDroODX15lcA==,type:str]
 local_private_env: ENC[AES256_GCM,data:qOPXTS2uo/1jyVEKCtBvuK/dzZaPf1K5tHuSVF2hBg4fdPYIsDPkM108cGVxJviebB3xVZejn/JVOdUDXQj6,iv:TtyMTOJXaPUrbSaAdtMaGPBlwLl/Y/IBYVCzhhiZozY=,tag:hUyVL8xk3w1iMwNAZw5QUw==,type:str]
+home_assistant_token: ENC[AES256_GCM,data:fNpoH60rXAsoVx/NBoDobDw/e6IoeoZfef1uDR7HbmnHNI101b+kQKkB8wBXEDLf7MlqlXVc1ExlgYFUo7+h2msx4WZUCGzuHtp1cMP0O9s2PZoQU8KQM2Frd69vOUcOT5y5ShJZPRrf6H2UKrm//3jE9zDOxxy3cQj6Q+jQLXX4AfZ12JAKzSiee9URWdU8eCHbnquUl1RNHF7zZQ8Mr9m41sBOrpBXt7ErsrhxxRhwlk7qprLt,iv:1j/QmOkLYd7nA+wXS49drBHU09HzMP3XxPbPdaErV6E=,tag:ftyOeNaN5PwNebeMPNAmTg==,type:str]
 immich_kiosk_basic_auth: ENC[AES256_GCM,data:R/vaYXe0SRwSUg8zC367vA/OYhjIcgfbHxR3OhLt1YYUhPNXVuUGmn199zLD,iv:GBEjJKvbwHAS75negE2whTlI4wS8K2nQYNm3015DFoY=,tag:zs+bJEr9JlIrJtikHycN4g==,type:str]
 mealie: ENC[AES256_GCM,data:HuO2c5na5r271x98GLqYBP/ME+phH58e2scelEH7F3R+2liLRGK9xCA0ciHLJCo8,iv:bo6hTIsWfnWueRCiLcpxYoiol8oGOoPTHDgPQ7FnYqM=,tag:rHRfIv4vnIjO/eotxYmI/Q==,type:str]
 nextcloud_admin_pass: ENC[AES256_GCM,data:KztB3Tkqlt73PEO41lthGYElrbwVdfqQgT6f,iv:kRwXqGJO4AUOMq+uYzndGhscaJiyvG4ANKabHHd78YM=,tag:dP3PgKafDTv8x7huKJGDqA==,type:str]
@@ -22,8 +23,8 @@ sops:
             bHZlNTZDV2NYU1hQQy9mem80SFF6TFkKfmjkJBfTdh0vTtGaVx1t3tHJvSsAwdYD
             PF025X9U+yG2oIopwXEVBkxcD70eyuJn3OqH0xoVLBkbhNM9i8LHrA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-01-10T16:08:06Z"
+    lastmodified: "2025-01-14T04:38:46Z"
-    mac: ENC[AES256_GCM,data:bwPvp28nSVurnSdyuLoFCZ5XbyZlAqeXVjSUk97H1R5EuMx+2UWGNIBm5Llgm+MwsUUkOpEI/V49M1zS8wRuKxsUA1JkmXG4nis856Tx2tl0SDoIkVdCYiq+8nZTWRv1BAYblny6HHUeBzWX3Sgx8NvXC+Qfkl1fuwJcSP2zvCk=,iv:WfzIryQ78UR4Rc9kOJFgBtpV5PkTt2qT3yx/4vDcASI=,tag:49RBwahEAIvicte0S89idw==,type:str]
+    mac: ENC[AES256_GCM,data:yXoXF3BgOLegwht2BEDhplCxcVB72NdWRKmRzymJMjybuCLdO/IWk7Mk5Kqd5ckpYLq7vuRl2I+QoOGSiZwr8c8FT0eE8YKevMH6QQkGDZfEDAqJIO3FNT5G+ZaVkFLA8OwkAOhIbz3eFNwYU7mAwooKM/mlcEa7v0bmq17RMz0=,iv:LKZ2sP8YD1uGgKVfJT42/QpYPH5PnmzpJd5J+vlEhfg=,tag:Lc0BBOwa0OI3WwEp9Ctu6w==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.2