diff --git a/modules/hosts/common/secrets.yaml b/modules/hosts/common/secrets.yaml index d6b4e97..8927e35 100644 --- a/modules/hosts/common/secrets.yaml +++ b/modules/hosts/common/secrets.yaml @@ -3,6 +3,7 @@ gandi_api: ENC[AES256_GCM,data:YsdDMk75miIKO4LkCZjfwJw6gxfrmsTL,iv:BOPRxB661sPJn restic_env: ENC[AES256_GCM,data:FCYR8tkClRwfcjUotcr28D6uRz7sNihn50nw38CaYnqOD/U9+5kU0iAPSvqAbeuw+xUoKKKAPAfMHI12dPTYt17Wz1N7i4a+MRkiIR9pjyv5KZTK59G+,iv:jStc8GMbZUQUgooZiRdImSZskdckYN1cRm2gsKbUyYY=,tag:HpQQIj1j7fjCmxkSeY/k4g==,type:str] restic_repo: ENC[AES256_GCM,data:kCoNYVKwB87W4h5doa3IXj4n,iv:jKEw/Hki/tp3RSTsRB4dlg593I5B4pCLBav84ADCh70=,tag:+GFF5vHOVw0r/G8BbhcCjw==,type:str] restic_password: ENC[AES256_GCM,data:PfQsxJul1Qpt3WQoUEI941l+yng3lVjhDd8=,iv:U5KjhcVqyksN2ay19RBjNhYIB31tUbfNRIqCEx/+Wbc=,tag:jsoU+B1mjAprPK+M5I0pAQ==,type:str] +uptimekuma_grafana_api_key: ENC[AES256_GCM,data:irDlzCrDO5n/pMRHagueBTrXmm3herjOSioAz9E5O4a9dJD4ALhp5LPpmTs=,iv:oJhm8eZWSEvqF3Whv/UrS6sgIRdrZ0eqWF98vdwD0V8=,tag:HvOBFo4CJ9NLSz9FwMCP2w==,type:str] wifi_creds: ENC[AES256_GCM,data:9lgTtI8YHyCHrvqss4W7coLnqfOAoQzrCQne6dLv0x66pt7jLo4Y6YSd3TklRTurS9usvNk3sg==,iv:6g86hOmpnOxf4p4C+wPit7EP0DD+xb+cINiWRJnTRDM=,tag:ZW336IhXtrf5l5n/RJecoQ==,type:str] sops: kms: [] @@ -100,8 +101,8 @@ sops: Y3I0d2xaYlRwVU9CdE44UW9vZ1NJeFUK5DQu30MuGjMq5YRSTh2II2uNvWm2XF9B YDcK/E1xKGIA/tKk/DDmpbUZMTIzh+tmYcN72EQQqlT/9a2HyINChg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-01-11T02:27:36Z" - mac: ENC[AES256_GCM,data:Eezf5E3vh10bZTQCxGMZxioUoJqoc9rNBdMu+Wske1SC34Z8GzWbxy3s1T4RPB8I84woIigSgiwj03bZ/F5dchUqNtZwZDObpB44Ru9SEXMB/zEgM8g8LLha/Dgj1MFNbsVR9j2VhacDVv5XJKkjzr/TY5tKedi2mTSHUkpjAGI=,iv:U6YQWh0ISZ0LV6TrwA9QEGal/+lNyTV+tA6yCfNjd6A=,tag:6AUhKo1PniUxW98Yrq3Ulg==,type:str] + lastmodified: "2025-01-14T03:30:50Z" + mac: ENC[AES256_GCM,data:lOrSir70ZiZKjajRLUN83FoQQ0+hwLznbul6Z8hVjbxBvXdwvaMfi/BmfG/+wOMFjShU+fEStAjryoKCcaB3RJod2MyncvE4+fY2lmq7U/T1GHEknQ5xm42J6+Dd79P48mDsJ9kUQXO1wpp9CEVkW5hTfzPGYV2tRWY3a9hgz3Y=,iv:+3hE34n4f2zy17TeoDF/lWvFaX2Rd7ZsojlpZq9R4fE=,tag:Bcs3CAKIk+mTwy0dGuzVMg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.2 diff --git a/modules/hosts/nixos/nixnuc/default.nix b/modules/hosts/nixos/nixnuc/default.nix index 353e08f..c2873f1 100644 --- a/modules/hosts/nixos/nixnuc/default.nix +++ b/modules/hosts/nixos/nixnuc/default.nix @@ -37,8 +37,10 @@ in { jellyfin-web net-snmp nginx + nvme-cli podman-compose podman-tui # status of containers in the terminal + smartmontools yt-dlp ]; @@ -75,16 +77,19 @@ in { # Open ports in the firewall. firewall = { allowedTCPPorts = [ - 22 # ssh - 80 # http to local Nginx - 443 # https to local Nginx - 3000 # PsiTransfer in oci-container - 3001 # immich-kiosk in compose - 3030 # Forgejo - 8001 # Tube Archivist - 8384 # Syncthing gui - 8888 # Atuin - 8090 # Wallabag in docker compose + 22 # ssh + 80 # http to local Nginx + 443 # https to local Nginx + 3000 # PsiTransfer in oci-container + 3001 # immich-kiosk in compose + 3002 # grafana + 3030 # Forgejo + 8001 # Tube Archivist + 8384 # Syncthing gui + 8888 # Atuin + 8090 # Wallabag in docker compose + 9090 # Prometheus Server + 9273 # Telegraf's Prometheus endpoint 13378 # Audiobookshelf in oci-container 22000 # Syncthing transfers ]; @@ -194,6 +199,18 @@ in { stateDir = "/orico/forgejo"; }; fwupd.enable = true; + grafana = { + enable = true; + settings = { + server = { + domain = "monitoring.${home_domain}"; + http_addr = "0.0.0.0"; + http_port = 3002; + root_url = "https://monitoring.${home_domain}/grafana/"; # Not needed if it is `https://your.domain/` + serve_from_sub_path = true; + }; + }; + }; jellyfin = { enable = true; openFirewall = true; @@ -300,6 +317,13 @@ in { add_header Content-Type text/html; ''; }; + locations."/server_status" = { + extraConfig = '' + stub_status; + allow 127.0.0.1; + deny all; + ''; + }; }; "ab.${home_domain}" = { listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; @@ -393,6 +417,13 @@ in { client_max_body_size 10M; ''; }; + "monitoring.${home_domain}" = { + listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; + enableACME = true; + acmeRoot = null; + forceSSL = true; + locations."/grafana/".proxyPass = "http://${backend_ip}:3002"; + }; "nc.${home_domain}" = { listen = [{ port = https_port; addr = "0.0.0.0"; ssl = true; }]; enableACME = true; @@ -444,9 +475,45 @@ in { backupAll = true; startAt = "*-*-* 23:00:00"; }; + prometheus = { + enable = true; + checkConfig = "syntax-only"; + globalConfig.scrape_interval = "10s"; # "1m" + scrapeConfigs = [ + { + job_name = "hass"; + scrape_interval = "30s"; + metrics_path = "/api/prometheus"; + static_configs = [{ + targets = [ "192.168.22.22:8123" ]; + }]; + bearer_token_file = config.sops.secrets.home_assistant_token.path; + } + { + job_name = "telegraf"; + scrape_interval = "5s"; + static_configs = [{ + targets = [ "localhost:9273" ]; + }]; + } + { + job_name = "uptimekuma"; + scheme = "https"; + scrape_interval = "30s"; + static_configs = [{ + targets = [ "utk.technicalissues.us" ]; + }]; + basic_auth = { + password_file = config.sops.secrets.uptimekuma_grafana_api_key.path; + username = ""; + }; + } + ]; + }; resolved.enable = true; restic.backups.daily.paths = [ config.services.forgejo.stateDir + config.services.grafana.dataDir config.services.mealie.settings.DATA_DIR config.services.nextcloud.home "${config.users.users.${username}.home}/compose-files/wallabag" @@ -454,6 +521,7 @@ in { "/orico/jellyfin/data" "/orico/jellyfin/staging/downloaded-files" "/var/backup/postgresql" + "/var/lib/prometheus2" ]; syncthing = { enable = true; @@ -461,6 +529,83 @@ in { openDefaultPorts = true; guiAddress = "0.0.0.0:8384"; }; + telegraf = { + enable = true; + extraConfig = { + agent = { + interval = "15s"; + round_interval = true; + collection_jitter = "0s"; + debug = false; + }; + inputs = { + cpu = { + percpu = true; + totalcpu = true; + report_active = true; + collect_cpu_time = true; + }; + disk = { + ignore_fs = [ + "aufs" + "devfs" + "devtmpfs" + "iso9660" + "overlay" + "squashfs" + "tmpfs" + ]; + }; + diskio = { + skip_serial_number = true; + }; + docker = { + endpoint = "unix:///run/docker.sock"; + container_name_include = []; + storage_objects = [ + "container" + "volume" + ]; + perdevice_include = [ + "cpu" + "blkio" + "network" + ]; + total_include = [ + "cpu" + "blkio" + "network" + ]; + timeout = "5s"; + gather_services = false; + }; + mem = {}; + net = { + ignore_protocol_stats = true; + }; + nginx = { + insecure_skip_verify = true; + urls = [ "https://127.0.0.1/server_status" ]; + }; + #smart = {}; + system = {}; + #systemd_units = {}; + zfs = { + datasetMetrics = true; + poolMetrics = true; + }; + }; + outputs = { + prometheus_client = { + collectors_exclude = [ + "gocollector" + "process" + ]; + listen = ":9273"; + }; + }; + }; + }; zfs.autoScrub.enable = true; }; @@ -468,6 +613,10 @@ in { age.keyFile = "${config.users.users.${username}.home}/.config/sops/age/keys.txt"; defaultSopsFile = ./secrets.yaml; secrets = { + home_assistant_token = { + owner = config.users.users.prometheus.name; + restartUnits = ["prometheus.service"]; + }; immich_kiosk_basic_auth = { owner = config.users.users.nginx.name; restartUnits = ["nginx.service"]; @@ -482,6 +631,11 @@ in { }; mealie.mode = "0444"; nextcloud_admin_pass.owner = config.users.users.nextcloud.name; + uptimekuma_grafana_api_key = { + owner = config.users.users.prometheus.name; + restartUnits = ["prometheus.service"]; + sopsFile = ../../common/secrets.yaml; + }; }; }; @@ -507,6 +661,8 @@ in { ]; }; + users.users.telegraf.extraGroups = [ "docker" ]; + # Enable common container config files in /etc/containers virtualisation.containers.enable = true; diff --git a/modules/hosts/nixos/nixnuc/hardware-configuration.nix b/modules/hosts/nixos/nixnuc/hardware-configuration.nix index 17b0c38..4af6582 100644 --- a/modules/hosts/nixos/nixnuc/hardware-configuration.nix +++ b/modules/hosts/nixos/nixnuc/hardware-configuration.nix @@ -28,6 +28,11 @@ fsType = "zfs"; }; + fileSystems."/var/lib/prometheus2" = + { device = "orico/prometheus"; + fsType = "zfs"; + }; + swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/modules/hosts/nixos/nixnuc/secrets.yaml b/modules/hosts/nixos/nixnuc/secrets.yaml index 4d0dd68..0800e7e 100644 --- a/modules/hosts/nixos/nixnuc/secrets.yaml +++ b/modules/hosts/nixos/nixnuc/secrets.yaml @@ -1,6 +1,7 @@ tailscale_key: ENC[AES256_GCM,data:aB3KUD4QYm+ZDrjjLcU3gQ8kneVGkVYBsrkVcioOhxunal2FekLDrpKxJwNXuiwx2M5vipnGAEPO,iv:e+tPPfVYkv4U0KRGwspWb1O3ZQom/WFFGm9H9cd/KKE=,tag:ZG5z1C18bj1L7DcGzunQ0w==,type:str] local_git_config: ENC[AES256_GCM,data:Nqwog5C4wnRzNoS4oqaYQ4J1DIj7fUL1y/nXESquR0N7KQ+ebhvuJnM=,iv:Q6o45LZStS3k8iO7s2P6u7OrKFu5alplshZuGgeRKmk=,tag:NcLJrI9AK4eDroODX15lcA==,type:str] local_private_env: ENC[AES256_GCM,data:qOPXTS2uo/1jyVEKCtBvuK/dzZaPf1K5tHuSVF2hBg4fdPYIsDPkM108cGVxJviebB3xVZejn/JVOdUDXQj6,iv:TtyMTOJXaPUrbSaAdtMaGPBlwLl/Y/IBYVCzhhiZozY=,tag:hUyVL8xk3w1iMwNAZw5QUw==,type:str] +home_assistant_token: ENC[AES256_GCM,data:fNpoH60rXAsoVx/NBoDobDw/e6IoeoZfef1uDR7HbmnHNI101b+kQKkB8wBXEDLf7MlqlXVc1ExlgYFUo7+h2msx4WZUCGzuHtp1cMP0O9s2PZoQU8KQM2Frd69vOUcOT5y5ShJZPRrf6H2UKrm//3jE9zDOxxy3cQj6Q+jQLXX4AfZ12JAKzSiee9URWdU8eCHbnquUl1RNHF7zZQ8Mr9m41sBOrpBXt7ErsrhxxRhwlk7qprLt,iv:1j/QmOkLYd7nA+wXS49drBHU09HzMP3XxPbPdaErV6E=,tag:ftyOeNaN5PwNebeMPNAmTg==,type:str] immich_kiosk_basic_auth: ENC[AES256_GCM,data:R/vaYXe0SRwSUg8zC367vA/OYhjIcgfbHxR3OhLt1YYUhPNXVuUGmn199zLD,iv:GBEjJKvbwHAS75negE2whTlI4wS8K2nQYNm3015DFoY=,tag:zs+bJEr9JlIrJtikHycN4g==,type:str] mealie: ENC[AES256_GCM,data:HuO2c5na5r271x98GLqYBP/ME+phH58e2scelEH7F3R+2liLRGK9xCA0ciHLJCo8,iv:bo6hTIsWfnWueRCiLcpxYoiol8oGOoPTHDgPQ7FnYqM=,tag:rHRfIv4vnIjO/eotxYmI/Q==,type:str] nextcloud_admin_pass: ENC[AES256_GCM,data:KztB3Tkqlt73PEO41lthGYElrbwVdfqQgT6f,iv:kRwXqGJO4AUOMq+uYzndGhscaJiyvG4ANKabHHd78YM=,tag:dP3PgKafDTv8x7huKJGDqA==,type:str] @@ -22,8 +23,8 @@ sops: bHZlNTZDV2NYU1hQQy9mem80SFF6TFkKfmjkJBfTdh0vTtGaVx1t3tHJvSsAwdYD PF025X9U+yG2oIopwXEVBkxcD70eyuJn3OqH0xoVLBkbhNM9i8LHrA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-01-10T16:08:06Z" - mac: ENC[AES256_GCM,data:bwPvp28nSVurnSdyuLoFCZ5XbyZlAqeXVjSUk97H1R5EuMx+2UWGNIBm5Llgm+MwsUUkOpEI/V49M1zS8wRuKxsUA1JkmXG4nis856Tx2tl0SDoIkVdCYiq+8nZTWRv1BAYblny6HHUeBzWX3Sgx8NvXC+Qfkl1fuwJcSP2zvCk=,iv:WfzIryQ78UR4Rc9kOJFgBtpV5PkTt2qT3yx/4vDcASI=,tag:49RBwahEAIvicte0S89idw==,type:str] + lastmodified: "2025-01-14T04:38:46Z" + mac: ENC[AES256_GCM,data:yXoXF3BgOLegwht2BEDhplCxcVB72NdWRKmRzymJMjybuCLdO/IWk7Mk5Kqd5ckpYLq7vuRl2I+QoOGSiZwr8c8FT0eE8YKevMH6QQkGDZfEDAqJIO3FNT5G+ZaVkFLA8OwkAOhIbz3eFNwYU7mAwooKM/mlcEa7v0bmq17RMz0=,iv:LKZ2sP8YD1uGgKVfJT42/QpYPH5PnmzpJd5J+vlEhfg=,tag:Lc0BBOwa0OI3WwEp9Ctu6w==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.2