feat(mx): add ntfy push notifications for disk alerts and service failures

- Disk check scripts now send ntfy alerts in addition to email
- New ntfy-failure@ template service notifies on any systemd service failure
- Uses sops-managed token for ntfy authentication

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

systems/x86_64-linux/mx/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, lib, ... }:
+{ pkgs, lib, config, ... }:
 {
   imports = [
     # ./goaccess.nix
@@ -82,6 +82,10 @@
     zola
   ];
 
+  sops.secrets.ntfy = {
+    sopsFile = ../../../.secrets/hetzner/ntfy.yaml;
+  };
+
   sops.age.sshKeyPaths = [ "/var/lib/secrets/ssh_host_ed25519_key" ];
 
   services.openssh = {
@@ -113,6 +117,12 @@
                 ${pkgs.mailutils}/bin/mail -s '/boot Disk Space Alert' harald << EOF
             Your /boot partition remaining free space is critically low. Used: $CURRENT%
             EOF
+                TOKEN=$(cat ${config.sops.secrets.ntfy.path})
+                ${pkgs.curl}/bin/curl -s -H "Authorization: Bearer $TOKEN" \
+                  -H "Title: /boot Disk Space Alert" \
+                  -H "Priority: high" \
+                  -d "Boot partition at $CURRENT%" \
+                  http://127.0.0.1:2586/alerts
             fi
           ''
         );
@@ -132,6 +142,12 @@
                 ${pkgs.mailutils}/bin/mail -s '/ Disk Space Alert' harald << EOF
             Your root partition remaining free space is critically low. Used: $CURRENT%
             EOF
+                TOKEN=$(cat ${config.sops.secrets.ntfy.path})
+                ${pkgs.curl}/bin/curl -s -H "Authorization: Bearer $TOKEN" \
+                  -H "Title: / Disk Space Alert" \
+                  -H "Priority: high" \
+                  -d "Root partition at $CURRENT%" \
+                  http://127.0.0.1:2586/alerts
             fi
           ''
         );

systems/x86_64-linux/mx/ntfy.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ config, pkgs, lib, ... }:
 {
   services.ntfy-sh = {
     enable = true;
@@ -18,4 +18,31 @@
       proxyWebsockets = true;
     };
   };
+
+  # Notify via ntfy on any service failure (alongside email)
+  systemd.services."ntfy-failure@" = {
+    description = "Send ntfy notification on service failure";
+    onFailure = lib.mkForce [ ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = pkgs.writeShellScript "ntfy-failure-notify" ''
+        TOKEN=$(cat ${config.sops.secrets.ntfy.path})
+        UNIT="$1"
+        ${pkgs.curl}/bin/curl -s \
+          -H "Authorization: Bearer $TOKEN" \
+          -H "Title: Service failed: $UNIT" \
+          -H "Priority: urgent" \
+          -H "Tags: rotating_light" \
+          -d "$(systemctl status --full "$UNIT" 2>&1 | head -40)" \
+          http://127.0.0.1:2586/alerts
+      '';
+    };
+    scriptArgs = "%i";
+  };
+
+  systemd.services = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule {
+      config.onFailure = [ "ntfy-failure@%n.service" ];
+    });
+  };
 }