diff --git a/systems/x86_64-linux/mx/default.nix b/systems/x86_64-linux/mx/default.nix index e1d348a..f0043c3 100644 --- a/systems/x86_64-linux/mx/default.nix +++ b/systems/x86_64-linux/mx/default.nix @@ -1,4 +1,4 @@ -{ pkgs, lib, ... }: +{ pkgs, lib, config, ... }: { imports = [ # ./goaccess.nix @@ -82,6 +82,10 @@ zola ]; + sops.secrets.ntfy = { + sopsFile = ../../../.secrets/hetzner/ntfy.yaml; + }; + sops.age.sshKeyPaths = [ "/var/lib/secrets/ssh_host_ed25519_key" ]; services.openssh = { @@ -113,6 +117,12 @@ ${pkgs.mailutils}/bin/mail -s '/boot Disk Space Alert' harald << EOF Your /boot partition remaining free space is critically low. Used: $CURRENT% EOF + TOKEN=$(cat ${config.sops.secrets.ntfy.path}) + ${pkgs.curl}/bin/curl -s -H "Authorization: Bearer $TOKEN" \ + -H "Title: /boot Disk Space Alert" \ + -H "Priority: high" \ + -d "Boot partition at $CURRENT%" \ + http://127.0.0.1:2586/alerts fi '' ); @@ -132,6 +142,12 @@ ${pkgs.mailutils}/bin/mail -s '/ Disk Space Alert' harald << EOF Your root partition remaining free space is critically low. Used: $CURRENT% EOF + TOKEN=$(cat ${config.sops.secrets.ntfy.path}) + ${pkgs.curl}/bin/curl -s -H "Authorization: Bearer $TOKEN" \ + -H "Title: / Disk Space Alert" \ + -H "Priority: high" \ + -d "Root partition at $CURRENT%" \ + http://127.0.0.1:2586/alerts fi '' ); diff --git a/systems/x86_64-linux/mx/ntfy.nix b/systems/x86_64-linux/mx/ntfy.nix index 7319c98..037532a 100644 --- a/systems/x86_64-linux/mx/ntfy.nix +++ b/systems/x86_64-linux/mx/ntfy.nix @@ -1,4 +1,4 @@ -{ ... }: +{ config, pkgs, lib, ... }: { services.ntfy-sh = { enable = true; @@ -18,4 +18,31 @@ proxyWebsockets = true; }; }; + + # Notify via ntfy on any service failure (alongside email) + systemd.services."ntfy-failure@" = { + description = "Send ntfy notification on service failure"; + onFailure = lib.mkForce [ ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "ntfy-failure-notify" '' + TOKEN=$(cat ${config.sops.secrets.ntfy.path}) + UNIT="$1" + ${pkgs.curl}/bin/curl -s \ + -H "Authorization: Bearer $TOKEN" \ + -H "Title: Service failed: $UNIT" \ + -H "Priority: urgent" \ + -H "Tags: rotating_light" \ + -d "$(systemctl status --full "$UNIT" 2>&1 | head -40)" \ + http://127.0.0.1:2586/alerts + ''; + }; + scriptArgs = "%i"; + }; + + systemd.services = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + config.onFailure = [ "ntfy-failure@%n.service" ]; + }); + }; }