poc(monitoring): add monitoring for gitserver backup job

rubenhoenle · Nov 4, 2024 · 37988f8 · 37988f8
1 parent ddfcebc
commit 37988f8
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 0 deletions.
diff --git a/modules/secrets.nix b/modules/secrets.nix
@@ -12,6 +12,12 @@
     group = "backup";
     mode = "440";
   };
+  age.secrets.healthchecksIoUuid = {
+    file = ../secrets/healthchecks-io-uuid.age;
+    owner = "ruben";
+    group = "backup";
+    mode = "440";
+  };
   age.secrets.paperlessPassword = {
     file = ../secrets/paperless-password.age;
     owner = "paperless";

diff --git a/secrets/healthchecks-io-uuid.age b/secrets/healthchecks-io-uuid.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 N8lbiw 8r4vdhOkcRzbxR9EaCjSToJoqb6AZnYAOZ71IvfigU4
++1YesObAVKnq3Pol129VxS7xmNMTKXrOOGDwM6U75Q4
+-> ssh-ed25519 Zy726Q HDV3dD0FGrejPT/nwBwO8BUbTmFf6HklGtwDN1cZqjM
+bsitFgTgV+QELXFxP3l5zpFuiDWfiec4BriwdPyXBYE
+--- uLQsTpLAAddVG9mnyGS3GhC3w+ZioQclTzkrea6XZJ0
+C&��l\)���9�{I����V	aJG8��p�r��m��'��F������t���-#
diff --git a/secrets/secrets.nix b/secrets/secrets.nix
@@ -17,4 +17,7 @@ in
 
   /* bk nextcloud backup */
   "bk-nextcloud-backup-netrc.age".publicKeys = [ mandalore millenium-falcon ];
+
+  /* uuid for healthchecks.io cronjob monitoring */
+  "healthchecks-io-uuid.age".publicKeys = [ mandalore millenium-falcon ];
 }
diff --git a/services/gitserver.nix b/services/gitserver.nix
@@ -42,6 +42,7 @@
         paths = [
           "/var/lib/git-server"
         ];
+        backupPrepareCommand = "${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 https://hc-ping.com/$(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthchecksIoUuid.path})/backup-gitserver/start";
         pruneOpts = [
           "--keep-hourly 48"
           "--keep-daily 7"
@@ -55,6 +56,26 @@
           Persistent = true;
         };
       };
+
+      systemd.services."restic-backups-gitserver" = {
+        onSuccess = [ "[email protected]" ];
+        onFailure = [ "[email protected]" ];
+      };
+
+      systemd.services."restic-notify-gitserver@" =
+        let
+          script = pkgs.writeText "restic-notify-gitserver.sh"
+            ''
+              ${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 https://hc-ping.com/$(cat ${config.age.secrets.healthchecksIoUuid.path})/backup-gitserver/''${MONITOR_EXIT_STATUS}
+            '';
+        in
+        {
+          serviceConfig = {
+            Type = "oneshot";
+            User = "git";
+            ExecStart = "${pkgs.bash}/bin/bash ${script}";
+          };
+        };
     };
 }