monitoring: add grafana annotations for zfs scrub events

services/monitoring.nix

@@ -108,6 +108,18 @@ let
         type = "tags";
         tags = [ "jellyfin" ];
       }
+      {
+        name = "ZFS Scrubs";
+        datasource = {
+          type = "grafana";
+          uid = "-- Grafana --";
+        };
+        enable = true;
+        iconColor = "orange";
+        showIn = 0;
+        type = "tags";
+        tags = [ "zfs-scrub" ];
+      }
     ];
 
     panels = [

services/zfs-scrub-annotations.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  pkgs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  grafanaUrl = "http://127.0.0.1:${toString service_configs.ports.private.grafana.port}";
+
+  script = pkgs.writeShellApplication {
+    name = "zfs-scrub-annotations";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+      coreutils
+      gnugrep
+      gnused
+      config.boot.zfs.package
+    ];
+    text = builtins.readFile ./zfs-scrub-annotations.sh;
+  };
+in
+{
+  systemd.services.zfs-scrub = {
+    environment = {
+      GRAFANA_URL = grafanaUrl;
+      STATE_DIR = "/run/zfs-scrub-annotations";
+    };
+    serviceConfig = {
+      RuntimeDirectory = "zfs-scrub-annotations";
+      ExecStartPre = [ "-${lib.getExe script} start" ];
+      ExecStopPost = [ "${lib.getExe script} stop" ];
+    };
+  };
+}

services/zfs-scrub-annotations.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# ZFS scrub annotation script for Grafana
+# Usage: zfs-scrub-annotations.sh {start|stop}
+# Required env: GRAFANA_URL, STATE_DIR
+# Required on PATH: zpool, curl, jq, paste, date, grep, sed
+
+set -euo pipefail
+
+ACTION="${1:-}"
+GRAFANA_URL="${GRAFANA_URL:?GRAFANA_URL required}"
+STATE_DIR="${STATE_DIR:?STATE_DIR required}"
+
+case "$ACTION" in
+  start)
+    POOLS=$(zpool list -H -o name | paste -sd ', ')
+    NOW_MS=$(date +%s%3N)
+
+    RESPONSE=$(curl -sf --max-time 5 \
+      -X POST "$GRAFANA_URL/api/annotations" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "ZFS scrub: $POOLS" --argjson time "$NOW_MS" \
+        '{time: $time, text: $text, tags: ["zfs-scrub"]}')" \
+    ) || exit 0
+
+    echo "$RESPONSE" | jq -r '.id' > "$STATE_DIR/annotation-id"
+    ;;
+
+  stop)
+    ANN_ID=$(cat "$STATE_DIR/annotation-id" 2>/dev/null) || exit 0
+    [ -z "$ANN_ID" ] && exit 0
+
+    NOW_MS=$(date +%s%3N)
+
+    RESULTS=""
+    while IFS= read -r pool; do
+      scan_line=$(zpool status "$pool" | grep "scan:" | sed 's/^[[:space:]]*//')
+      RESULTS="${RESULTS}${pool}: ${scan_line}"$'\n'
+    done < <(zpool list -H -o name)
+
+    TEXT=$(printf "ZFS scrub completed\n%s" "$RESULTS")
+
+    curl -sf --max-time 5 \
+      -X PATCH "$GRAFANA_URL/api/annotations/$ANN_ID" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "$TEXT" --argjson timeEnd "$NOW_MS" \
+        '{timeEnd: $timeEnd, text: $text}')" || true
+
+    rm -f "$STATE_DIR/annotation-id"
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop}" >&2
+    exit 1
+    ;;
+esac