tests: extract shared jellyfin test helpers and use real jellyfin in annotations test

configuration.nix

@@ -49,6 +49,7 @@
     ./services/ups.nix
     ./services/monitoring.nix
     ./services/jellyfin-annotations.nix
+    ./services/zfs-scrub-annotations.nix
 
     ./services/bitwarden.nix
     ./services/firefox-syncserver.nix

services/monitoring.nix

@@ -108,6 +108,18 @@ let
         type = "tags";
         tags = [ "jellyfin" ];
       }
+      {
+        name = "ZFS Scrubs";
+        datasource = {
+          type = "grafana";
+          uid = "-- Grafana --";
+        };
+        enable = true;
+        iconColor = "orange";
+        showIn = 0;
+        type = "tags";
+        tags = [ "zfs-scrub" ];
+      }
     ];
 
     panels = [

services/zfs-scrub-annotations.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  pkgs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  grafanaUrl = "http://127.0.0.1:${toString service_configs.ports.private.grafana.port}";
+
+  script = pkgs.writeShellApplication {
+    name = "zfs-scrub-annotations";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+      coreutils
+      gnugrep
+      gnused
+      config.boot.zfs.package
+    ];
+    text = builtins.readFile ./zfs-scrub-annotations.sh;
+  };
+in
+{
+  systemd.services.zfs-scrub = {
+    environment = {
+      GRAFANA_URL = grafanaUrl;
+      STATE_DIR = "/run/zfs-scrub-annotations";
+    };
+    serviceConfig = {
+      RuntimeDirectory = "zfs-scrub-annotations";
+      ExecStartPre = [ "-${lib.getExe script} start" ];
+      ExecStopPost = [ "${lib.getExe script} stop" ];
+    };
+  };
+}

services/zfs-scrub-annotations.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# ZFS scrub annotation script for Grafana
+# Usage: zfs-scrub-annotations.sh {start|stop}
+# Required env: GRAFANA_URL, STATE_DIR
+# Required on PATH: zpool, curl, jq, paste, date, grep, sed
+
+set -euo pipefail
+
+ACTION="${1:-}"
+GRAFANA_URL="${GRAFANA_URL:?GRAFANA_URL required}"
+STATE_DIR="${STATE_DIR:?STATE_DIR required}"
+
+case "$ACTION" in
+  start)
+    POOLS=$(zpool list -H -o name | paste -sd ', ')
+    NOW_MS=$(date +%s%3N)
+
+    RESPONSE=$(curl -sf --max-time 5 \
+      -X POST "$GRAFANA_URL/api/annotations" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "ZFS scrub: $POOLS" --argjson time "$NOW_MS" \
+        '{time: $time, text: $text, tags: ["zfs-scrub"]}')" \
+    ) || exit 0
+
+    echo "$RESPONSE" | jq -r '.id' > "$STATE_DIR/annotation-id"
+    ;;
+
+  stop)
+    ANN_ID=$(cat "$STATE_DIR/annotation-id" 2>/dev/null) || exit 0
+    [ -z "$ANN_ID" ] && exit 0
+
+    NOW_MS=$(date +%s%3N)
+
+    RESULTS=""
+    while IFS= read -r pool; do
+      scan_line=$(zpool status "$pool" | grep "scan:" | sed 's/^[[:space:]]*//')
+      RESULTS="${RESULTS}${pool}: ${scan_line}"$'\n'
+    done < <(zpool list -H -o name)
+
+    TEXT=$(printf "ZFS scrub completed\n%s" "$RESULTS")
+
+    curl -sf --max-time 5 \
+      -X PATCH "$GRAFANA_URL/api/annotations/$ANN_ID" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "$TEXT" --argjson timeEnd "$NOW_MS" \
+        '{timeEnd: $timeEnd, text: $text}')" || true
+
+    rm -f "$STATE_DIR/annotation-id"
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop}" >&2
+    exit 1
+    ;;
+esac

tests/jellyfin-annotations.nix

@@ -4,81 +4,8 @@
   ...
 }:
 let
-  mockServer = pkgs.writeText "mock-server.py" ''
-    import http.server, json, os, sys
-    from urllib.parse import urlparse
-
-    MODE = sys.argv[1]
-    PORT = int(sys.argv[2])
-    DATA_FILE = sys.argv[3]
-
-    class Handler(http.server.BaseHTTPRequestHandler):
-        def log_message(self, fmt, *args):
-            pass
-
-        def _read_body(self):
-            length = int(self.headers.get("Content-Length", 0))
-            return json.loads(self.rfile.read(length)) if length else {}
-
-        def _json(self, code, body):
-            data = json.dumps(body).encode()
-            self.send_response(code)
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
-            self.wfile.write(data)
-
-        def do_GET(self):
-            if MODE == "jellyfin" and self.path.startswith("/Sessions"):
-                try:
-                    with open(DATA_FILE) as f:
-                        sessions = json.load(f)
-                except Exception:
-                    sessions = []
-                self._json(200, sessions)
-            else:
-                self.send_response(404)
-                self.end_headers()
-
-        def do_POST(self):
-            if MODE == "grafana" and self.path == "/api/annotations":
-                body = self._read_body()
-                try:
-                    with open(DATA_FILE) as f:
-                        annotations = json.load(f)
-                except Exception:
-                    annotations = []
-                aid = len(annotations) + 1
-                body["id"] = aid
-                annotations.append(body)
-                with open(DATA_FILE, "w") as f:
-                    json.dump(annotations, f)
-                self._json(200, {"id": aid, "message": "Annotation added"})
-            else:
-                self.send_response(404)
-                self.end_headers()
-
-        def do_PATCH(self):
-            if MODE == "grafana" and self.path.startswith("/api/annotations/"):
-                aid = int(self.path.rsplit("/", 1)[-1])
-                body = self._read_body()
-                try:
-                    with open(DATA_FILE) as f:
-                        annotations = json.load(f)
-                except Exception:
-                    annotations = []
-                for a in annotations:
-                    if a["id"] == aid:
-                        a.update(body)
-                with open(DATA_FILE, "w") as f:
-                    json.dump(annotations, f)
-                self._json(200, {"message": "Annotation patched"})
-            else:
-                self.send_response(404)
-                self.end_headers()
-
-    http.server.HTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
-  '';
-
+  jfLib = import ./jellyfin-test-lib.nix { inherit pkgs lib; };
+  mockGrafana = ./mock-grafana-server.py;
   script = ../services/jellyfin-annotations.py;
   python = pkgs.python3;
 in
@@ -88,6 +15,7 @@ pkgs.testers.runNixOSTest {
   nodes.machine =
     { pkgs, ... }:
     {
+      imports = [ jfLib.jellyfinTestConfig ];
       environment.systemPackages = [ pkgs.python3 ];
     };
 
@@ -95,37 +23,41 @@ pkgs.testers.runNixOSTest {
     import json
     import time
 
-    JELLYFIN_PORT = 18096
+    import importlib.util
+    _spec = importlib.util.spec_from_file_location("jf_helpers", "${jfLib.helpers}")
+    assert _spec and _spec.loader
+    _jf = importlib.util.module_from_spec(_spec)
+    _spec.loader.exec_module(_jf)
+    setup_jellyfin = _jf.setup_jellyfin
+    jellyfin_api = _jf.jellyfin_api
+
     GRAFANA_PORT  = 13000
-    SESSIONS_FILE = "/tmp/sessions.json"
     ANNOTS_FILE   = "/tmp/annotations.json"
     STATE_FILE    = "/tmp/annotations-state.json"
     CREDS_DIR     = "/tmp/test-creds"
     PYTHON        = "${python}/bin/python3"
-    MOCK          = "${mockServer}"
+    MOCK_GRAFANA  = "${mockGrafana}"
     SCRIPT        = "${script}"
 
+    auth_header  = 'MediaBrowser Client="Infuse", DeviceId="test-dev-1", Device="iPhone", Version="1.0"'
+    auth_header2 = 'MediaBrowser Client="Jellyfin Web", DeviceId="test-dev-2", Device="Chrome", Version="1.0"'
+
     def read_annotations():
         out = machine.succeed(f"cat {ANNOTS_FILE} 2>/dev/null || echo '[]'")
         return json.loads(out.strip())
 
     start_all()
-    machine.wait_for_unit("multi-user.target")
+    token, user_id, movie_id, media_source_id = setup_jellyfin(
+        machine, retry, auth_header,
+        "${jfLib.payloads.auth}", "${jfLib.payloads.empty}",
+    )
 
-    with subtest("Setup mock credentials and data files"):
-        machine.succeed(f"mkdir -p {CREDS_DIR} && echo 'fake-api-key' > {CREDS_DIR}/jellyfin-api-key")
-        machine.succeed(f"echo '[]' > {SESSIONS_FILE}")
+    with subtest("Setup mock Grafana and credentials"):
+        machine.succeed(f"mkdir -p {CREDS_DIR}")
+        machine.succeed(f"echo '{token}' > {CREDS_DIR}/jellyfin-api-key")
         machine.succeed(f"echo '[]' > {ANNOTS_FILE}")
-
-    with subtest("Start mock Jellyfin and Grafana servers"):
         machine.succeed(
-            f"systemd-run --unit=mock-jellyfin {PYTHON} {MOCK} jellyfin {JELLYFIN_PORT} {SESSIONS_FILE}"
-        )
-        machine.succeed(
-            f"systemd-run --unit=mock-grafana {PYTHON} {MOCK} grafana {GRAFANA_PORT} {ANNOTS_FILE}"
-        )
-        machine.wait_until_succeeds(
-            f"curl -sf http://127.0.0.1:{JELLYFIN_PORT}/Sessions", timeout=10
+            f"systemd-run --unit=mock-grafana {PYTHON} {MOCK_GRAFANA} {GRAFANA_PORT} {ANNOTS_FILE}"
         )
         machine.wait_until_succeeds(
             f"curl -sf -X POST http://127.0.0.1:{GRAFANA_PORT}/api/annotations "
@@ -134,10 +66,10 @@ pkgs.testers.runNixOSTest {
         )
         machine.succeed(f"echo '[]' > {ANNOTS_FILE}")
 
-    with subtest("Start annotation service pointing at mock servers"):
+    with subtest("Start annotation service"):
         machine.succeed(
             f"systemd-run --unit=annotations-svc "
-            f"--setenv=JELLYFIN_URL=http://127.0.0.1:{JELLYFIN_PORT} "
+            f"--setenv=JELLYFIN_URL=http://127.0.0.1:8096 "
             f"--setenv=GRAFANA_URL=http://127.0.0.1:{GRAFANA_PORT} "
             f"--setenv=CREDENTIALS_DIRECTORY={CREDS_DIR} "
             f"--setenv=STATE_FILE={STATE_FILE} "
@@ -146,38 +78,24 @@ pkgs.testers.runNixOSTest {
         )
         time.sleep(2)
 
-    with subtest("No annotations pushed when no streams active"):
+    with subtest("No annotations when no streams active"):
         time.sleep(4)
         annots = read_annotations()
         assert annots == [], f"Expected no annotations, got: {annots}"
 
-    with subtest("Annotation created when stream starts"):
-        rich_session = json.dumps([{
-            "Id": "sess-1",
-            "UserName": "alice",
-            "Client": "Infuse",
-            "DeviceName": "iPhone",
-            "PlayState": {"PlayMethod": "Transcode"},
-            "NowPlayingItem": {
-                "Name": "Inception",
-                "Type": "Movie",
-                "Bitrate": 20000000,
-                "MediaStreams": [
-                    {"Type": "Video", "Codec": "h264", "Width": 1920, "Height": 1080},
-                    {"Type": "Audio", "Codec": "dts", "Channels": 6, "IsDefault": True},
-                ],
-            },
-            "TranscodingInfo": {
-                "IsVideoDirect": True,
-                "IsAudioDirect": False,
-                "VideoCodec": "h264",
-                "AudioCodec": "aac",
-                "AudioChannels": 2,
-                "Bitrate": 8000000,
-                "TranscodeReasons": ["AudioCodecNotSupported"],
-            },
-        }])
-        machine.succeed(f"echo {repr(rich_session)} > {SESSIONS_FILE}")
+    with subtest("Annotation created when playback starts"):
+        playback_start = json.dumps({
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-1",
+            "CanSeek": True,
+            "IsPaused": False,
+        })
+        machine.succeed(
+            f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing' "
+            f"-d '{playback_start}' -H 'Content-Type:application/json' "
+            f"-H 'X-Emby-Authorization:{auth_header}, Token={token}'"
+        )
         machine.wait_until_succeeds(
             f"cat {ANNOTS_FILE} | python3 -c \"import sys,json; a=json.load(sys.stdin); exit(0 if a else 1)\"",
             timeout=15,
@@ -185,18 +103,24 @@ pkgs.testers.runNixOSTest {
         annots = read_annotations()
         assert len(annots) == 1, f"Expected 1 annotation, got: {annots}"
         text = annots[0]["text"]
-        assert "alice: Inception (movie)" in text, f"Missing title in: {text}"
-        assert "Transcode" in text, f"Missing method in: {text}"
-        assert "H.264" in text, f"Missing video codec in: {text}"
-        assert "DTS" in text and "AAC" in text, f"Missing audio codec in: {text}"
-        assert "8.0 Mbps" in text, f"Missing bitrate in: {text}"
-        assert "AudioCodecNotSupported" in text, f"Missing transcode reason in: {text}"
-        assert "Infuse" in text and "iPhone" in text, f"Missing client in: {text}"
         assert "jellyfin" in annots[0].get("tags", []), f"Missing jellyfin tag: {annots[0]}"
+        assert "Test Movie" in text, f"Missing title in: {text}"
+        assert "Infuse" in text, f"Missing client in: {text}"
+        assert "iPhone" in text, f"Missing device in: {text}"
         assert "timeEnd" not in annots[0], f"timeEnd should not be set yet: {annots[0]}"
 
-    with subtest("Annotation closed when stream ends"):
-        machine.succeed(f"echo '[]' > {SESSIONS_FILE}")
+    with subtest("Annotation closed when playback stops"):
+        playback_stop = json.dumps({
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-1",
+            "PositionTicks": 50000000,
+        })
+        machine.succeed(
+            f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing/Stopped' "
+            f"-d '{playback_stop}' -H 'Content-Type:application/json' "
+            f"-H 'X-Emby-Authorization:{auth_header}, Token={token}'"
+        )
         machine.wait_until_succeeds(
             f"cat {ANNOTS_FILE} | python3 -c \"import sys,json; a=json.load(sys.stdin); exit(0 if a and 'timeEnd' in a[0] else 1)\"",
             timeout=15,
@@ -208,11 +132,37 @@ pkgs.testers.runNixOSTest {
 
     with subtest("Multiple concurrent streams each get their own annotation"):
         machine.succeed(f"echo '[]' > {ANNOTS_FILE}")
+
+        auth_result2 = json.loads(machine.succeed(
+            f"curl -sf -X POST 'http://localhost:8096/Users/AuthenticateByName' "
+            f"-d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' "
+            f"-H 'X-Emby-Authorization:{auth_header2}'"
+        ))
+        token2 = auth_result2["AccessToken"]
+
+        playback1 = json.dumps({
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-multi-1",
+            "CanSeek": True,
+            "IsPaused": False,
+        })
         machine.succeed(
-            f"""echo '[
-              {{"Id":"sess-2","UserName":"bob","NowPlayingItem":{{"Name":"Breaking Bad","SeriesName":"Breaking Bad","ParentIndexNumber":1,"IndexNumber":1}}}},
-              {{"Id":"sess-3","UserName":"carol","NowPlayingItem":{{"Name":"Inception","Type":"Movie"}}}}
-            ]' > {SESSIONS_FILE}"""
+            f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing' "
+            f"-d '{playback1}' -H 'Content-Type:application/json' "
+            f"-H 'X-Emby-Authorization:{auth_header}, Token={token}'"
+        )
+        playback2 = json.dumps({
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-multi-2",
+            "CanSeek": True,
+            "IsPaused": False,
+        })
+        machine.succeed(
+            f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing' "
+            f"-d '{playback2}' -H 'Content-Type:application/json' "
+            f"-H 'X-Emby-Authorization:{auth_header2}, Token={token2}'"
         )
         machine.wait_until_succeeds(
             f"cat {ANNOTS_FILE} | python3 -c \"import sys,json; a=json.load(sys.stdin); exit(0 if len(a)==2 else 1)\"",
@@ -220,16 +170,13 @@ pkgs.testers.runNixOSTest {
         )
         annots = read_annotations()
         assert len(annots) == 2, f"Expected 2 annotations, got: {annots}"
-        texts = sorted(a["text"] for a in annots)
-        assert any("Breaking Bad" in t and "S01E01" in t for t in texts), f"Missing Bob's annotation: {texts}"
-        assert any("carol" in t and "Inception" in t for t in texts), f"Missing Carol's annotation: {texts}"
 
     with subtest("State survives service restart (no duplicate annotations)"):
         machine.succeed("systemctl stop annotations-svc || true")
         time.sleep(1)
         machine.succeed(
             f"systemd-run --unit=annotations-svc-2 "
-            f"--setenv=JELLYFIN_URL=http://127.0.0.1:{JELLYFIN_PORT} "
+            f"--setenv=JELLYFIN_URL=http://127.0.0.1:8096 "
             f"--setenv=GRAFANA_URL=http://127.0.0.1:{GRAFANA_PORT} "
             f"--setenv=CREDENTIALS_DIRECTORY={CREDS_DIR} "
             f"--setenv=STATE_FILE={STATE_FILE} "

tests/jellyfin-qbittorrent-monitor.nix

@@ -5,10 +5,7 @@
   ...
 }:
 let
-  payloads = {
-    auth = pkgs.writeText "auth.json" (builtins.toJSON { Username = "jellyfin"; });
-    empty = pkgs.writeText "empty.json" (builtins.toJSON { });
-  };
+  jfLib = import ./jellyfin-test-lib.nix { inherit pkgs lib; };
 in
 pkgs.testers.runNixOSTest {
   name = "jellyfin-qbittorrent-monitor";
@@ -18,11 +15,10 @@ pkgs.testers.runNixOSTest {
       { ... }:
       {
         imports = [
+          jfLib.jellyfinTestConfig
           inputs.vpn-confinement.nixosModules.default
         ];
 
-        services.jellyfin.enable = true;
-
         # Real qBittorrent service
         services.qbittorrent = {
           enable = true;
@@ -56,11 +52,6 @@ pkgs.testers.runNixOSTest {
           };
         };
 
-        environment.systemPackages = with pkgs; [
-          curl
-          ffmpeg
-        ];
-        virtualisation.diskSize = 3 * 1024;
         networking.firewall.allowedTCPPorts = [
           8096
           8080
@@ -106,20 +97,17 @@ pkgs.testers.runNixOSTest {
   testScript = ''
     import json
     import time
-    from urllib.parse import urlencode
+
+    import importlib.util
+    _spec = importlib.util.spec_from_file_location("jf_helpers", "${jfLib.helpers}")
+    assert _spec and _spec.loader
+    _jf = importlib.util.module_from_spec(_spec)
+    _spec.loader.exec_module(_jf)
+    setup_jellyfin = _jf.setup_jellyfin
+    jellyfin_api = _jf.jellyfin_api
 
     auth_header = 'MediaBrowser Client="NixOS Test", DeviceId="test-1337", Device="TestDevice", Version="1.0"'
 
-    def api_get(path, token=None):
-        header = auth_header + (f", Token={token}" if token else "")
-        return f"curl -sf 'http://server:8096{path}' -H 'X-Emby-Authorization:{header}'"
-
-    def api_post(path, json_file=None, token=None):
-        header = auth_header + (f", Token={token}" if token else "")
-        if json_file:
-            return f"curl -sf -X POST 'http://server:8096{path}' -d '@{json_file}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{header}'"
-        return f"curl -sf -X POST 'http://server:8096{path}' -H 'X-Emby-Authorization:{header}'"
-
     def is_throttled():
         return server.succeed("curl -s http://localhost:8080/api/v2/transfer/speedLimitsMode").strip() == "1"
 
@@ -137,57 +125,15 @@ pkgs.testers.runNixOSTest {
             return False
         return all(t["state"].startswith("stopped") for t in torrents)
 
-    movie_id: str = ""
-    media_source_id: str = ""
-
     start_all()
-    server.wait_for_unit("jellyfin.service")
-    server.wait_for_open_port(8096)
-    server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
     server.wait_for_unit("qbittorrent.service")
     server.wait_for_open_port(8080)
-
-    # Wait for qBittorrent WebUI to be responsive
     server.wait_until_succeeds("curl -sf http://localhost:8080/api/v2/app/version", timeout=30)
 
-    with subtest("Complete Jellyfin setup wizard"):
-        server.wait_until_succeeds(api_get("/Startup/Configuration"))
-        server.succeed(api_get("/Startup/FirstUser"))
-        server.succeed(api_post("/Startup/Complete"))
-
-    with subtest("Authenticate and get token"):
-        auth_result = json.loads(server.succeed(api_post("/Users/AuthenticateByName", "${payloads.auth}")))
-        token = auth_result["AccessToken"]
-        user_id = auth_result["User"]["Id"]
-
-    with subtest("Create test video library"):
-        tempdir = server.succeed("mktemp -d -p /var/lib/jellyfin").strip()
-        server.succeed(f"chmod 755 '{tempdir}'")
-        server.succeed(f"ffmpeg -f lavfi -i testsrc2=duration=5 '{tempdir}/Test Movie (2024) [1080p].mkv'")
-
-        add_folder_query = urlencode({
-            "name": "Test Library",
-            "collectionType": "Movies",
-            "paths": tempdir,
-            "refreshLibrary": "true",
-        })
-        server.succeed(api_post(f"/Library/VirtualFolders?{add_folder_query}", "${payloads.empty}", token))
-
-        def is_library_ready(_):
-            folders = json.loads(server.succeed(api_get("/Library/VirtualFolders", token)))
-            return all(f.get("RefreshStatus") == "Idle" for f in folders)
-        retry(is_library_ready, timeout=60)
-
-        def get_movie(_):
-            global movie_id, media_source_id
-            items = json.loads(server.succeed(api_get(f"/Users/{user_id}/Items?IncludeItemTypes=Movie&Recursive=true", token)))
-            if items["TotalRecordCount"] > 0:
-                movie_id = items["Items"][0]["Id"]
-                item_info = json.loads(server.succeed(api_get(f"/Users/{user_id}/Items/{movie_id}", token)))
-                media_source_id = item_info["MediaSources"][0]["Id"]
-                return True
-            return False
-        retry(get_movie, timeout=60)
+    token, user_id, movie_id, media_source_id = setup_jellyfin(
+        server, retry, auth_header,
+        "${jfLib.payloads.auth}", "${jfLib.payloads.empty}",
+    )
 
     with subtest("Start monitor service"):
         python = "${pkgs.python3.withPackages (ps: [ ps.requests ])}/bin/python"
@@ -214,12 +160,12 @@ pkgs.testers.runNixOSTest {
     server_ip = "192.168.1.1"
 
     with subtest("Client authenticates from external network"):
-        auth_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+        auth_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
         client_auth_result = json.loads(client.succeed(auth_cmd))
         client_token = client_auth_result["AccessToken"]
 
     with subtest("Second client authenticates from external network"):
-        auth_cmd2 = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+        auth_cmd2 = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
         client_auth_result2 = json.loads(client.succeed(auth_cmd2))
         client_token2 = client_auth_result2["AccessToken"]
 
@@ -430,7 +376,7 @@ pkgs.testers.runNixOSTest {
     with subtest("Local playback does NOT trigger throttling"):
         local_auth = 'MediaBrowser Client="Local Client", DeviceId="local-1111", Device="LocalDevice", Version="1.0"'
         local_auth_result = json.loads(server.succeed(
-            f"curl -sf -X POST 'http://localhost:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{local_auth}'"
+            f"curl -sf -X POST 'http://localhost:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{local_auth}'"
         ))
         local_token = local_auth_result["AccessToken"]
 
@@ -527,11 +473,11 @@ pkgs.testers.runNixOSTest {
 
         # Re-authenticate (old token invalid after restart)
         client_auth_result = json.loads(client.succeed(
-            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
         ))
         client_token = client_auth_result["AccessToken"]
         client_auth_result2 = json.loads(client.succeed(
-            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
         ))
         client_token2 = client_auth_result2["AccessToken"]
 
@@ -542,11 +488,11 @@ pkgs.testers.runNixOSTest {
     with subtest("Monitor recovers after Jellyfin temporary unavailability"):
         # Re-authenticate with fresh token
         client_auth_result = json.loads(client.succeed(
-            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
         ))
         client_token = client_auth_result["AccessToken"]
         client_auth_result2 = json.loads(client.succeed(
-            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${jfLib.payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
         ))
         client_token2 = client_auth_result2["AccessToken"]
 

tests/jellyfin-test-lib.nix

@@ -0,0 +1,20 @@
+{ pkgs, lib }:
+{
+  payloads = {
+    auth = pkgs.writeText "auth.json" (builtins.toJSON { Username = "jellyfin"; });
+    empty = pkgs.writeText "empty.json" (builtins.toJSON { });
+  };
+
+  helpers = ./jellyfin-test-lib.py;
+
+  jellyfinTestConfig =
+    { pkgs, ... }:
+    {
+      services.jellyfin.enable = true;
+      environment.systemPackages = with pkgs; [
+        curl
+        ffmpeg
+      ];
+      virtualisation.diskSize = lib.mkDefault (3 * 1024);
+    };
+}

tests/jellyfin-test-lib.py

@@ -0,0 +1,90 @@
+import json
+from urllib.parse import urlencode
+
+
+def jellyfin_api(machine, method, path, auth_header, token=None, data_file=None, data=None):
+    hdr = auth_header + (f", Token={token}" if token else "")
+    cmd = f"curl -sf -X {method} 'http://localhost:8096{path}'"
+    if data_file:
+        cmd += f" -d '@{data_file}' -H 'Content-Type:application/json'"
+    elif data:
+        payload = json.dumps(data) if isinstance(data, dict) else data
+        cmd += f" -d '{payload}' -H 'Content-Type:application/json'"
+    cmd += f" -H 'X-Emby-Authorization:{hdr}'"
+    return machine.succeed(cmd)
+
+
+def setup_jellyfin(machine, retry, auth_header, auth_payload, empty_payload):
+    machine.wait_for_unit("jellyfin.service")
+    machine.wait_for_open_port(8096)
+    machine.wait_until_succeeds(
+        "curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60
+    )
+
+    machine.wait_until_succeeds(
+        f"curl -sf 'http://localhost:8096/Startup/Configuration' "
+        f"-H 'X-Emby-Authorization:{auth_header}'"
+    )
+    jellyfin_api(machine, "GET", "/Startup/FirstUser", auth_header)
+    jellyfin_api(machine, "POST", "/Startup/Complete", auth_header)
+
+    result = json.loads(
+        jellyfin_api(
+            machine, "POST", "/Users/AuthenticateByName",
+            auth_header, data_file=auth_payload,
+        )
+    )
+    token = result["AccessToken"]
+    user_id = result["User"]["Id"]
+
+    tempdir = machine.succeed("mktemp -d -p /var/lib/jellyfin").strip()
+    machine.succeed(f"chmod 755 '{tempdir}'")
+    machine.succeed(
+        f"ffmpeg -f lavfi -i testsrc2=duration=5 -f lavfi -i sine=frequency=440:duration=5 "
+        f"-c:v libx264 -c:a aac '{tempdir}/Test Movie (2024).mkv'"
+    )
+
+    query = urlencode({
+        "name": "Test Library",
+        "collectionType": "Movies",
+        "paths": tempdir,
+        "refreshLibrary": "true",
+    })
+    jellyfin_api(
+        machine, "POST", f"/Library/VirtualFolders?{query}",
+        auth_header, token=token, data_file=empty_payload,
+    )
+
+    def is_ready(_):
+        folders = json.loads(
+            jellyfin_api(machine, "GET", "/Library/VirtualFolders", auth_header, token=token)
+        )
+        return all(f.get("RefreshStatus") == "Idle" for f in folders)
+    retry(is_ready, timeout=60)
+
+    movie_id = None
+    media_source_id = None
+
+    def get_movie(_):
+        nonlocal movie_id, media_source_id
+        items = json.loads(
+            jellyfin_api(
+                machine, "GET",
+                f"/Users/{user_id}/Items?IncludeItemTypes=Movie&Recursive=true",
+                auth_header, token=token,
+            )
+        )
+        if items["TotalRecordCount"] > 0:
+            movie_id = items["Items"][0]["Id"]
+            info = json.loads(
+                jellyfin_api(
+                    machine, "GET", f"/Users/{user_id}/Items/{movie_id}",
+                    auth_header, token=token,
+                )
+            )
+            media_source_id = info["MediaSources"][0]["Id"]
+            return True
+        return False
+    retry(get_movie, timeout=60)
+
+    return token, user_id, movie_id, media_source_id

tests/mock-grafana-server.py

@@ -0,0 +1,58 @@
+import http.server, json, sys
+
+PORT = int(sys.argv[1])
+DATA_FILE = sys.argv[2]
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def log_message(self, fmt, *args):
+        pass
+
+    def _read_body(self):
+        length = int(self.headers.get("Content-Length", 0))
+        return json.loads(self.rfile.read(length)) if length else {}
+
+    def _json(self, code, body):
+        data = json.dumps(body).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_POST(self):
+        if self.path == "/api/annotations":
+            body = self._read_body()
+            try:
+                with open(DATA_FILE) as f:
+                    annotations = json.load(f)
+            except Exception:
+                annotations = []
+            aid = len(annotations) + 1
+            body["id"] = aid
+            annotations.append(body)
+            with open(DATA_FILE, "w") as f:
+                json.dump(annotations, f)
+            self._json(200, {"id": aid, "message": "Annotation added"})
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def do_PATCH(self):
+        if self.path.startswith("/api/annotations/"):
+            aid = int(self.path.rsplit("/", 1)[-1])
+            body = self._read_body()
+            try:
+                with open(DATA_FILE) as f:
+                    annotations = json.load(f)
+            except Exception:
+                annotations = []
+            for a in annotations:
+                if a["id"] == aid:
+                    a.update(body)
+            with open(DATA_FILE, "w") as f:
+                json.dump(annotations, f)
+            self._json(200, {"message": "Annotation patched"})
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+http.server.HTTPServer(("127.0.0.1", PORT), Handler).serve_forever()

tests/tests.nix

@@ -25,6 +25,9 @@ in
   # jellyfin annotation service test
   jellyfinAnnotationsTest = handleTest ./jellyfin-annotations.nix;
 
+  # zfs scrub annotations test
+  zfsScrubAnnotationsTest = handleTest ./zfs-scrub-annotations.nix;
+
   # ntfy alerts test
   ntfyAlertsTest = handleTest ./ntfy-alerts.nix;
 

tests/zfs-scrub-annotations.nix

@@ -0,0 +1,123 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mockServer = ./mock-grafana-server.py;
+
+  mockZpool = pkgs.writeShellScript "zpool" ''
+    case "$1" in
+      list)
+        echo "tank"
+        echo "hdds"
+        ;;
+      status)
+        pool="$2"
+        if [ "$pool" = "tank" ]; then
+          echo "  scan: scrub repaired 0B in 00:24:39 with 0 errors on Mon Jan  1 02:24:39 2024"
+        elif [ "$pool" = "hdds" ]; then
+          echo "  scan: scrub repaired 0B in 04:12:33 with 0 errors on Mon Jan  1 06:12:33 2024"
+        fi
+        ;;
+    esac
+  '';
+
+  script = ../services/zfs-scrub-annotations.sh;
+  python = pkgs.python3;
+in
+pkgs.testers.runNixOSTest {
+  name = "zfs-scrub-annotations";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      environment.systemPackages = with pkgs; [
+        python3
+        curl
+        jq
+      ];
+    };
+
+  testScript = ''
+    import json
+
+    GRAFANA_PORT = 13000
+    ANNOTS_FILE  = "/tmp/annotations.json"
+    STATE_DIR    = "/tmp/scrub-state"
+    PYTHON       = "${python}/bin/python3"
+    MOCK         = "${mockServer}"
+    SCRIPT       = "${script}"
+    MOCK_ZPOOL   = "${mockZpool}"
+
+    MOCK_BIN = "/tmp/mock-bin"
+    ENV_PREFIX = (
+        f"GRAFANA_URL=http://127.0.0.1:{GRAFANA_PORT} "
+        f"STATE_DIR={STATE_DIR} "
+        f"PATH={MOCK_BIN}:$PATH "
+    )
+
+    def read_annotations():
+        out = machine.succeed(f"cat {ANNOTS_FILE} 2>/dev/null || echo '[]'")
+        return json.loads(out.strip())
+
+    start_all()
+    machine.wait_for_unit("multi-user.target")
+
+    with subtest("Setup state directory and mock zpool"):
+        machine.succeed(f"mkdir -p {STATE_DIR}")
+        machine.succeed(f"mkdir -p {MOCK_BIN} && cp {MOCK_ZPOOL} {MOCK_BIN}/zpool && chmod +x {MOCK_BIN}/zpool")
+
+    with subtest("Start mock Grafana server"):
+        machine.succeed(f"echo '[]' > {ANNOTS_FILE}")
+        machine.succeed(
+            f"systemd-run --unit=mock-grafana {PYTHON} {MOCK} {GRAFANA_PORT} {ANNOTS_FILE}"
+        )
+        machine.wait_until_succeeds(
+            f"curl -sf -X POST http://127.0.0.1:{GRAFANA_PORT}/api/annotations "
+            f"-H 'Content-Type: application/json' -d '{{\"text\":\"ping\",\"tags\":[]}}' | grep -q id",
+            timeout=10,
+        )
+        machine.succeed(f"echo '[]' > {ANNOTS_FILE}")
+
+    with subtest("Start action creates annotation with pool names and zfs-scrub tag"):
+        machine.succeed(f"{ENV_PREFIX} bash {SCRIPT} start")
+        annots = read_annotations()
+        assert len(annots) == 1, f"Expected 1 annotation, got: {annots}"
+        assert "zfs-scrub" in annots[0].get("tags", []), f"Missing zfs-scrub tag: {annots[0]}"
+        assert "tank" in annots[0]["text"], f"Missing tank in text: {annots[0]['text']}"
+        assert "hdds" in annots[0]["text"], f"Missing hdds in text: {annots[0]['text']}"
+        assert "time" in annots[0], f"Missing time field: {annots[0]}"
+        assert "timeEnd" not in annots[0], f"timeEnd should not be set yet: {annots[0]}"
+
+    with subtest("State file contains annotation ID"):
+        ann_id = machine.succeed(f"cat {STATE_DIR}/annotation-id").strip()
+        assert ann_id == "1", f"Expected annotation ID 1, got: {ann_id}"
+
+    with subtest("Stop action closes annotation with per-pool scrub results"):
+        machine.succeed(f"{ENV_PREFIX} bash {SCRIPT} stop")
+        annots = read_annotations()
+        assert len(annots) == 1, f"Expected 1 annotation, got: {annots}"
+        assert "timeEnd" in annots[0], f"timeEnd should be set: {annots[0]}"
+        assert annots[0]["timeEnd"] > annots[0]["time"], "timeEnd should be after time"
+        text = annots[0]["text"]
+        assert "ZFS scrub completed" in text, f"Missing completed text: {text}"
+        assert "tank:" in text, f"Missing tank results: {text}"
+        assert "hdds:" in text, f"Missing hdds results: {text}"
+        assert "00:24:39" in text, f"Missing tank scrub duration: {text}"
+        assert "04:12:33" in text, f"Missing hdds scrub duration: {text}"
+
+    with subtest("State file cleaned up after stop"):
+        machine.fail(f"test -f {STATE_DIR}/annotation-id")
+
+    with subtest("Stop action handles missing state file gracefully"):
+        machine.succeed(f"{ENV_PREFIX} bash {SCRIPT} stop")
+        annots = read_annotations()
+        assert len(annots) == 1, f"Expected no new annotations, got: {annots}"
+
+    with subtest("Start action handles Grafana being down gracefully"):
+        machine.succeed("systemctl stop mock-grafana")
+        machine.succeed(f"{ENV_PREFIX} bash {SCRIPT} start")
+        machine.fail(f"test -f {STATE_DIR}/annotation-id")
+  '';
+}