arr-init/tests/health-checks.nix

{
  pkgs,
  lib,
  self,
}:

pkgs.testers.runNixOSTest {
  name = "arr-init-health-checks";

  nodes.machine =
    { pkgs, lib, ... }:
    let
      mocks = import ./lib/mocks.nix { inherit pkgs; };
    in
    {
      imports = [ self.nixosModules.default ];

      system.stateVersion = "24.11";

      virtualisation.memorySize = 4096;

      environment.systemPackages = with pkgs; [
        curl
        jq
        gnugrep
      ];

      systemd.services.mock-qbittorrent = mocks.mkMockQbittorrent {
        initialCategories = {
          tv = { name = "tv"; savePath = "/downloads"; };
          movies = { name = "movies"; savePath = "/downloads"; };
        };
      };

      systemd.tmpfiles.rules = [
        "d /media/tv 0755 sonarr sonarr -"
        "d /media/movies 0755 radarr radarr -"
      ];

      services.sonarr = {
        enable = true;
        dataDir = "/var/lib/sonarr/.config/NzbDrone";
        settings.server.port = lib.mkDefault 8989;
      };

      services.radarr = {
        enable = true;
        dataDir = "/var/lib/radarr/.config/Radarr";
        settings.server.port = lib.mkDefault 7878;
      };

      services.prowlarr = {
        enable = true;
      };

      services.arrInit.sonarr = {
        enable = true;
        serviceName = "sonarr";
        dataDir = "/var/lib/sonarr/.config/NzbDrone";
        port = 8989;
        healthChecks = true;
        healthCheckRetries = 2;
        healthCheckInterval = 2;
        downloadClients = [
          {
            name = "qBittorrent";
            implementation = "QBittorrent";
            configContract = "QBittorrentSettings";
            protocol = "torrent";
            serviceName = "mock-qbittorrent";
            fields = {
              host = "127.0.0.1";
              port = 6011;
              useSsl = false;
              tvCategory = "tv";
            };
          }
        ];
        rootFolders = [ "/media/tv" ];
      };

      services.arrInit.radarr = {
        enable = true;
        serviceName = "radarr";
        dataDir = "/var/lib/radarr/.config/Radarr";
        port = 7878;
        healthChecks = true;
        healthCheckRetries = 2;
        healthCheckInterval = 2;
        downloadClients = [
          {
            name = "qBittorrent";
            implementation = "QBittorrent";
            configContract = "QBittorrentSettings";
            protocol = "torrent";
            serviceName = "mock-qbittorrent";
            fields = {
              host = "127.0.0.1";
              port = 6011;
              useSsl = false;
              movieCategory = "movies";
            };
          }
        ];
        rootFolders = [ "/media/movies" ];
      };

      services.arrInit.prowlarr = {
        enable = true;
        serviceName = "prowlarr";
        dataDir = "/var/lib/prowlarr";
        port = 9696;
        apiVersion = "v1";
        healthChecks = true;
        healthCheckRetries = 2;
        healthCheckInterval = 2;
        syncedApps = [
          {
            name = "Sonarr";
            implementation = "Sonarr";
            configContract = "SonarrSettings";
            prowlarrUrl = "http://localhost:9696";
            baseUrl = "http://localhost:8989";
            apiKeyFrom = "/var/lib/sonarr/.config/NzbDrone/config.xml";
            syncCategories = [
              5000
              5010
              5020
            ];
            serviceName = "sonarr";
          }
          {
            name = "Radarr";
            implementation = "Radarr";
            configContract = "RadarrSettings";
            prowlarrUrl = "http://localhost:9696";
            baseUrl = "http://localhost:7878";
            apiKeyFrom = "/var/lib/radarr/.config/Radarr/config.xml";
            syncCategories = [
              2000
              2010
              2020
            ];
            serviceName = "radarr";
          }
        ];
      };
    };

  testScript = ''
    start_all()

    # Wait for services to start
    machine.wait_for_unit("mock-qbittorrent.service")
    machine.wait_until_succeeds("curl -sf http://localhost:6011/api/v2/app/version", timeout=30)
    machine.wait_for_unit("sonarr.service")
    machine.wait_for_unit("radarr.service")
    machine.wait_for_unit("prowlarr.service")

    # Wait for APIs to be ready
    machine.wait_until_succeeds(
        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
        "curl -sf http://localhost:8989/api/v3/system/status -H \"X-Api-Key: $API_KEY\"",
        timeout=120,
    )
    machine.wait_until_succeeds(
        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
        "curl -sf http://localhost:7878/api/v3/system/status -H \"X-Api-Key: $API_KEY\"",
        timeout=120,
    )
    machine.wait_until_succeeds(
        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/prowlarr/config.xml) && "
        "curl -sf http://localhost:9696/api/v1/system/status -H \"X-Api-Key: $API_KEY\"",
        timeout=180,
    )

    # Restart init services to ensure they run with config.xml present
    machine.succeed("systemctl restart sonarr-init.service")
    machine.succeed("systemctl restart radarr-init.service")
    machine.wait_for_unit("sonarr-init.service")
    machine.wait_for_unit("radarr-init.service")
    machine.succeed("systemctl restart prowlarr-init.service")
    machine.wait_for_unit("prowlarr-init.service")

    with subtest("Health checks pass when download clients are reachable"):
        # Sonarr init should succeed with healthChecks enabled since mock qBittorrent is running
        exit_code = machine.succeed(
            "systemctl show sonarr-init.service --property=ExecMainStatus | cut -d= -f2"
        ).strip()
        assert exit_code == "0", f"sonarr-init should succeed when download client is reachable, got exit code {exit_code}"

        # Radarr init should also succeed
        exit_code = machine.succeed(
            "systemctl show radarr-init.service --property=ExecMainStatus | cut -d= -f2"
        ).strip()
        assert exit_code == "0", f"radarr-init should succeed when download client is reachable, got exit code {exit_code}"

    with subtest("Health checks pass for Prowlarr synced apps"):
        exit_code = machine.succeed(
            "systemctl show prowlarr-init.service --property=ExecMainStatus | cut -d= -f2"
        ).strip()
        assert exit_code == "0", f"prowlarr-init should succeed when synced apps are reachable, got exit code {exit_code}"

    with subtest("Health check logs confirm validation ran"):
        sonarr_journal = machine.succeed("journalctl -u sonarr-init.service --no-pager")
        assert "health check" in sonarr_journal.lower() or "testing" in sonarr_journal.lower(), \
            "Expected health check log messages in sonarr-init journal"

        radarr_journal = machine.succeed("journalctl -u radarr-init.service --no-pager")
        assert "health check" in radarr_journal.lower() or "testing" in radarr_journal.lower(), \
            "Expected health check log messages in radarr-init journal"

        prowlarr_journal = machine.succeed("journalctl -u prowlarr-init.service --no-pager")
        assert "health check" in prowlarr_journal.lower() or "testing" in prowlarr_journal.lower(), \
            "Expected health check log messages in prowlarr-init journal"

    with subtest("Init service stops when download client service stops (Requires dependency)"):
        # With serviceName set, sonarr-init has Requires=mock-qbittorrent.service.
        # When mock-qbittorrent stops, sonarr-init should also be pulled down —
        # no spurious restart loops or alert spam.
        machine.succeed("systemctl stop mock-qbittorrent.service")

        # sonarr-init should be inactive (pulled down by dependency)
        machine.wait_until_succeeds(
            "systemctl show sonarr-init.service --property=ActiveState | grep -q 'inactive'",
            timeout=30,
        )

        # radarr-init should also be inactive (same Requires dependency)
        machine.wait_until_succeeds(
            "systemctl show radarr-init.service --property=ActiveState | grep -q 'inactive'",
            timeout=30,
        )

    with subtest("Health check fails with retries when Prowlarr synced app is unreachable"):
        # Stop radarr to ensure synced apps are unreachable
        machine.succeed("systemctl stop sonarr.service || true")
        machine.succeed("systemctl stop radarr.service || true")

        # Restart prowlarr-init - it should FAIL because synced app connectivity test fails
        # even after retries (2 retries * 2s interval = ~4s + attempt time)
        machine.execute("systemctl restart prowlarr-init.service")

        # Wait for the service to settle into failed state
        # With retries: up to 3 attempts * (30s max-time + 2s interval) + restart delay
        machine.wait_until_succeeds(
            "systemctl show prowlarr-init.service --property=Result | grep -q 'exit-code'",
            timeout=300,
        )

        journal = machine.succeed("journalctl -u prowlarr-init.service --no-pager")
        assert "health check failed" in journal.lower(), \
            "Expected health check failure message in prowlarr-init journal, got: " + journal[-500:]
        assert "retrying" in journal.lower(), \
            "Expected retry log messages before final failure in prowlarr-init journal, got: " + journal[-500:]
  '';
}