From 9635aecb810a2d7d8db4f066b75a421d0a45bdf3 Mon Sep 17 00:00:00 2001
From: Simon Gardling <titaniumtown@proton.me>
Date: Thu, 16 Apr 2026 16:35:28 -0400
Subject: [PATCH] test: add permanent failure test

Verifies the service enters failed state after exhausting all
StartLimitBurst retries when the API never becomes available.
Checks StartLimitIntervalSec/Burst configuration and confirms
repeated timeout messages appear in the journal.
---
 tests/default.nix           |  1 +
 tests/permanent-failure.nix | 89 +++++++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 tests/permanent-failure.nix

diff --git a/tests/default.nix b/tests/default.nix
index 5388d22..103d3ab 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -15,4 +15,5 @@
   jellyseerr = import ./jellyseerr.nix { inherit pkgs lib self; };
   naming = import ./naming.nix { inherit pkgs lib self; };
   network-namespace = import ./network-namespace.nix { inherit pkgs lib self; };
+  permanent-failure = import ./permanent-failure.nix { inherit pkgs lib self; };
 }
diff --git a/tests/permanent-failure.nix b/tests/permanent-failure.nix
new file mode 100644
index 0000000..cba490a
--- /dev/null
+++ b/tests/permanent-failure.nix
@@ -0,0 +1,89 @@
+{ pkgs, lib, self }:
+pkgs.testers.runNixOSTest {
+  name = "arr-init-permanent-failure";
+  nodes.machine = { pkgs, lib, ... }: {
+    imports = [ self.nixosModules.default ];
+    system.stateVersion = "24.11";
+    virtualisation.memorySize = 2048;
+    environment.systemPackages = with pkgs; [ curl jq gnugrep ];
+
+    # Mock that always returns 503
+    systemd.services.mock-sonarr = let
+      mockScript = pkgs.writeScript "mock-sonarr-fail.py" ''
+        from http.server import HTTPServer, BaseHTTPRequestHandler
+
+        class FailMock(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(503)
+                self.send_header("Content-Type", "text/plain")
+                self.end_headers()
+                self.wfile.write(b"Service Unavailable")
+
+            def do_POST(self):
+                self.do_GET()
+
+            def log_message(self, format, *args):
+                pass
+
+        HTTPServer(("0.0.0.0", 8989), FailMock).serve_forever()
+      '';
+    in {
+      description = "Mock Sonarr that never becomes ready";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.python3}/bin/python3 ${mockScript}";
+        Type = "simple";
+      };
+    };
+
+    # Pre-seed config.xml
+    systemd.tmpfiles.rules = [
+      "d /var/lib/mock-sonarr 0755 root root -"
+      "f /var/lib/mock-sonarr/config.xml 0644 root root - <Config><ApiKey>test-api-key-fail</ApiKey></Config>"
+    ];
+
+    services.arrInit.sonarr = {
+      enable = true;
+      serviceName = "mock-sonarr";
+      dataDir = "/var/lib/mock-sonarr";
+      port = 8989;
+      healthChecks = false;
+      # Very short timeout so retries happen fast
+      apiTimeout = 3;
+    };
+
+    # Speed up retries for test
+    systemd.services.mock-sonarr-init.serviceConfig.RestartSec = lib.mkForce 2;
+  };
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("mock-sonarr.service")
+
+    with subtest("Start limit is configured correctly"):
+        unit_content = machine.succeed("systemctl cat mock-sonarr-init.service")
+        # StartLimitIntervalSec = 5 * (3 + 30) = 165
+        assert "StartLimitIntervalSec=165" in unit_content, \
+            f"Expected StartLimitIntervalSec=165, got:\n{unit_content}"
+        assert "StartLimitBurst=5" in unit_content, \
+            f"Expected StartLimitBurst=5, got:\n{unit_content}"
+
+    with subtest("Service enters permanent failure after exhausting retries"):
+        # Wait for the start-limit to be hit: "Start request repeated too quickly" in journal
+        machine.wait_until_succeeds(
+            "journalctl -u mock-sonarr-init.service --no-pager | grep -q 'Start request repeated too quickly'",
+            timeout=120,
+        )
+
+        state = machine.succeed(
+            "systemctl show mock-sonarr-init.service --property=ActiveState | cut -d= -f2"
+        ).strip()
+        assert state == "failed", f"Expected 'failed' state, got '{state}'"
+
+    with subtest("Journal shows repeated timeout messages"):
+        journal = machine.succeed("journalctl -u mock-sonarr-init.service --no-pager")
+        # Count timeout messages - should have multiple
+        timeout_count = journal.lower().count("not available after 3 seconds")
+        assert timeout_count >= 2, \
+            f"Expected at least 2 timeout messages, got {timeout_count}. Journal:\n{journal[-1000:]}"
+  '';
+}