diff --git a/services/arr/torrent-audit.py b/services/arr/torrent-audit.py index 8f15a2a..70adec1 100644 --- a/services/arr/torrent-audit.py +++ b/services/arr/torrent-audit.py @@ -74,6 +74,26 @@ def gib(size_bytes: int) -> str: return f"{size_bytes / 1073741824:.1f}" +def _is_keeper_viable( + keeper_hash: str, + all_qbit: dict[str, dict], + file_path: str | None, +) -> bool: + """True iff the keeper (newer import) is actually viable. + + A keeper is viable when its data is reachable: either the keeper + torrent is complete in qBittorrent, or (when no torrent exists) the + file is present on the filesystem. + """ + keeper_torrent = all_qbit.get(keeper_hash) + if keeper_torrent is not None: + return is_complete(keeper_torrent) + # Keeper hash not in qBittorrent -- trust Radarr's file mapping + # only if the file actually exists on disk. + if file_path: + return os.path.exists(file_path) + return False + # --------------------------------------------------------------------------- # Collect all known hashes from *arr history + queue # --------------------------------------------------------------------------- @@ -107,7 +127,7 @@ def find_unmanaged(qbit_torrents: dict, known_hashes: set) -> list[dict]: # --------------------------------------------------------------------------- -def find_movie_abandoned(radarr, qbit_movies): +def find_movie_abandoned(radarr, qbit_movies, all_qbit=None): log.info("Analysing Radarr import history ...") imports_by_movie = defaultdict(list) for rec in paginate(radarr, "history"): @@ -126,11 +146,14 @@ def find_movie_abandoned(radarr, qbit_movies): # Identify keeper (latest) and abandoned (older) hashes per movie. abandoned_hashes: set[str] = set() keeper_hashes: set[str] = set() - hash_to_movie: dict[str, int] = {} + hash_to_movie: dict[str, int] = {} # abandoned hash -> movieId + movie_to_keeper: dict[int, str] = {} # movieId -> keeper hash for mid, events in imports_by_movie.items(): ordered = sorted(events, key=lambda e: e["date"]) - keeper_hashes.add(ordered[-1]["downloadId"]) + keeper = ordered[-1]["downloadId"] + keeper_hashes.add(keeper) + movie_to_keeper[mid] = keeper for e in ordered[:-1]: abandoned_hashes.add(e["downloadId"]) hash_to_movie[e["downloadId"]] = mid @@ -152,7 +175,6 @@ def find_movie_abandoned(radarr, qbit_movies): # re-download in progress. if not is_complete(torrent): continue - mid = hash_to_movie.get(ahash) movie = radarr_movies.get(mid) if mid else None mf = (movie or {}).get("movieFile") or {} @@ -163,16 +185,27 @@ def find_movie_abandoned(radarr, qbit_movies): status = "SAFE" notes = [] + # Verify the keeper (newer import) is actually viable before + # declaring the abandoned torrent safe to delete. + keeper_hash = movie_to_keeper.get(mid) if mid else None + if keeper_hash and all_qbit is not None: + file_path = mf.get("path") + if not _is_keeper_viable(keeper_hash, all_qbit, file_path): + notes.append( + "keeper torrent incomplete or missing, " + "file not on disk" + ) + status = "REVIEW" + if not movie or not movie.get("hasFile"): notes.append("movie removed or has no file in Radarr") status = "REVIEW" - elif torrent["size"] > current_size * 1.05: + if current_size > 0 and torrent["size"] > current_size * 1.05: notes.append( f"abandoned is larger than current " f"({gib(torrent['size'])} > {gib(current_size)} GiB)" ) status = "REVIEW" - results.append( { "name": torrent["name"], @@ -195,7 +228,7 @@ def find_movie_abandoned(radarr, qbit_movies): # --------------------------------------------------------------------------- -def find_tv_abandoned(sonarr, qbit_tvshows): +def find_tv_abandoned(sonarr, qbit_tvshows, all_qbit=None): log.info("Analysing Sonarr import history ...") episode_imports = defaultdict(list) all_download_ids: set[str] = set() @@ -222,6 +255,15 @@ def find_tv_abandoned(sonarr, qbit_tvshows): abandoned_hashes = all_download_ids - active_hashes + # For keeper-viability checks: find the active hash for each + # episode an abandoned hash once covered. + hash_to_active: dict[str, list[str]] = defaultdict(list) + for eid, events in episode_imports.items(): + active = max(events, key=lambda e: e["date"])["downloadId"] + for e in events: + if e["downloadId"] in abandoned_hashes: + hash_to_active[e["downloadId"]].append(active) + log.info("Fetching Sonarr current series state ...") current_series = {s["id"] for s in sonarr.get_series()} @@ -235,6 +277,17 @@ def find_tv_abandoned(sonarr, qbit_tvshows): status = "SAFE" notes = [] + + # Verify every keeper that replaced this hash is viable. + if all_qbit is not None: + for keeper_h in hash_to_active.get(ahash, []): + if not _is_keeper_viable(keeper_h, all_qbit, None): + notes.append( + "replacement torrent incomplete or missing" + ) + status = "REVIEW" + break + sid = hash_to_series.get(ahash) if sid and sid not in current_series: notes.append("series removed from Sonarr") @@ -369,17 +422,23 @@ def main(): print(f"--- {cat} ({len(unmanaged)} unmanaged / {len(qbit_torrents[cat])} total) ---\n") print_section(unmanaged) + # Flatten all qBittorrent torrents into one lookup for keeper + # viability checks across categories. + all_qbit = {} + for torrents in qbit_torrents.values(): + all_qbit.update(torrents) + # -- Abandoned -- print("========== ABANDONED UPGRADE LEFTOVERS ==========\n") movie_abandoned = find_movie_abandoned( - radarr, qbit_torrents.get("movies", {}) + radarr, qbit_torrents.get("movies", {}), all_qbit ) print(f"--- movies ({len(movie_abandoned)} abandoned) ---\n") print_section(movie_abandoned, show_status=True) tv_abandoned = find_tv_abandoned( - sonarr, qbit_torrents.get("tvshows", {}) + sonarr, qbit_torrents.get("tvshows", {}), all_qbit ) print(f"--- tvshows ({len(tv_abandoned)} abandoned) ---\n") print_section(tv_abandoned, show_status=True) diff --git a/tests/torrent-audit.nix b/tests/torrent-audit.nix index ffe4fbb..fac511d 100644 --- a/tests/torrent-audit.nix +++ b/tests/torrent-audit.nix @@ -53,6 +53,10 @@ let SINGLE8_NEW = "A" * 38 + "0D" # movieId=8, newer import → keeper (not in qBit) QUEUED_MOV = "A" * 38 + "0E" # in Radarr queue, not in history INPROGRESS_MOV = "A" * 38 + "0F" # movieId=10, older import, currently re-downloading + KEEPER_GONE_OLD = "A" * 38 + "11" # movieId=11, older import, keeper incomplete in qBit + KEEPER_GONE_NEW = "A" * 38 + "12" # movieId=11, newer import (keeper), incomplete in qBit + INPROGRESS_MOV_NEW = "A" * 38 + "10" # movieId=10, newer import (not in qBit) + # TV UNMANAGED_TV = "B" * 38 + "01" @@ -65,7 +69,9 @@ let REMOVED_TV_NEW = "B" * 38 + "08" # episodeId=400, newer import (not in qBit) INPROGRESS_TV = "B" * 38 + "09" # episodeId=500, older import, currently re-downloading INPROGRESS_TV_NEW = "B" * 38 + "0A" # episodeId=500, newer import (not in qBit) - INPROGRESS_MOV_NEW = "A" * 38 + "10" # movieId=10, newer import (not in qBit) + KEEPER_GONE_TV = "B" * 38 + "0B" # episodeId=600, older import, keeper incomplete in qBit + KEEPER_GONE_TV_NEW = "B" * 38 + "0C" # episodeId=600, newer import (active), incomplete in qBit + def make_torrent(h, name, size, added_on, state="uploading", progress=1.0): return { @@ -92,6 +98,10 @@ let # In-progress re-download: hash matches an old import, but data is # not yet on disk. Must NOT be flagged as abandoned (regression). make_torrent(INPROGRESS_MOV, "InProgress.Movie.2024", 8_000_000_000, 1704067209, state="downloading", progress=0.05), + # Keeper-incomplete regression: old torrent is complete, keeper is + # incomplete in qBittorrent. Must be REVIEW, not SAFE. + make_torrent(KEEPER_GONE_OLD, "KeeperGone.Movie.2024", 2_000_000_000, 1704067210), + make_torrent(KEEPER_GONE_NEW, "KeeperGone.Movie.2024.Upgr", 3_000_000_000, 1704067211, state="downloading", progress=0.10), ], "tvshows": [ make_torrent(UNMANAGED_TV, "Unmanaged.Show.S01E01", 1_000_000_000, 1704067200), @@ -101,6 +111,9 @@ let make_torrent(SEASON_PACK, "Season.Pack.S02", 5_000_000_000, 1704067204), make_torrent(REMOVED_TV, "Removed.Show.S01E01", 900_000_000, 1704067205), make_torrent(INPROGRESS_TV, "InProgress.Show.S01E01", 1_500_000_000, 1704067209, state="downloading", progress=0.05), + # Keeper-incomplete regression for TV + make_torrent(KEEPER_GONE_TV, "KeeperGone.Show.S01E01", 500_000_000, 1704067210), + make_torrent(KEEPER_GONE_TV_NEW, "KeeperGone.Show.S01E01.Upgr", 800_000_000, 1704067211, state="downloading", progress=0.10), ], } @@ -127,6 +140,10 @@ let # In-progress re-download regression case for movies {"movieId": 10, "downloadId": INPROGRESS_MOV, "eventType": "downloadFolderImported", "date": "2024-01-01T00:00:00Z"}, {"movieId": 10, "downloadId": INPROGRESS_MOV_NEW,"eventType": "downloadFolderImported", "date": "2024-06-01T00:00:00Z"}, + # Keeper-incomplete regression: old import + newer import (keeper) that + # is incomplete in qBittorrent. The old torrent must be REVIEW. + {"movieId": 11, "downloadId": KEEPER_GONE_OLD, "eventType": "downloadFolderImported", "date": "2024-01-01T00:00:00Z"}, + {"movieId": 11, "downloadId": KEEPER_GONE_NEW, "eventType": "downloadFolderImported", "date": "2024-06-01T00:00:00Z"}, ] RADARR_MOVIES = [ @@ -139,6 +156,7 @@ let {"id": 7, "hasFile": True, "movieFile": {"size": 4_000_000_000, "quality": {"quality": {"name": "Bluray-1080p"}}}}, {"id": 8, "hasFile": True, "movieFile": {"size": 5_000_000_000, "quality": {"quality": {"name": "Remux-1080p"}}}}, {"id": 10, "hasFile": True, "movieFile": {"size": 8_000_000_000, "quality": {"quality": {"name": "Remux-2160p"}}}}, + {"id": 11, "hasFile": True, "movieFile": {"size": 3_000_000_000, "quality": {"quality": {"name": "Remux-1080p"}}}}, ] # ── Sonarr mock data ────────────────────────────────────────────────── @@ -164,6 +182,9 @@ let # In-progress re-download regression case for TV {"episodeId": 500, "seriesId": 1, "downloadId": INPROGRESS_TV, "eventType": "downloadFolderImported", "date": "2024-01-01T00:00:00Z"}, {"episodeId": 500, "seriesId": 1, "downloadId": INPROGRESS_TV_NEW,"eventType": "downloadFolderImported", "date": "2024-06-01T00:00:00Z"}, + # Keeper-incomplete regression for TV + {"episodeId": 600, "seriesId": 1, "downloadId": KEEPER_GONE_TV, "eventType": "downloadFolderImported", "date": "2024-01-01T00:00:00Z"}, + {"episodeId": 600, "seriesId": 1, "downloadId": KEEPER_GONE_TV_NEW,"eventType": "downloadFolderImported", "date": "2024-06-01T00:00:00Z"}, ] SONARR_HISTORY_ALL = SONARR_HISTORY_PAGE1 + SONARR_HISTORY_PAGE2 @@ -335,14 +356,14 @@ pkgs.testers.runNixOSTest { with subtest("Detects unmanaged movie torrent"): assert "Unmanaged.Movie.2024" in unmanaged_section, \ "Should detect unmanaged movie" - assert "1 unmanaged / 10 total" in unmanaged_section, \ - "Should show 1 unmanaged movie out of 10" + assert "1 unmanaged / 12 total" in unmanaged_section, \ + "Should show 1 unmanaged movie out of 12" with subtest("Detects unmanaged TV torrent"): assert "Unmanaged.Show.S01E01" in unmanaged_section, \ "Should detect unmanaged TV show" - assert "1 unmanaged / 7 total" in unmanaged_section, \ - "Should show 1 unmanaged TV show out of 7" + assert "1 unmanaged / 9 total" in unmanaged_section, \ + "Should show 1 unmanaged TV show out of 9" with subtest("Empty category shows zero counts"): assert "0 unmanaged / 0 total" in unmanaged_section, \ @@ -434,15 +455,42 @@ pkgs.testers.runNixOSTest { break with subtest("Correct abandoned counts per category"): - assert "movies (3 abandoned)" in abandoned_section, \ - "Should show 3 abandoned movies" - assert "tvshows (2 abandoned)" in abandoned_section, \ - "Should show 2 abandoned TV shows" + assert "movies (4 abandoned)" in abandoned_section, \ + "Should show 4 abandoned movies" + assert "tvshows (3 abandoned)" in abandoned_section, \ + "Should show 3 abandoned TV shows" - with subtest("Correct summary totals"): - assert "ABANDONED: 5 total (2 safe to delete)" in output, \ - "Summary should show 5 total abandoned, 2 safe to delete" + assert "ABANDONED: 7 total (2 safe to delete)" in output, \ + "Summary should show 7 total abandoned, 2 safe to delete" assert "SAFE TO RECLAIM: 3.4 GiB" in output, \ "Should report 3.4 GiB reclaimable (2.8 GiB movie + 0.7 GiB TV)" + + with subtest("Keeper-incomplete movie triggers REVIEW"): + assert "KeeperGone.Movie.2024" in abandoned_section, \ + "Should detect abandoned torrent with incomplete keeper" + assert_note_near(abandoned_section, "KeeperGone.Movie.2024", "keeper torrent incomplete") + for line in abandoned_section.splitlines(): + if "KeeperGone.Movie.2024" in line: + assert "REVIEW" in line, f"Keeper-incomplete movie should be REVIEW, got: {line}" + break + + with subtest("Keeper-incomplete TV triggers REVIEW"): + assert "KeeperGone.Show.S01E01" in abandoned_section, \ + "Should detect abandoned TV torrent with incomplete keeper" + assert_note_near(abandoned_section, "KeeperGone.Show.S01E01", "replacement torrent incomplete") + for line in abandoned_section.splitlines(): + if "KeeperGone.Show.S01E01" in line: + assert "REVIEW" in line, f"Keeper-incomplete TV should be REVIEW, got: {line}" + break + + with subtest("Keeper-incomplete does not affect existing SAFE verdicts"): + for line in abandoned_section.splitlines(): + if "Old.Movie.Quality.2024" in line: + assert "SAFE" in line, f"Old movie should still be SAFE, got: {line}" + break + for line in abandoned_section.splitlines(): + if "Old.Show.S01E01" in line: + assert "SAFE" in line, f"Old TV should still be SAFE, got: {line}" + break ''; }