phase 2: promote services/, tests/, patches/, lib/, scripts/

services/grafana/dashboard.nix

@@ -0,0 +1,698 @@
+{
+  ...
+}:
+let
+  promDs = {
+    type = "prometheus";
+    uid = "prometheus";
+  };
+
+  dashboard = {
+    editable = true;
+    graphTooltip = 1;
+    schemaVersion = 39;
+    tags = [
+      "system"
+      "monitoring"
+    ];
+    time = {
+      from = "now-6h";
+      to = "now";
+    };
+    timezone = "browser";
+    title = "System Overview";
+    uid = "system-overview";
+
+    annotations.list = [
+      {
+        name = "Jellyfin Streams";
+        datasource = {
+          type = "grafana";
+          uid = "-- Grafana --";
+        };
+        enable = true;
+        iconColor = "green";
+        showIn = 0;
+        type = "tags";
+        tags = [ "jellyfin" ];
+      }
+      {
+        name = "ZFS Scrubs";
+        datasource = {
+          type = "grafana";
+          uid = "-- Grafana --";
+        };
+        enable = true;
+        iconColor = "orange";
+        showIn = 0;
+        type = "tags";
+        tags = [ "zfs-scrub" ];
+      }
+      {
+        name = "LLM Requests";
+        datasource = promDs;
+        enable = true;
+        iconColor = "purple";
+        target = {
+          datasource = promDs;
+          expr = "llamacpp:requests_processing > 0";
+          instant = false;
+          range = true;
+          refId = "A";
+        };
+        titleFormat = "LLM inference";
+      }
+    ];
+
+    panels = [
+      # -- Row 1: UPS --
+      {
+        id = 1;
+        type = "timeseries";
+        title = "UPS Power Draw";
+        gridPos = {
+          h = 8;
+          w = 8;
+          x = 0;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_ups_load_percent / 100 * apcupsd_nominal_power_watts";
+            legendFormat = "Power (W)";
+            refId = "A";
+          }
+          {
+            datasource = promDs;
+            expr = "avg_over_time((apcupsd_ups_load_percent / 100 * apcupsd_nominal_power_watts + 4.5)[5m:])";
+            legendFormat = "5m average (W)";
+            refId = "B";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "watt";
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 20;
+              spanNulls = true;
+            };
+          };
+          overrides = [
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "A";
+              };
+              properties = [
+                {
+                  id = "custom.lineStyle";
+                  value = {
+                    fill = "dot";
+                  };
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 10;
+                }
+                {
+                  id = "custom.lineWidth";
+                  value = 1;
+                }
+                {
+                  id = "custom.pointSize";
+                  value = 1;
+                }
+              ];
+            }
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "B";
+              };
+              properties = [
+                {
+                  id = "custom.lineWidth";
+                  value = 4;
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 0;
+                }
+              ];
+            }
+          ];
+        };
+      }
+      {
+        id = 7;
+        type = "stat";
+        title = "Energy Usage (24h)";
+        gridPos = {
+          h = 8;
+          w = 4;
+          x = 8;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "avg_over_time((apcupsd_ups_load_percent / 100 * apcupsd_nominal_power_watts + 4.5)[24h:]) * 24 / 1000";
+            legendFormat = "";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "kwatth";
+            decimals = 2;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 5;
+                }
+                {
+                  color = "red";
+                  value = 10;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "none";
+        };
+      }
+      {
+        id = 2;
+        type = "gauge";
+        title = "UPS Load";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 12;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_ups_load_percent";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 70;
+                }
+                {
+                  color = "red";
+                  value = 90;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options.reduceOptions = {
+          calcs = [ "lastNotNull" ];
+          fields = "";
+          values = false;
+        };
+      }
+      {
+        id = 3;
+        type = "gauge";
+        title = "UPS Battery";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 18;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_battery_charge_percent";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "red";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 20;
+                }
+                {
+                  color = "green";
+                  value = 50;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options.reduceOptions = {
+          calcs = [ "lastNotNull" ];
+          fields = "";
+          values = false;
+        };
+      }
+
+      # -- Row 2: System --
+      {
+        id = 4;
+        type = "timeseries";
+        title = "CPU Temperature";
+        gridPos = {
+          h = 8;
+          w = 12;
+          x = 0;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = ''node_hwmon_temp_celsius{chip=~"pci.*"}'';
+            legendFormat = "CPU {{sensor}}";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "celsius";
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 10;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+      {
+        id = 5;
+        type = "stat";
+        title = "System Uptime";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 12;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "time() - node_boot_time_seconds";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "s";
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "none";
+        };
+      }
+      {
+        id = 6;
+        type = "stat";
+        title = "Jellyfin Active Streams";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 18;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "count(jellyfin_now_playing_state) or vector(0)";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 3;
+                }
+                {
+                  color = "red";
+                  value = 6;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "area";
+        };
+      }
+
+      # -- Row 3: qBittorrent --
+      {
+        id = 11;
+        type = "timeseries";
+        title = "qBittorrent Speed";
+        gridPos = {
+          h = 8;
+          w = 24;
+          x = 0;
+          y = 16;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "sum(qbit_dlspeed) or vector(0)";
+            legendFormat = "Download";
+            refId = "A";
+          }
+          {
+            datasource = promDs;
+            expr = "sum(qbit_upspeed) or vector(0)";
+            legendFormat = "Upload";
+            refId = "B";
+          }
+          {
+            datasource = promDs;
+            expr = "avg_over_time((sum(qbit_dlspeed) or vector(0))[10m:])";
+            legendFormat = "Download (10m avg)";
+            refId = "C";
+          }
+          {
+            datasource = promDs;
+            expr = "avg_over_time((sum(qbit_upspeed) or vector(0))[10m:])";
+            legendFormat = "Upload (10m avg)";
+            refId = "D";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "binBps";
+            min = 0;
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 1;
+              fillOpacity = 10;
+              spanNulls = true;
+            };
+          };
+          overrides = [
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "A";
+              };
+              properties = [
+                {
+                  id = "color";
+                  value = {
+                    fixedColor = "green";
+                    mode = "fixed";
+                  };
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 5;
+                }
+              ];
+            }
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "B";
+              };
+              properties = [
+                {
+                  id = "color";
+                  value = {
+                    fixedColor = "blue";
+                    mode = "fixed";
+                  };
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 5;
+                }
+              ];
+            }
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "C";
+              };
+              properties = [
+                {
+                  id = "color";
+                  value = {
+                    fixedColor = "green";
+                    mode = "fixed";
+                  };
+                }
+                {
+                  id = "custom.lineWidth";
+                  value = 3;
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 0;
+                }
+              ];
+            }
+            {
+              matcher = {
+                id = "byFrameRefID";
+                options = "D";
+              };
+              properties = [
+                {
+                  id = "color";
+                  value = {
+                    fixedColor = "blue";
+                    mode = "fixed";
+                  };
+                }
+                {
+                  id = "custom.lineWidth";
+                  value = 3;
+                }
+                {
+                  id = "custom.fillOpacity";
+                  value = 0;
+                }
+              ];
+            }
+          ];
+        };
+      }
+
+      # -- Row 4: Intel GPU --
+      {
+        id = 8;
+        type = "timeseries";
+        title = "Intel GPU Utilization";
+        gridPos = {
+          h = 8;
+          w = 24;
+          x = 0;
+          y = 24;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "igpu_engines_busy_percent";
+            legendFormat = "{{engine}}";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 10;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+
+      # -- Row 5: Storage --
+      {
+        id = 12;
+        type = "timeseries";
+        title = "ZFS Pool Utilization";
+        gridPos = {
+          h = 8;
+          w = 12;
+          x = 0;
+          y = 32;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "zfs_pool_allocated_bytes{pool=\"tank\"} / zfs_pool_size_bytes{pool=\"tank\"} * 100";
+            legendFormat = "tank";
+            refId = "A";
+          }
+          {
+            datasource = promDs;
+            expr = "zfs_pool_allocated_bytes{pool=\"hdds\"} / zfs_pool_size_bytes{pool=\"hdds\"} * 100";
+            legendFormat = "hdds";
+            refId = "B";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 20;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+      {
+        id = 13;
+        type = "timeseries";
+        title = "Boot Drive Partitions";
+        gridPos = {
+          h = 8;
+          w = 12;
+          x = 12;
+          y = 32;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "(node_filesystem_size_bytes{mountpoint=\"/boot\"} - node_filesystem_avail_bytes{mountpoint=\"/boot\"}) / node_filesystem_size_bytes{mountpoint=\"/boot\"} * 100";
+            legendFormat = "/boot";
+            refId = "A";
+          }
+          {
+            datasource = promDs;
+            expr = "(node_filesystem_size_bytes{mountpoint=\"/persistent\"} - node_filesystem_avail_bytes{mountpoint=\"/persistent\"}) / node_filesystem_size_bytes{mountpoint=\"/persistent\"} * 100";
+            legendFormat = "/persistent";
+            refId = "B";
+          }
+          {
+            datasource = promDs;
+            expr = "(node_filesystem_size_bytes{mountpoint=\"/nix\"} - node_filesystem_avail_bytes{mountpoint=\"/nix\"}) / node_filesystem_size_bytes{mountpoint=\"/nix\"} * 100";
+            legendFormat = "/nix";
+            refId = "C";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 20;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+    ];
+  };
+in
+{
+  environment.etc."grafana-dashboards/system-overview.json" = {
+    text = builtins.toJSON dashboard;
+    mode = "0444";
+  };
+}

services/grafana/default.nix

@@ -0,0 +1,10 @@
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+    ./dashboard.nix
+    ./exporters.nix
+    ./jellyfin-annotations.nix
+    ./zfs-scrub-annotations.nix
+  ];
+}

services/grafana/exporters.nix

@@ -0,0 +1,112 @@
+{
+  config,
+  pkgs,
+  inputs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  jellyfinExporterPort = service_configs.ports.private.jellyfin_exporter.port;
+  qbitExporterPort = service_configs.ports.private.qbittorrent_exporter.port;
+  igpuExporterPort = service_configs.ports.private.igpu_exporter.port;
+in
+{
+  # -- Jellyfin Prometheus Exporter --
+  # Replaces custom jellyfin-collector.nix textfile timer.
+  # Exposes per-session metrics (jellyfin_now_playing_state) and library stats.
+  systemd.services.jellyfin-exporter =
+    lib.mkIf (config.services.grafana.enable && config.services.jellyfin.enable)
+      {
+        description = "Prometheus exporter for Jellyfin";
+        after = [
+          "network.target"
+          "jellyfin.service"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          ExecStart = lib.getExe (
+            pkgs.writeShellApplication {
+              name = "jellyfin-exporter-wrapper";
+              runtimeInputs = [ pkgs.jellyfin-exporter ];
+              text = ''
+                exec jellyfin_exporter \
+                  --jellyfin.address=http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port} \
+                  --jellyfin.token="$(cat "$CREDENTIALS_DIRECTORY/jellyfin-api-key")" \
+                  --web.listen-address=127.0.0.1:${toString jellyfinExporterPort}
+              '';
+            }
+          );
+          Restart = "on-failure";
+          RestartSec = "10s";
+          DynamicUser = true;
+          NoNewPrivileges = true;
+          ProtectSystem = "strict";
+          ProtectHome = true;
+          PrivateTmp = true;
+          MemoryDenyWriteExecute = true;
+          LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
+        };
+      };
+
+  # -- qBittorrent Prometheus Exporter --
+  # Replaces custom qbittorrent-collector.nix textfile timer.
+  # Exposes per-torrent metrics (qbit_dlspeed, qbit_upspeed) and aggregate stats.
+  # qBittorrent runs in a VPN namespace; the exporter reaches it via namespace address.
+  systemd.services.qbittorrent-exporter =
+    lib.mkIf (config.services.grafana.enable && config.services.qbittorrent.enable)
+      {
+        description = "Prometheus exporter for qBittorrent";
+        after = [
+          "network.target"
+          "qbittorrent.service"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          ExecStart =
+            lib.getExe' inputs.qbittorrent-metrics-exporter.packages.${pkgs.system}.default
+              "qbittorrent-metrics-exporter";
+          Restart = "on-failure";
+          RestartSec = "10s";
+          DynamicUser = true;
+          NoNewPrivileges = true;
+          ProtectSystem = "strict";
+          ProtectHome = true;
+          PrivateTmp = true;
+        };
+        environment = {
+          HOST = "127.0.0.1";
+          PORT = toString qbitExporterPort;
+          SCRAPE_INTERVAL = "15";
+          BACKEND = "in-memory";
+          # qBittorrent has AuthSubnetWhitelist=0.0.0.0/0, so no real password needed.
+          # The exporter still expects the env var to be set.
+          QBITTORRENT_PASSWORD = "unused";
+          QBITTORRENT_USERNAME = "admin";
+          TORRENT_HOSTS = "qbit:main=http://${config.vpnNamespaces.wg.namespaceAddress}:${toString config.services.qbittorrent.webuiPort}|http://${config.vpnNamespaces.wg.namespaceAddress}:${toString config.services.qbittorrent.webuiPort}";
+          RUST_LOG = "warn";
+        };
+      };
+
+  # -- Intel GPU Prometheus Exporter --
+  # Replaces custom intel-gpu-collector.nix + intel-gpu-collector.py textfile timer.
+  # Exposes engine busy%, frequency, and RC6 metrics via /metrics.
+  # Requires privileged access to GPU debug interfaces (intel_gpu_top).
+  systemd.services.igpu-exporter = lib.mkIf config.services.grafana.enable {
+    description = "Prometheus exporter for Intel integrated GPU";
+    wantedBy = [ "multi-user.target" ];
+    path = [ pkgs.intel-gpu-tools ];
+    serviceConfig = {
+      ExecStart = lib.getExe pkgs.igpu-exporter;
+      Restart = "on-failure";
+      RestartSec = "10s";
+      # intel_gpu_top requires root-level access to GPU debug interfaces
+      ProtectHome = true;
+      PrivateTmp = true;
+    };
+    environment = {
+      PORT = toString igpuExporterPort;
+      REFRESH_PERIOD_MS = "30000";
+    };
+  };
+}

services/grafana/grafana.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "grafana" service_configs.zpool_ssds [
+      service_configs.grafana.dir
+    ])
+    (lib.serviceFilePerms "grafana" [
+      "Z ${service_configs.grafana.dir} 0700 grafana grafana"
+    ])
+    (lib.mkCaddyReverseProxy {
+      domain = service_configs.grafana.domain;
+      port = service_configs.ports.private.grafana.port;
+      auth = true;
+    })
+  ];
+
+  services.grafana = {
+    enable = true;
+    dataDir = service_configs.grafana.dir;
+
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = service_configs.ports.private.grafana.port;
+        domain = service_configs.grafana.domain;
+        root_url = "https://${service_configs.grafana.domain}";
+      };
+
+      database = {
+        type = "postgres";
+        host = service_configs.postgres.socket;
+        user = "grafana";
+      };
+
+      "auth.anonymous" = {
+        enabled = true;
+        org_role = "Admin";
+      };
+      "auth.basic".enabled = false;
+      "auth".disable_login_form = true;
+
+      analytics.reporting_enabled = false;
+
+      feature_toggles.enable = "dataConnectionsConsole=false";
+
+      users.default_theme = "dark";
+
+      # Disable unused built-in integrations
+      alerting.enabled = false;
+      "unified_alerting".enabled = false;
+      explore.enabled = false;
+      news.news_feed_enabled = false;
+
+      plugins = {
+        enable_alpha = false;
+        plugin_admin_enabled = false;
+      };
+    };
+
+    provision = {
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://127.0.0.1:${toString service_configs.ports.private.prometheus.port}";
+            access = "proxy";
+            isDefault = true;
+            editable = false;
+            uid = "prometheus";
+          }
+        ];
+      };
+
+      dashboards.settings.providers = [
+        {
+          name = "system";
+          type = "file";
+          options.path = "/etc/grafana-dashboards";
+          disableDeletion = true;
+          updateIntervalSeconds = 60;
+        }
+      ];
+    };
+  };
+
+  services.postgresql = {
+    ensureDatabases = [ "grafana" ];
+    ensureUsers = [
+      {
+        name = "grafana";
+        ensureDBOwnership = true;
+        ensureClauses.login = true;
+      }
+    ];
+  };
+}

services/grafana/jellyfin-annotations.nix

@@ -0,0 +1,18 @@
+{
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+lib.mkIf (config.services.grafana.enable && config.services.jellyfin.enable) (
+  lib.mkGrafanaAnnotationService {
+    name = "jellyfin";
+    description = "Jellyfin stream annotation service for Grafana";
+    script = ./jellyfin-annotations.py;
+    environment = {
+      JELLYFIN_URL = "http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port}";
+      POLL_INTERVAL = "30";
+    };
+    loadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
+  }
+)

services/grafana/jellyfin-annotations.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+JELLYFIN_URL = os.environ.get("JELLYFIN_URL", "http://127.0.0.1:8096")
+GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000")
+STATE_FILE = os.environ.get("STATE_FILE", "/var/lib/jellyfin-annotations/state.json")
+POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "30"))
+
+
+def get_api_key():
+    cred_dir = os.environ.get("CREDENTIALS_DIRECTORY")
+    if cred_dir:
+        return Path(cred_dir, "jellyfin-api-key").read_text().strip()
+    for p in ["/run/agenix/jellyfin-api-key"]:
+        if Path(p).exists():
+            return Path(p).read_text().strip()
+    sys.exit("ERROR: Cannot find jellyfin-api-key")
+
+
+def http_json(method, url, body=None):
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(
+        url,
+        data=data,
+        headers={"Content-Type": "application/json", "Accept": "application/json"},
+        method=method,
+    )
+    with urllib.request.urlopen(req, timeout=5) as resp:
+        return json.loads(resp.read())
+
+
+def get_active_sessions(api_key):
+    try:
+        req = urllib.request.Request(
+            f"{JELLYFIN_URL}/Sessions?api_key={api_key}",
+            headers={"Accept": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=5) as resp:
+            sessions = json.loads(resp.read())
+        return [s for s in sessions if s.get("NowPlayingItem")]
+    except Exception as e:
+        print(f"Error fetching sessions: {e}", file=sys.stderr)
+        return None
+
+
+def _codec(name):
+    if not name:
+        return ""
+    aliases = {"h264": "H.264", "h265": "H.265", "hevc": "H.265", "av1": "AV1",
+               "vp9": "VP9", "vp8": "VP8", "mpeg4": "MPEG-4", "mpeg2video": "MPEG-2",
+               "aac": "AAC", "ac3": "AC3", "eac3": "EAC3", "dts": "DTS",
+               "truehd": "TrueHD", "mp3": "MP3", "opus": "Opus", "flac": "FLAC",
+               "vorbis": "Vorbis"}
+    return aliases.get(name.lower(), name.upper())
+
+
+def _res(width, height):
+    if not height:
+        return ""
+    common = {2160: "4K", 1440: "1440p", 1080: "1080p", 720: "720p",
+              480: "480p", 360: "360p"}
+    return common.get(height, f"{height}p")
+
+
+def _channels(n):
+    labels = {1: "Mono", 2: "Stereo", 6: "5.1", 7: "6.1", 8: "7.1"}
+    return labels.get(n, f"{n}ch") if n else ""
+
+
+def format_label(session):
+    user = session.get("UserName", "Unknown")
+    item = session.get("NowPlayingItem", {}) or {}
+    transcode = session.get("TranscodingInfo") or {}
+    play_state = session.get("PlayState") or {}
+    client = session.get("Client", "")
+    device = session.get("DeviceName", "")
+
+    name = item.get("Name", "Unknown")
+    series = item.get("SeriesName", "")
+    season = item.get("ParentIndexNumber")
+    episode = item.get("IndexNumber")
+    media_type = item.get("Type", "")
+
+    if series and season and episode:
+        title = f"{series} S{season:02d}E{episode:02d} \u2013 {name}"
+    elif series:
+        title = f"{series} \u2013 {name}"
+    elif media_type == "Movie":
+        title = f"{name} (movie)"
+    else:
+        title = name
+
+    play_method = play_state.get("PlayMethod", "")
+    if play_method == "DirectPlay":
+        method = "Direct Play"
+    elif play_method == "DirectStream":
+        method = "Direct Stream"
+    elif play_method == "Transcode" or transcode:
+        method = "Transcode"
+    else:
+        method = "Direct Play"
+
+    media_streams = item.get("MediaStreams") or []
+    video_streams = [s for s in media_streams if s.get("Type") == "Video"]
+    audio_streams = [s for s in media_streams if s.get("Type") == "Audio"]
+    default_audio = next((s for s in audio_streams if s.get("IsDefault")), None)
+    audio_stream = default_audio or (audio_streams[0] if audio_streams else {})
+    video_stream = video_streams[0] if video_streams else {}
+
+    src_vcodec = _codec(video_stream.get("Codec", ""))
+    src_res = _res(video_stream.get("Width") or item.get("Width"),
+                   video_stream.get("Height") or item.get("Height"))
+    src_acodec = _codec(audio_stream.get("Codec", ""))
+    src_channels = _channels(audio_stream.get("Channels"))
+
+    is_video_direct = transcode.get("IsVideoDirect", True)
+    is_audio_direct = transcode.get("IsAudioDirect", True)
+
+    if transcode and not is_video_direct:
+        dst_vcodec = _codec(transcode.get("VideoCodec", ""))
+        dst_res = _res(transcode.get("Width"), transcode.get("Height")) or src_res
+        if src_vcodec and dst_vcodec and src_vcodec != dst_vcodec:
+            video_part = f"{src_vcodec}\u2192{dst_vcodec} {dst_res}".strip()
+        else:
+            video_part = f"{dst_vcodec or src_vcodec} {dst_res}".strip()
+    else:
+        video_part = f"{src_vcodec} {src_res}".strip()
+
+    if transcode and not is_audio_direct:
+        dst_acodec = _codec(transcode.get("AudioCodec", ""))
+        dst_channels = _channels(transcode.get("AudioChannels")) or src_channels
+        if src_acodec and dst_acodec and src_acodec != dst_acodec:
+            audio_part = f"{src_acodec}\u2192{dst_acodec} {dst_channels}".strip()
+        else:
+            audio_part = f"{dst_acodec or src_acodec} {dst_channels}".strip()
+    else:
+        audio_part = f"{src_acodec} {src_channels}".strip()
+
+    bitrate = transcode.get("Bitrate") or item.get("Bitrate")
+    bitrate_part = f"{bitrate / 1_000_000:.1f} Mbps" if bitrate else ""
+
+    reasons = transcode.get("TranscodeReasons") or []
+    reason_part = f"[{', '.join(reasons)}]" if reasons else ""
+
+    stream_parts = [p for p in [method, video_part, audio_part, bitrate_part, reason_part] if p]
+    client_str = " \u00b7 ".join(filter(None, [client, device]))
+
+    lines = [f"{user}: {title}", " | ".join(stream_parts)]
+    if client_str:
+        lines.append(client_str)
+
+    return "\n".join(lines)
+
+
+def load_state():
+    try:
+        with open(STATE_FILE) as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def save_state(state):
+    os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
+    tmp = STATE_FILE + ".tmp"
+    with open(tmp, "w") as f:
+        json.dump(state, f)
+    os.replace(tmp, STATE_FILE)
+
+
+def grafana_post(label, start_ms):
+    try:
+        result = http_json(
+            "POST",
+            f"{GRAFANA_URL}/api/annotations",
+            {"time": start_ms, "text": label, "tags": ["jellyfin"]},
+        )
+        return result.get("id")
+    except Exception as e:
+        print(f"Error posting annotation: {e}", file=sys.stderr)
+        return None
+
+
+def grafana_close(grafana_id, end_ms):
+    try:
+        http_json(
+            "PATCH",
+            f"{GRAFANA_URL}/api/annotations/{grafana_id}",
+            {"timeEnd": end_ms},
+        )
+    except Exception as e:
+        print(f"Error closing annotation {grafana_id}: {e}", file=sys.stderr)
+
+
+def main():
+    api_key = get_api_key()
+    state = load_state()
+
+    while True:
+        now_ms = int(time.time() * 1000)
+        sessions = get_active_sessions(api_key)
+
+        if sessions is not None:
+            current_ids = {s["Id"] for s in sessions}
+
+            for s in sessions:
+                sid = s["Id"]
+                if sid not in state:
+                    label = format_label(s)
+                    grafana_id = grafana_post(label, now_ms)
+                    if grafana_id is not None:
+                        state[sid] = {
+                            "grafana_id": grafana_id,
+                            "label": label,
+                            "start_ms": now_ms,
+                        }
+                        save_state(state)
+
+            for sid in [k for k in state if k not in current_ids]:
+                info = state.pop(sid)
+                grafana_close(info["grafana_id"], now_ms)
+                save_state(state)
+
+        time.sleep(POLL_INTERVAL)
+
+
+if __name__ == "__main__":
+    main()

services/grafana/prometheus.nix

@@ -0,0 +1,110 @@
+{
+  service_configs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+in
+{
+  imports = [
+    (lib.serviceMountWithZpool "prometheus" service_configs.zpool_ssds [
+      "/var/lib/prometheus"
+    ])
+    (lib.serviceFilePerms "prometheus" [
+      "Z /var/lib/prometheus 0700 prometheus prometheus"
+    ])
+  ];
+
+  services.prometheus = {
+    enable = true;
+    port = service_configs.ports.private.prometheus.port;
+    listenAddress = "127.0.0.1";
+    stateDir = "prometheus";
+    retentionTime = "0d"; # 0 disables time-based retention (keep forever)
+
+    exporters = {
+      node = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_node.port;
+        listenAddress = "127.0.0.1";
+        enabledCollectors = [
+          "hwmon"
+          "systemd"
+          "textfile"
+        ];
+        extraFlags = [
+          "--collector.textfile.directory=${textfileDir}"
+        ];
+      };
+
+      apcupsd = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_apcupsd.port;
+        listenAddress = "127.0.0.1";
+        apcupsdAddress = "127.0.0.1:3551";
+      };
+
+      zfs = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_zfs.port;
+        listenAddress = "127.0.0.1";
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "prometheus";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus.port}" ]; }
+        ];
+      }
+      {
+        job_name = "node";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_node.port}" ]; }
+        ];
+      }
+      {
+        job_name = "apcupsd";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_apcupsd.port}" ]; }
+        ];
+      }
+      {
+        job_name = "llama-cpp";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.llama_cpp.port}" ]; }
+        ];
+      }
+      {
+        job_name = "jellyfin";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.jellyfin_exporter.port}" ]; }
+        ];
+      }
+      {
+        job_name = "qbittorrent";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.qbittorrent_exporter.port}" ]; }
+        ];
+      }
+      {
+        job_name = "igpu";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.igpu_exporter.port}" ]; }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_zfs.port}" ]; }
+        ];
+      }
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d ${textfileDir} 0755 root root -"
+  ];
+}

services/grafana/zfs-scrub-annotations.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  pkgs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  grafanaUrl = "http://127.0.0.1:${toString service_configs.ports.private.grafana.port}";
+
+  script = pkgs.writeShellApplication {
+    name = "zfs-scrub-annotations";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+      coreutils
+      gnugrep
+      gnused
+      config.boot.zfs.package
+    ];
+    text = builtins.readFile ./zfs-scrub-annotations.sh;
+  };
+in
+lib.mkIf (config.services.grafana.enable && config.services.zfs.autoScrub.enable) {
+  systemd.services.zfs-scrub = {
+    environment = {
+      GRAFANA_URL = grafanaUrl;
+      STATE_DIR = "/run/zfs-scrub-annotations";
+    };
+    serviceConfig = {
+      RuntimeDirectory = "zfs-scrub-annotations";
+      ExecStartPre = [ "-${lib.getExe script} start" ];
+      ExecStopPost = [ "${lib.getExe script} stop" ];
+    };
+  };
+}

services/grafana/zfs-scrub-annotations.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# ZFS scrub annotation script for Grafana
+# Usage: zfs-scrub-annotations.sh {start|stop}
+# Required env: GRAFANA_URL, STATE_DIR
+# Required on PATH: zpool, curl, jq, paste, date, grep, sed
+
+set -euo pipefail
+
+ACTION="${1:-}"
+GRAFANA_URL="${GRAFANA_URL:?GRAFANA_URL required}"
+STATE_DIR="${STATE_DIR:?STATE_DIR required}"
+
+case "$ACTION" in
+  start)
+    POOLS=$(zpool list -H -o name | paste -sd ', ')
+    NOW_MS=$(date +%s%3N)
+
+    RESPONSE=$(curl -sf --max-time 5 \
+      -X POST "$GRAFANA_URL/api/annotations" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "ZFS scrub: $POOLS" --argjson time "$NOW_MS" \
+        '{time: $time, text: $text, tags: ["zfs-scrub"]}')" \
+    ) || exit 0
+
+    echo "$RESPONSE" | jq -r '.id' > "$STATE_DIR/annotation-id"
+    ;;
+
+  stop)
+    ANN_ID=$(cat "$STATE_DIR/annotation-id" 2>/dev/null) || exit 0
+    [ -z "$ANN_ID" ] && exit 0
+
+    NOW_MS=$(date +%s%3N)
+
+    RESULTS=""
+    while IFS= read -r pool; do
+      scan_line=$(zpool status "$pool" | grep "scan:" | sed 's/^[[:space:]]*//')
+      RESULTS="${RESULTS}${pool}: ${scan_line}"$'\n'
+    done < <(zpool list -H -o name)
+
+    TEXT=$(printf "ZFS scrub completed\n%s" "$RESULTS")
+
+    curl -sf --max-time 5 \
+      -X PATCH "$GRAFANA_URL/api/annotations/$ANN_ID" \
+      -H "Content-Type: application/json" \
+      -d "$(jq -n --arg text "$TEXT" --argjson timeEnd "$NOW_MS" \
+        '{timeEnd: $timeEnd, text: $text}')" || true
+
+    rm -f "$STATE_DIR/annotation-id"
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop}" >&2
+    exit 1
+    ;;
+esac