grafana: init

Shows powerdraw, temps, uptime, and jellyfin streams

.gitea/workflows/deploy.yml

@@ -0,0 +1,60 @@
+name: Build and Deploy
+on:
+  push:
+    branches: [main]
+
+jobs:
+  deploy:
+    runs-on: nix
+    env:
+      GIT_SSH_COMMAND: "ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts"
+    steps:
+      - uses: https://github.com/actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Unlock git-crypt
+        run: |
+          git-crypt unlock /run/agenix/git-crypt-key-server-config
+
+      - name: Build NixOS configuration
+        run: |
+          nix build .#nixosConfigurations.muffin.config.system.build.toplevel -L
+
+      - name: Deploy via deploy-rs
+        run: |
+          eval $(ssh-agent -s)
+          ssh-add /run/agenix/ci-deploy-key
+          nix run github:serokell/deploy-rs -- .#muffin --skip-checks --ssh-opts="-o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts"
+
+      - name: Health check
+        run: |
+          sleep 10
+          ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts root@server-public \
+            "systemctl is-active gitea && systemctl is-active caddy && systemctl is-active continuwuity && systemctl is-active coturn"
+
+      - name: Notify success
+        if: success()
+        run: |
+          TOPIC=$(cat /run/agenix/ntfy-alerts-topic | tr -d '[:space:]')
+          TOKEN=$(cat /run/agenix/ntfy-alerts-token | tr -d '[:space:]')
+          curl -sf -o /dev/null -X POST \
+            "https://ntfy.sigkill.computer/$TOPIC" \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Title: [muffin] Deploy succeeded" \
+            -H "Priority: default" \
+            -H "Tags: white_check_mark" \
+            -d "server-config deployed from commit ${GITHUB_SHA::8}"
+
+      - name: Notify failure
+        if: failure()
+        run: |
+          TOPIC=$(cat /run/agenix/ntfy-alerts-topic 2>/dev/null | tr -d '[:space:]')
+          TOKEN=$(cat /run/agenix/ntfy-alerts-token 2>/dev/null | tr -d '[:space:]')
+          curl -sf -o /dev/null -X POST \
+            "https://ntfy.sigkill.computer/$TOPIC" \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Title: [muffin] Deploy FAILED" \
+            -H "Priority: urgent" \
+            -H "Tags: rotating_light" \
+            -d "server-config deploy failed at commit ${GITHUB_SHA::8}" || true

AGENTS.md

@@ -99,7 +99,11 @@ Each service file in `services/` follows this structure:
 - **git-crypt**: `secrets/` directory and `usb-secrets/usb-secrets-key*` are encrypted (see `.gitattributes`)
 - **agenix**: secrets declared in `modules/age-secrets.nix`, decrypted at runtime to `/run/agenix/`
 - **Identity**: USB drive at `/mnt/usb-secrets/usb-secrets-key`
-- **Encrypting new secrets**: The agenix encryption key is in `usb-secrets/usb-secrets-key` (SSH private key, git-crypt encrypted). To create a new secret: derive the age public key with `ssh-keygen -y -f usb-secrets/usb-secrets-key | ssh-to-age`, then encrypt with `age -r <public-key> -o secrets/<name>.age`.
+- **Encrypting new secrets**: The agenix identity is an SSH private key at `usb-secrets/usb-secrets-key` (git-crypt encrypted). To encrypt a new secret, use the SSH public key directly with `age -R`:
+  ```bash
+  age -R <(ssh-keygen -y -f usb-secrets/usb-secrets-key) -o secrets/<name>.age /path/to/plaintext
+  ```
+- **DO NOT use `ssh-to-age`**. Using `ssh-to-age` to derive a native age public key and then encrypting with `age -r age1...` produces `X25519` recipient stanzas. The SSH private key identity on the server can only decrypt `ssh-ed25519` stanzas. This mismatch causes `age: error: no identity matched any of the recipients` at deploy time. Always use `age -R` with the SSH public key directly.
 - Never read or commit plaintext secrets. Never log secret values.
 
 ### Important Patterns

configuration.nix

@@ -26,6 +26,7 @@
     ./services/caddy.nix
     ./services/immich.nix
     ./services/gitea.nix
+    ./services/gitea-actions-runner.nix
     ./services/minecraft.nix
 
     ./services/wg.nix
@@ -46,6 +47,7 @@
     ./services/soulseek.nix
 
     ./services/ups.nix
+    ./services/monitoring.nix
 
     ./services/bitwarden.nix
     ./services/firefox-syncserver.nix
@@ -73,6 +75,18 @@
     ./services/mollysocket.nix
   ];
 
+  # Hosts entries for CI/CD deploy targets
+  networking.hosts."192.168.1.50" = [ "server-public" ];
+  networking.hosts."192.168.1.223" = [ "desktop" ];
+
+  # SSH known_hosts for CI runner (pinned host keys)
+  environment.etc."ci-known-hosts".text = ''
+    server-public ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFMjgaMnE+zS7tL+m5E7gh9Q9U1zurLdmU0qcmEmaucu
+    192.168.1.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFMjgaMnE+zS7tL+m5E7gh9Q9U1zurLdmU0qcmEmaucu
+    git.sigkill.computer ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFMjgaMnE+zS7tL+m5E7gh9Q9U1zurLdmU0qcmEmaucu
+    git.gardling.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFMjgaMnE+zS7tL+m5E7gh9Q9U1zurLdmU0qcmEmaucu
+  '';
+
   services.kmscon.enable = true;
 
   systemd.targets = {
@@ -249,6 +263,14 @@
 
   users.groups.${service_configs.media_group} = { };
 
+  users.users.gitea-runner = {
+    isSystemUser = true;
+    group = "gitea-runner";
+    home = "/var/lib/gitea-runner";
+    description = "Gitea Actions CI runner";
+  };
+  users.groups.gitea-runner = { };
+
   users.users.${username} = {
     isNormalUser = true;
     extraGroups = [
@@ -290,7 +312,8 @@
     enable = true;
     openFirewall = true;
     welcometext = "meow meow meow meow meow :3 xd";
-    password = builtins.readFile ./secrets/murmur_password;
+    password = "$MURMURD_PASSWORD";
+    environmentFile = config.age.secrets.murmur-password-env.path;
     port = service_configs.ports.public.murmur.port;
   };
 

modules/age-secrets.nix

@@ -68,19 +68,19 @@
       group = "root";
     };
 
-    # ntfy-alerts secrets
+    # ntfy-alerts secrets (group-readable for CI runner notifications)
     ntfy-alerts-topic = {
       file = ../secrets/ntfy-alerts-topic.age;
-      mode = "0400";
+      mode = "0440";
       owner = "root";
-      group = "root";
+      group = "gitea-runner";
     };
 
     ntfy-alerts-token = {
       file = ../secrets/ntfy-alerts-token.age;
-      mode = "0400";
+      mode = "0440";
       owner = "root";
-      group = "root";
+      group = "gitea-runner";
     };
 
     # Firefox Sync server secrets (SYNC_MASTER_SECRET)
@@ -94,5 +94,70 @@
       file = ../secrets/mollysocket-env.age;
       mode = "0400";
     };
+
+    # Murmur (Mumble) server password
+    murmur-password-env = {
+      file = ../secrets/murmur-password-env.age;
+      mode = "0400";
+      owner = "murmur";
+      group = "murmur";
+    };
+
+    # Coturn static auth secret
+    coturn-auth-secret = {
+      file = ../secrets/coturn-auth-secret.age;
+      mode = "0400";
+      owner = "turnserver";
+      group = "turnserver";
+    };
+
+    # Matrix (continuwuity) registration token
+    matrix-reg-token = {
+      file = ../secrets/matrix-reg-token.age;
+      mode = "0400";
+      owner = "continuwuity";
+      group = "continuwuity";
+    };
+
+    # Matrix (continuwuity) TURN secret — same secret as coturn-auth-secret,
+    # decrypted separately so continuwuity can read it with its own ownership
+    matrix-turn-secret = {
+      file = ../secrets/coturn-auth-secret.age;
+      mode = "0400";
+      owner = "continuwuity";
+      group = "continuwuity";
+    };
+
+    # CI deploy SSH key
+    ci-deploy-key = {
+      file = ../secrets/ci-deploy-key.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Git-crypt symmetric key for dotfiles repo
+    git-crypt-key-dotfiles = {
+      file = ../secrets/git-crypt-key-dotfiles.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Git-crypt symmetric key for server-config repo
+    git-crypt-key-server-config = {
+      file = ../secrets/git-crypt-key-server-config.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Gitea Actions runner registration token
+    gitea-runner-token = {
+      file = ../secrets/gitea-runner-token.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
   };
 }

modules/hardware.nix

@@ -5,6 +5,20 @@
   service_configs,
   ...
 }:
+let
+  hddTuneIosched = pkgs.writeShellScript "hdd-tune-iosched" ''
+    # Called by udev with the partition kernel name (e.g. sdb1).
+    # Derives the parent disk and applies mq-deadline iosched params.
+    parent=''${1%%[0-9]*}
+    dev="/sys/block/$parent"
+    [ -d "$dev/queue/iosched" ] || exit 0
+    echo 15000 > "$dev/queue/iosched/read_expire"
+    echo 15000 > "$dev/queue/iosched/write_expire"
+    echo 128 > "$dev/queue/iosched/fifo_batch"
+    echo 16 > "$dev/queue/iosched/writes_starved"
+    echo 4096 > "$dev/queue/max_sectors_kb" 2>/dev/null || true
+  '';
+in
 {
   boot.initrd.availableKernelModules = [
     "xhci_pci"
@@ -30,48 +44,13 @@
   # fifo_batch=128 keeps sweeps long; writes_starved=16 heavily favors reads.
   # 4 MiB readahead matches libtorrent piece extent affinity for sequential prefetch.
   #
-  # This runs as a systemd oneshot rather than udev rules because the NixOS ZFS module
+  # The NixOS ZFS module hardcodes a udev rule that forces scheduler="none" on all
-  # hardcodes a udev rule that forces scheduler="none" on all ZFS member partitions'
+  # ZFS member partitions' parent disks (on both add AND change events). We counter
-  # parent disks, overriding any scheduler set via udev on the disk event.
+  # it with lib.mkAfter so our rule appears after theirs in 99-local.rules — our
-  systemd.services.hdd-io-tuning = {
+  # rule matches the same partition events and sets mq-deadline back, then a RUN
-    description = "HDD I/O scheduler and queue tuning";
+  # script applies the iosched params. Only targets rotational, non-removable disks
-    after = [
+  # (i.e. HDDs, not SSDs or USB).
-      "zfs-import.target"
+  services.udev.extraRules = lib.mkAfter ''
-      "systemd-udev-settle.service"
+    ACTION=="add|change", KERNEL=="sd[a-z]*[0-9]*", ENV{ID_FS_TYPE}=="zfs_member", ATTR{../queue/rotational}=="1", ATTR{../removable}=="0", ATTR{../queue/scheduler}="mq-deadline", ATTR{../queue/read_ahead_kb}="4096", ATTR{../queue/nr_requests}="512", RUN+="${hddTuneIosched} %k"
-    ];
+  '';
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-    };
-    path = with pkgs; [
-      coreutils
-      gawk
-      zfs
-    ];
-    script = ''
-      # Only tune disks in the hdds pool — not all rotational disks.
-      # zpool status gives by-id device names; we resolve to /sys/block/<name>.
-      zpool status hdds | awk '/^\t/ && $1 ~ /^(ata-|nvme-|scsi-)/ {print $1}' | while read -r id; do
-        link="/dev/disk/by-id/$id"
-        [ -L "$link" ] || continue
-        name=$(basename "$(readlink -f "$link")")
-        dev="/sys/block/$name"
-        [ -d "$dev" ] || continue
-
-        echo mq-deadline > "$dev/queue/scheduler"
-        echo 4096 > "$dev/queue/read_ahead_kb"
-        echo 512 > "$dev/queue/nr_requests"
-
-        echo 15000 > "$dev/queue/iosched/read_expire"
-        echo 15000 > "$dev/queue/iosched/write_expire"
-        echo 128 > "$dev/queue/iosched/fifo_batch"
-        echo 16 > "$dev/queue/iosched/writes_starved"
-
-        echo 4096 > "$dev/queue/max_sectors_kb" 2>/dev/null || true
-
-        echo "Tuned $id -> $name: mq-deadline, 4M readahead, 15s deadlines"
-      done
-    '';
-  };
 }

modules/impermanence.nix

@@ -24,6 +24,7 @@
       # ZFS cache directory - persisting the directory instead of the file
       # avoids "device busy" errors when ZFS atomically updates the cache
       "/etc/zfs"
+      "/var/lib/gitea-runner"
     ];
 
     files = [

service-configs.nix

@@ -153,6 +153,22 @@ rec {
         port = 8020;
         proto = "tcp";
       };
+      grafana = {
+        port = 3000;
+        proto = "tcp";
+      };
+      prometheus = {
+        port = 9090;
+        proto = "tcp";
+      };
+      prometheus_node = {
+        port = 9100;
+        proto = "tcp";
+      };
+      prometheus_apcupsd = {
+        port = 9162;
+        proto = "tcp";
+      };
     };
   };
 
@@ -212,6 +228,7 @@ rec {
 
   p2pool = {
     dataDir = services_dir + "/p2pool";
+    walletAddress = "49b6NT2k7fQHs8JvF7naUvchYwTQmRpoMMXb1KJTg5UcZVmyPJ7n6jgiH8DrvEsMg5GvMjJqPB1c1PTBAYtUTsbeHe5YMBx";
   };
 
   matrix = {
@@ -265,6 +282,11 @@ rec {
     domain = "firefox-sync.${https.domain}";
   };
 
+  grafana = {
+    dir = services_dir + "/grafana";
+    domain = "grafana.${https.domain}";
+  };
+
   media = {
     moviesDir = torrents_path + "/media/movies";
     tvDir = torrents_path + "/media/tv";

services/coturn.nix

@@ -9,7 +9,7 @@
     enable = true;
     realm = service_configs.https.domain;
     use-auth-secret = true;
-    static-auth-secret = lib.strings.trim (builtins.readFile ../secrets/coturn_static_auth_secret);
+    static-auth-secret-file = config.age.secrets.coturn-auth-secret.path;
     listening-port = service_configs.ports.public.coturn.port;
     tls-listening-port = service_configs.ports.public.coturn_tls.port;
     no-cli = true;

services/gitea-actions-runner.nix

@@ -0,0 +1,46 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  ...
+}:
+{
+  services.gitea-actions-runner.instances.muffin = {
+    enable = true;
+    name = "muffin";
+    url = config.services.gitea.settings.server.ROOT_URL;
+    tokenFile = config.age.secrets.gitea-runner-token.path;
+    labels = [ "nix:host" ];
+    hostPackages = with pkgs; [
+      bash
+      coreutils
+      curl
+      gawk
+      git
+      git-crypt
+      gnugrep
+      gnused
+      jq
+      nix
+      nodejs
+      openssh
+    ];
+    settings = {
+      runner = {
+        capacity = 1;
+        timeout = "3h";
+      };
+    };
+  };
+
+  # Override DynamicUser to use our static gitea-runner user
+  systemd.services."gitea-runner-muffin" = {
+    serviceConfig = {
+      DynamicUser = lib.mkForce false;
+      User = "gitea-runner";
+      Group = "gitea-runner";
+    };
+    environment.GIT_SSH_COMMAND = "ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts";
+  };
+}

services/gitea.nix

@@ -37,6 +37,7 @@
       };
       # only I shall use gitea
       service.DISABLE_REGISTRATION = true;
+      actions.ENABLED = true;
     };
   };
 

services/matrix.nix

@@ -21,7 +21,7 @@
       port = [ service_configs.ports.private.matrix.port ];
       server_name = service_configs.https.domain;
       allow_registration = true;
-      registration_token = lib.strings.trim (builtins.readFile ../secrets/matrix_reg_token);
+      registration_token_file = config.age.secrets.matrix-reg-token.path;
 
       new_user_displayname_suffix = "";
 
@@ -37,7 +37,7 @@
       ];
 
       # TURN server config (coturn)
-      turn_secret = config.services.coturn.static-auth-secret;
+      turn_secret_file = config.age.secrets.matrix-turn-secret.path;
       turn_uris = [
         "turn:${service_configs.https.domain}?transport=udp"
         "turn:${service_configs.https.domain}?transport=tcp"

services/monitoring.nix

@@ -0,0 +1,530 @@
+{
+  config,
+  pkgs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+
+  promDs = {
+    type = "prometheus";
+    uid = "prometheus";
+  };
+
+  jellyfinCollector = pkgs.writeShellApplication {
+    name = "jellyfin-metrics-collector";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+    ];
+    text = ''
+      API_KEY=$(cat "$CREDENTIALS_DIRECTORY/jellyfin-api-key")
+      JELLYFIN="http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port}"
+
+      if response=$(curl -sf --max-time 5 "''${JELLYFIN}/Sessions?api_key=''${API_KEY}"); then
+        active_streams=$(echo "$response" | jq '[.[] | select(.NowPlayingItem != null)] | length')
+      else
+        active_streams=0
+      fi
+
+      {
+        echo '# HELP jellyfin_active_streams Number of currently active Jellyfin streams'
+        echo '# TYPE jellyfin_active_streams gauge'
+        echo "jellyfin_active_streams $active_streams"
+      } > "${textfileDir}/jellyfin.prom.$$.tmp"
+      mv "${textfileDir}/jellyfin.prom.$$.tmp" "${textfileDir}/jellyfin.prom"
+    '';
+  };
+
+  dashboard = {
+    editable = true;
+    graphTooltip = 1;
+    schemaVersion = 39;
+    tags = [
+      "system"
+      "monitoring"
+    ];
+    time = {
+      from = "now-6h";
+      to = "now";
+    };
+    timezone = "browser";
+    title = "System Overview";
+    uid = "system-overview";
+
+    panels = [
+      # -- Row 1: UPS --
+      {
+        id = 1;
+        type = "timeseries";
+        title = "UPS Power Draw";
+        gridPos = {
+          h = 8;
+          w = 8;
+          x = 0;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_ups_load_percent / 100 * apcupsd_nominal_power_watts";
+            legendFormat = "Power (W)";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "watt";
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 20;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+      {
+        id = 7;
+        type = "stat";
+        title = "Energy Usage (24h)";
+        gridPos = {
+          h = 8;
+          w = 4;
+          x = 8;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "avg_over_time((apcupsd_ups_load_percent / 100 * apcupsd_nominal_power_watts)[24h:]) * 24 / 1000";
+            legendFormat = "";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "kwatth";
+            decimals = 2;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 5;
+                }
+                {
+                  color = "red";
+                  value = 10;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "none";
+        };
+      }
+      {
+        id = 2;
+        type = "gauge";
+        title = "UPS Load";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 12;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_ups_load_percent";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 70;
+                }
+                {
+                  color = "red";
+                  value = 90;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options.reduceOptions = {
+          calcs = [ "lastNotNull" ];
+          fields = "";
+          values = false;
+        };
+      }
+      {
+        id = 3;
+        type = "gauge";
+        title = "UPS Battery";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 18;
+          y = 0;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "apcupsd_battery_charge_percent";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "percent";
+            min = 0;
+            max = 100;
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "red";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 20;
+                }
+                {
+                  color = "green";
+                  value = 50;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options.reduceOptions = {
+          calcs = [ "lastNotNull" ];
+          fields = "";
+          values = false;
+        };
+      }
+
+      # -- Row 2: System --
+      {
+        id = 4;
+        type = "timeseries";
+        title = "CPU Temperature";
+        gridPos = {
+          h = 8;
+          w = 12;
+          x = 0;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = ''node_hwmon_temp_celsius{chip=~"pci.*"}'';
+            legendFormat = "CPU {{sensor}}";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "celsius";
+            color.mode = "palette-classic";
+            custom = {
+              lineWidth = 2;
+              fillOpacity = 10;
+              spanNulls = true;
+            };
+          };
+          overrides = [ ];
+        };
+      }
+      {
+        id = 5;
+        type = "stat";
+        title = "System Uptime";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 12;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "time() - node_boot_time_seconds";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            unit = "s";
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "none";
+        };
+      }
+      {
+        id = 6;
+        type = "stat";
+        title = "Jellyfin Active Streams";
+        gridPos = {
+          h = 8;
+          w = 6;
+          x = 18;
+          y = 8;
+        };
+        datasource = promDs;
+        targets = [
+          {
+            datasource = promDs;
+            expr = "jellyfin_active_streams";
+            refId = "A";
+          }
+        ];
+        fieldConfig = {
+          defaults = {
+            thresholds = {
+              mode = "absolute";
+              steps = [
+                {
+                  color = "green";
+                  value = null;
+                }
+                {
+                  color = "yellow";
+                  value = 3;
+                }
+                {
+                  color = "red";
+                  value = 6;
+                }
+              ];
+            };
+          };
+          overrides = [ ];
+        };
+        options = {
+          reduceOptions = {
+            calcs = [ "lastNotNull" ];
+            fields = "";
+            values = false;
+          };
+          colorMode = "value";
+          graphMode = "area";
+        };
+      }
+    ];
+  };
+in
+{
+  imports = [
+    (lib.serviceMountWithZpool "grafana" service_configs.zpool_ssds [
+      service_configs.grafana.dir
+    ])
+    (lib.serviceFilePerms "grafana" [
+      "Z ${service_configs.grafana.dir} 0700 grafana grafana"
+    ])
+    (lib.serviceMountWithZpool "prometheus" service_configs.zpool_ssds [
+      "/var/lib/prometheus"
+    ])
+    (lib.serviceFilePerms "prometheus" [
+      "Z /var/lib/prometheus 0700 prometheus prometheus"
+    ])
+  ];
+
+  # -- Prometheus --
+  services.prometheus = {
+    enable = true;
+    port = service_configs.ports.private.prometheus.port;
+    listenAddress = "127.0.0.1";
+    stateDir = "prometheus";
+    retentionTime = "90d";
+
+    exporters = {
+      node = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_node.port;
+        listenAddress = "127.0.0.1";
+        enabledCollectors = [
+          "hwmon"
+          "systemd"
+          "textfile"
+        ];
+        extraFlags = [
+          "--collector.textfile.directory=${textfileDir}"
+        ];
+      };
+
+      apcupsd = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_apcupsd.port;
+        listenAddress = "127.0.0.1";
+        apcupsdAddress = "127.0.0.1:3551";
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "prometheus";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus.port}" ]; }
+        ];
+      }
+      {
+        job_name = "node";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_node.port}" ]; }
+        ];
+      }
+      {
+        job_name = "apcupsd";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_apcupsd.port}" ]; }
+        ];
+      }
+    ];
+  };
+
+  # -- Grafana --
+  services.grafana = {
+    enable = true;
+    dataDir = service_configs.grafana.dir;
+
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = service_configs.ports.private.grafana.port;
+        domain = service_configs.grafana.domain;
+        root_url = "https://${service_configs.grafana.domain}";
+      };
+
+      # Caddy handles auth -- disable Grafana login entirely
+      "auth.anonymous" = {
+        enabled = true;
+        org_role = "Admin";
+      };
+      "auth.basic".enabled = false;
+      "auth".disable_login_form = true;
+
+      analytics.reporting_enabled = false;
+    };
+
+    provision = {
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://127.0.0.1:${toString service_configs.ports.private.prometheus.port}";
+            access = "proxy";
+            isDefault = true;
+            editable = false;
+            uid = "prometheus";
+          }
+        ];
+      };
+
+      dashboards.settings.providers = [
+        {
+          name = "system";
+          type = "file";
+          options.path = "/etc/grafana-dashboards";
+          disableDeletion = true;
+          updateIntervalSeconds = 60;
+        }
+      ];
+    };
+  };
+
+  # Provision dashboard JSON
+  environment.etc."grafana-dashboards/system-overview.json" = {
+    text = builtins.toJSON dashboard;
+    mode = "0444";
+  };
+
+  # Caddy reverse proxy with auth
+  services.caddy.virtualHosts."${service_configs.grafana.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.private.grafana.port}
+  '';
+
+  # -- Jellyfin metrics collector --
+  # Queries the Jellyfin API for active streams and writes a .prom file
+  # for the node_exporter textfile collector.
+  systemd.services.jellyfin-metrics-collector = {
+    description = "Collect Jellyfin metrics for Prometheus";
+    after = [ "network.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe jellyfinCollector;
+      LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
+    };
+  };
+
+  systemd.timers.jellyfin-metrics-collector = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*:*:0/30";
+      RandomizedDelaySec = "5s";
+    };
+  };
+
+  # Ensure textfile collector directory exists (tmpfs root -- recreated on boot)
+  systemd.tmpfiles.rules = [
+    "d ${textfileDir} 0755 root root -"
+  ];
+}

services/p2pool.nix

@@ -4,9 +4,6 @@
   lib,
   ...
 }:
-let
-  walletAddress = lib.strings.trim (builtins.readFile ../secrets/xmrig-wallet);
-in
 {
   imports = [
     (lib.serviceMountWithZpool "p2pool" service_configs.zpool_ssds [
@@ -20,7 +17,7 @@ in
   services.p2pool = {
     enable = true;
     dataDir = service_configs.p2pool.dataDir;
-    walletAddress = walletAddress;
+    walletAddress = service_configs.p2pool.walletAddress;
     sidechain = "nano";
     host = "127.0.0.1";
     rpcPort = service_configs.ports.public.monero_rpc.port;

services/ssh.nix

@@ -31,5 +31,8 @@
 
   # used for deploying configs to server
   users.users.root.openssh.authorizedKeys.keys =
-    config.users.users.${username}.openssh.authorizedKeys.keys;
+    config.users.users.${username}.openssh.authorizedKeys.keys
+    ++ [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5ZYN6idL/w/mUIfPOH1i+Q/SQXuzAMQUEuWpipx1Pc ci-deploy@muffin"
+    ];
 }

tests/gitea-runner.nix

@@ -0,0 +1,60 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+pkgs.testers.runNixOSTest {
+  name = "gitea-runner";
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      services.gitea = {
+        enable = true;
+        database.type = "sqlite3";
+        settings = {
+          server = {
+            HTTP_PORT = 3000;
+            ROOT_URL = "http://localhost:3000";
+            DOMAIN = "localhost";
+          };
+          actions.ENABLED = true;
+          service.DISABLE_REGISTRATION = true;
+        };
+      };
+
+      specialisation.runner = {
+        inheritParentConfig = true;
+        configuration.services.gitea-actions-runner.instances.test = {
+          enable = true;
+          name = "ci";
+          url = "http://localhost:3000";
+          labels = [ "native:host" ];
+          tokenFile = "/var/lib/gitea/runner_token";
+        };
+      };
+    };
+
+  testScript = ''
+    start_all()
+
+    machine.wait_for_unit("gitea.service")
+    machine.wait_for_open_port(3000)
+
+    # Generate runner token
+    machine.succeed(
+        "su -l gitea -s /bin/sh -c '${pkgs.gitea}/bin/gitea actions generate-runner-token --work-path /var/lib/gitea' | tail -1 | sed 's/^/TOKEN=/' > /var/lib/gitea/runner_token"
+    )
+
+    # Switch to runner specialisation
+    machine.succeed(
+        "/run/current-system/specialisation/runner/bin/switch-to-configuration test"
+    )
+
+    # Start the runner (specialisation switch doesn't auto-start new services)
+    machine.succeed("systemctl start gitea-runner-test.service")
+    machine.wait_for_unit("gitea-runner-test.service")
+    machine.succeed("sleep 5")
+    machine.succeed("test -f /var/lib/gitea-runner/test/.runner")
+  '';
+}

tests/tests.nix

@@ -27,4 +27,7 @@ in
 
   # torrent audit test
   torrentAuditTest = handleTest ./torrent-audit.nix;
+
+  # gitea runner test
+  giteaRunnerTest = handleTest ./gitea-runner.nix;
 }