fix mq-deadline for hdds: 3

.gitea/workflows/deploy.yml

@@ -0,0 +1,60 @@
+name: Build and Deploy
+on:
+  push:
+    branches: [main]
+
+jobs:
+  deploy:
+    runs-on: nix
+    env:
+      GIT_SSH_COMMAND: "ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=no"
+    steps:
+      - uses: https://github.com/actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Unlock git-crypt
+        run: |
+          git-crypt unlock /run/agenix/git-crypt-key-server-config
+
+      - name: Build NixOS configuration
+        run: |
+          nix build .#nixosConfigurations.muffin.config.system.build.toplevel -L
+
+      - name: Deploy via deploy-rs
+        run: |
+          eval $(ssh-agent -s)
+          ssh-add /run/agenix/ci-deploy-key
+          nix run github:serokell/deploy-rs -- .#muffin --skip-checks --ssh-opts="-o StrictHostKeyChecking=no"
+
+      - name: Health check
+        run: |
+          sleep 10
+          ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=no root@server-public \
+            "systemctl is-active gitea && systemctl is-active caddy && systemctl is-active continuwuity && systemctl is-active coturn"
+
+      - name: Notify success
+        if: success()
+        run: |
+          TOPIC=$(cat /run/agenix/ntfy-alerts-topic | tr -d '[:space:]')
+          TOKEN=$(cat /run/agenix/ntfy-alerts-token | tr -d '[:space:]')
+          curl -sf -X POST \
+            "https://ntfy.sigkill.computer/$TOPIC" \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Title: [muffin] Deploy succeeded" \
+            -H "Priority: default" \
+            -H "Tags: white_check_mark" \
+            -d "server-config deployed from commit ${GITHUB_SHA::8}"
+
+      - name: Notify failure
+        if: failure()
+        run: |
+          TOPIC=$(cat /run/agenix/ntfy-alerts-topic 2>/dev/null | tr -d '[:space:]')
+          TOKEN=$(cat /run/agenix/ntfy-alerts-token 2>/dev/null | tr -d '[:space:]')
+          curl -sf -X POST \
+            "https://ntfy.sigkill.computer/$TOPIC" \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Title: [muffin] Deploy FAILED" \
+            -H "Priority: urgent" \
+            -H "Tags: rotating_light" \
+            -d "server-config deploy failed at commit ${GITHUB_SHA::8}" || true

AGENTS.md

@@ -99,7 +99,11 @@ Each service file in `services/` follows this structure:
 - **git-crypt**: `secrets/` directory and `usb-secrets/usb-secrets-key*` are encrypted (see `.gitattributes`)
 - **agenix**: secrets declared in `modules/age-secrets.nix`, decrypted at runtime to `/run/agenix/`
 - **Identity**: USB drive at `/mnt/usb-secrets/usb-secrets-key`
-- **Encrypting new secrets**: The agenix encryption key is in `usb-secrets/usb-secrets-key` (SSH private key, git-crypt encrypted). To create a new secret: derive the age public key with `ssh-keygen -y -f usb-secrets/usb-secrets-key | ssh-to-age`, then encrypt with `age -r <public-key> -o secrets/<name>.age`.
+- **Encrypting new secrets**: The agenix identity is an SSH private key at `usb-secrets/usb-secrets-key` (git-crypt encrypted). To encrypt a new secret, use the SSH public key directly with `age -R`:
+  ```bash
+  age -R <(ssh-keygen -y -f usb-secrets/usb-secrets-key) -o secrets/<name>.age /path/to/plaintext
+  ```
+- **DO NOT use `ssh-to-age`**. Using `ssh-to-age` to derive a native age public key and then encrypting with `age -r age1...` produces `X25519` recipient stanzas. The SSH private key identity on the server can only decrypt `ssh-ed25519` stanzas. This mismatch causes `age: error: no identity matched any of the recipients` at deploy time. Always use `age -R` with the SSH public key directly.
 - Never read or commit plaintext secrets. Never log secret values.
 
 ### Important Patterns

configuration.nix

@@ -26,6 +26,7 @@
     ./services/caddy.nix
     ./services/immich.nix
     ./services/gitea.nix
+    ./services/gitea-actions-runner.nix
     ./services/minecraft.nix
 
     ./services/wg.nix
@@ -73,6 +74,10 @@
     ./services/mollysocket.nix
   ];
 
+  # Hosts entries for CI/CD deploy targets
+  networking.hosts."192.168.1.50" = [ "server-public" ];
+  networking.hosts."192.168.1.223" = [ "desktop" ];
+
   services.kmscon.enable = true;
 
   systemd.targets = {
@@ -249,6 +254,14 @@
 
   users.groups.${service_configs.media_group} = { };
 
+  users.users.gitea-runner = {
+    isSystemUser = true;
+    group = "gitea-runner";
+    home = "/var/lib/gitea-runner";
+    description = "Gitea Actions CI runner";
+  };
+  users.groups.gitea-runner = { };
+
   users.users.${username} = {
     isNormalUser = true;
     extraGroups = [

modules/age-secrets.nix

@@ -68,19 +68,19 @@
       group = "root";
     };
 
-    # ntfy-alerts secrets
+    # ntfy-alerts secrets (group-readable for CI runner notifications)
     ntfy-alerts-topic = {
       file = ../secrets/ntfy-alerts-topic.age;
-      mode = "0400";
+      mode = "0440";
       owner = "root";
-      group = "root";
+      group = "gitea-runner";
     };
 
     ntfy-alerts-token = {
       file = ../secrets/ntfy-alerts-token.age;
-      mode = "0400";
+      mode = "0440";
       owner = "root";
-      group = "root";
+      group = "gitea-runner";
     };
 
     # Firefox Sync server secrets (SYNC_MASTER_SECRET)
@@ -128,5 +128,36 @@
       group = "continuwuity";
     };
 
+    # CI deploy SSH key
+    ci-deploy-key = {
+      file = ../secrets/ci-deploy-key.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Git-crypt symmetric key for dotfiles repo
+    git-crypt-key-dotfiles = {
+      file = ../secrets/git-crypt-key-dotfiles.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Git-crypt symmetric key for server-config repo
+    git-crypt-key-server-config = {
+      file = ../secrets/git-crypt-key-server-config.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Gitea Actions runner registration token
+    gitea-runner-token = {
+      file = ../secrets/gitea-runner-token.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
   };
 }

modules/hardware.nix

@@ -5,6 +5,20 @@
   service_configs,
   ...
 }:
+let
+  hddTuneIosched = pkgs.writeShellScript "hdd-tune-iosched" ''
+    # Called by udev with the partition kernel name (e.g. sdb1).
+    # Derives the parent disk and applies mq-deadline iosched params.
+    parent=''${1%%[0-9]*}
+    dev="/sys/block/$parent"
+    [ -d "$dev/queue/iosched" ] || exit 0
+    echo 15000 > "$dev/queue/iosched/read_expire"
+    echo 15000 > "$dev/queue/iosched/write_expire"
+    echo 128 > "$dev/queue/iosched/fifo_batch"
+    echo 16 > "$dev/queue/iosched/writes_starved"
+    echo 4096 > "$dev/queue/max_sectors_kb" 2>/dev/null || true
+  '';
+in
 {
   boot.initrd.availableKernelModules = [
     "xhci_pci"
@@ -30,48 +44,13 @@
   # fifo_batch=128 keeps sweeps long; writes_starved=16 heavily favors reads.
   # 4 MiB readahead matches libtorrent piece extent affinity for sequential prefetch.
   #
-  # This runs as a systemd oneshot rather than udev rules because the NixOS ZFS module
-  # hardcodes a udev rule that forces scheduler="none" on all ZFS member partitions'
-  # parent disks, overriding any scheduler set via udev on the disk event.
-  systemd.services.hdd-io-tuning = {
-    description = "HDD I/O scheduler and queue tuning";
-    after = [
-      "zfs-import.target"
-      "systemd-udev-settle.service"
-    ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-    };
-    path = with pkgs; [
-      coreutils
-      gawk
-      zfs
-    ];
-    script = ''
-      # Only tune disks in the hdds pool — not all rotational disks.
-      # zpool status gives by-id device names; we resolve to /sys/block/<name>.
-      zpool status hdds | awk '/^\t/ && $1 ~ /^(ata-|nvme-|scsi-)/ {print $1}' | while read -r id; do
-        link="/dev/disk/by-id/$id"
-        [ -L "$link" ] || continue
-        name=$(basename "$(readlink -f "$link")")
-        dev="/sys/block/$name"
-        [ -d "$dev" ] || continue
-
-        echo mq-deadline > "$dev/queue/scheduler"
-        echo 4096 > "$dev/queue/read_ahead_kb"
-        echo 512 > "$dev/queue/nr_requests"
-
-        echo 15000 > "$dev/queue/iosched/read_expire"
-        echo 15000 > "$dev/queue/iosched/write_expire"
-        echo 128 > "$dev/queue/iosched/fifo_batch"
-        echo 16 > "$dev/queue/iosched/writes_starved"
-
-        echo 4096 > "$dev/queue/max_sectors_kb" 2>/dev/null || true
-
-        echo "Tuned $id -> $name: mq-deadline, 4M readahead, 15s deadlines"
-      done
-    '';
-  };
+  # The NixOS ZFS module hardcodes a udev rule that forces scheduler="none" on all
+  # ZFS member partitions' parent disks (on both add AND change events). We counter
+  # it with lib.mkAfter so our rule appears after theirs in 99-local.rules — our
+  # rule matches the same partition events and sets mq-deadline back, then a RUN
+  # script applies the iosched params. Only targets rotational, non-removable disks
+  # (i.e. HDDs, not SSDs or USB).
+  services.udev.extraRules = lib.mkAfter ''
+    ACTION=="add|change", KERNEL=="sd[a-z]*[0-9]*", ENV{ID_FS_TYPE}=="zfs_member", ATTR{../queue/rotational}=="1", ATTR{../removable}=="0", ATTR{../queue/scheduler}="mq-deadline", ATTR{../queue/read_ahead_kb}="4096", ATTR{../queue/nr_requests}="512", RUN+="${hddTuneIosched} %k"
+  '';
 }

modules/impermanence.nix

@@ -24,6 +24,7 @@
       # ZFS cache directory - persisting the directory instead of the file
       # avoids "device busy" errors when ZFS atomically updates the cache
       "/etc/zfs"
+      "/var/lib/gitea-runner"
     ];
 
     files = [

services/gitea-actions-runner.nix

@@ -0,0 +1,46 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  ...
+}:
+{
+  services.gitea-actions-runner.instances.muffin = {
+    enable = true;
+    name = "muffin";
+    url = config.services.gitea.settings.server.ROOT_URL;
+    tokenFile = config.age.secrets.gitea-runner-token.path;
+    labels = [ "nix:host" ];
+    hostPackages = with pkgs; [
+      bash
+      coreutils
+      curl
+      gawk
+      git
+      git-crypt
+      gnugrep
+      gnused
+      jq
+      nix
+      nodejs
+      openssh
+    ];
+    settings = {
+      runner = {
+        capacity = 1;
+        timeout = "3h";
+      };
+    };
+  };
+
+  # Override DynamicUser to use our static gitea-runner user
+  systemd.services."gitea-runner-muffin" = {
+    serviceConfig = {
+      DynamicUser = lib.mkForce false;
+      User = "gitea-runner";
+      Group = "gitea-runner";
+    };
+    environment.GIT_SSH_COMMAND = "ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=no";
+  };
+}

services/gitea.nix

@@ -37,6 +37,7 @@
       };
       # only I shall use gitea
       service.DISABLE_REGISTRATION = true;
+      actions.ENABLED = true;
     };
   };
 

services/ssh.nix

@@ -31,5 +31,8 @@
 
   # used for deploying configs to server
   users.users.root.openssh.authorizedKeys.keys =
-    config.users.users.${username}.openssh.authorizedKeys.keys;
+    config.users.users.${username}.openssh.authorizedKeys.keys
+    ++ [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5ZYN6idL/w/mUIfPOH1i+Q/SQXuzAMQUEuWpipx1Pc ci-deploy@muffin"
+    ];
 }

tests/gitea-runner.nix

@@ -0,0 +1,60 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+pkgs.testers.runNixOSTest {
+  name = "gitea-runner";
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      services.gitea = {
+        enable = true;
+        database.type = "sqlite3";
+        settings = {
+          server = {
+            HTTP_PORT = 3000;
+            ROOT_URL = "http://localhost:3000";
+            DOMAIN = "localhost";
+          };
+          actions.ENABLED = true;
+          service.DISABLE_REGISTRATION = true;
+        };
+      };
+
+      specialisation.runner = {
+        inheritParentConfig = true;
+        configuration.services.gitea-actions-runner.instances.test = {
+          enable = true;
+          name = "ci";
+          url = "http://localhost:3000";
+          labels = [ "native:host" ];
+          tokenFile = "/var/lib/gitea/runner_token";
+        };
+      };
+    };
+
+  testScript = ''
+    start_all()
+
+    machine.wait_for_unit("gitea.service")
+    machine.wait_for_open_port(3000)
+
+    # Generate runner token
+    machine.succeed(
+        "su -l gitea -s /bin/sh -c '${pkgs.gitea}/bin/gitea actions generate-runner-token --work-path /var/lib/gitea' | tail -1 | sed 's/^/TOKEN=/' > /var/lib/gitea/runner_token"
+    )
+
+    # Switch to runner specialisation
+    machine.succeed(
+        "/run/current-system/specialisation/runner/bin/switch-to-configuration test"
+    )
+
+    # Start the runner (specialisation switch doesn't auto-start new services)
+    machine.succeed("systemctl start gitea-runner-test.service")
+    machine.wait_for_unit("gitea-runner-test.service")
+    machine.succeed("sleep 5")
+    machine.succeed("test -f /var/lib/gitea-runner/test/.runner")
+  '';
+}

tests/tests.nix

@@ -27,4 +27,7 @@ in
 
   # torrent audit test
   torrentAuditTest = handleTest ./torrent-audit.nix;
+
+  # gitea runner test
+  giteaRunnerTest = handleTest ./gitea-runner.nix;
 }