diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..d566da3 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,102 @@ +name: Build and Deploy +on: + push: + branches: [main] + +# The runner has capacity=1 so these serialize; order matters for the +# healthcheck (muffin runs last so yarn's pull-update can test against the +# freshly-deployed harmonia if needed). + +jobs: + mreow: + runs-on: nix + steps: + - uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Unlock git-crypt + run: git-crypt unlock /run/agenix/git-crypt-key-nixos + + - name: Build mreow + run: nix build .#nixosConfigurations.mreow.config.system.build.toplevel -L + + - name: Record mreow store path + continue-on-error: true + run: | + install -d /var/lib/nix-deploy + readlink -f result > /var/lib/nix-deploy/mreow + nix-store --add-root /var/lib/nix-deploy/mreow-gcroot -r "$(readlink -f result)" + + yarn: + runs-on: nix + steps: + - uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Unlock git-crypt + run: git-crypt unlock /run/agenix/git-crypt-key-nixos + + - name: Build yarn + run: nix build .#nixosConfigurations.yarn.config.system.build.toplevel -L + + - name: Record yarn store path for pull-update + continue-on-error: true + run: | + install -d /var/lib/nix-deploy + readlink -f result > /var/lib/nix-deploy/yarn + nix-store --add-root /var/lib/nix-deploy/yarn-gcroot -r "$(readlink -f result)" + + muffin: + runs-on: nix + env: + GIT_SSH_COMMAND: "ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts" + steps: + - uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Unlock git-crypt + run: git-crypt unlock /run/agenix/git-crypt-key-nixos + + - name: Build muffin + run: nix build .#nixosConfigurations.muffin.config.system.build.toplevel -L + + - name: Deploy via deploy-rs + run: | + eval $(ssh-agent -s) + ssh-add /run/agenix/ci-deploy-key + nix run github:serokell/deploy-rs -- .#muffin --skip-checks --ssh-opts="-o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts" + + - name: Health check + run: | + sleep 10 + ssh -i /run/agenix/ci-deploy-key -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ci-known-hosts root@server-public \ + "systemctl is-active gitea && systemctl is-active caddy && systemctl is-active continuwuity && systemctl is-active coturn" + + - name: Notify success + if: success() + run: | + TOPIC=$(cat /run/agenix/ntfy-alerts-topic | tr -d '[:space:]') + TOKEN=$(cat /run/agenix/ntfy-alerts-token | tr -d '[:space:]') + curl -sf -o /dev/null -X POST \ + "https://ntfy.sigkill.computer/$TOPIC" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Title: [muffin] Deploy succeeded" \ + -H "Priority: default" \ + -H "Tags: white_check_mark" \ + -d "nixos deployed from commit ${GITHUB_SHA::8}" + + - name: Notify failure + if: failure() + run: | + TOPIC=$(cat /run/agenix/ntfy-alerts-topic 2>/dev/null | tr -d '[:space:]') + TOKEN=$(cat /run/agenix/ntfy-alerts-token 2>/dev/null | tr -d '[:space:]') + curl -sf -o /dev/null -X POST \ + "https://ntfy.sigkill.computer/$TOPIC" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Title: [muffin] Deploy FAILED" \ + -H "Priority: urgent" \ + -H "Tags: rotating_light" \ + -d "nixos muffin deploy failed at commit ${GITHUB_SHA::8}" || true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6bad30c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,181 @@ +# AGENTS.md + +## Project Overview + +Unified NixOS flake for three hosts: + +| Host | Role | nixpkgs channel | Activation | +|------|------|----------------|-----------| +| `mreow` | Framework 13 AMD AI 300 laptop (niri, greetd, swaylock) | `nixos-unstable` | `./deploy.sh` locally | +| `yarn` | AMD Zen 5 desktop (niri + Jovian-NixOS Steam deck mode, impermanence) | `nixos-unstable` | pull from CI binary cache | +| `muffin` | AMD Zen 3 server (Caddy, ZFS, agenix, deploy-rs, 25+ services) | `nixos-25.11` | deploy-rs from CI | + +One `flake.nix` declares both channels (`nixpkgs` and `nixpkgs-stable`) and composes each host from the correct channel. No single-channel migration is intended. + +History pre-dating this repo lives in the merged subtree branches from `dotfiles` (commit `e9a44f6`) and `server-config` (commit `4bc5d57`). Use `git log ` (without `--follow`) and traverse back through the merge commits `dc481c2` and `6448a04` for pre-unify history. + +## Layout + +``` +flake.nix # 3 hosts, 2 channels +deploy.sh # wrapper: current-host rebuild or `muffin` deploy-rs +hosts// # host entrypoints (default.nix, home.nix, disk.nix, …) +modules/ # flat namespace; see module naming below + common-*.nix # imported by ALL hosts (nix settings, doas, fish shim) + desktop-*.nix # imported by mreow/yarn only + server-*.nix # imported by muffin only + .nix # scoped by filename (age-secrets, zfs, no-rgb, …) +home/ + profiles/{gui,desktop,no-gui}.nix # home-manager profiles + progs/.nix # one file per program (fish, helix, niri, zen/, emacs, …) + util/.nix # small derivations +services/ # muffin-only: caddy, jellyfin, gitea, matrix, monero, … +tests/ # pkgs.testers.runNixOSTest suite +lib/ + default.nix # extends nixpkgs-stable.lib with mkCaddyReverseProxy, serviceMountWithZpool, … + overlays.nix # jellyfin-exporter, igpu-exporter, reflac, ensureZfsMounts +patches/nixpkgs/ # applied to nixpkgs-stable for muffin builds +secrets/ + desktop/ # git-crypt: mreow + yarn share these (wifi, nix-cache-netrc, secureboot.tar, password-hash, disk-password) + home/ # git-crypt: per-user HM secrets (api keys, steam id) + server/ # agenix *.age + git-crypt *.nix/*.tar/livekit_keys + usb-secrets/ # USB-resident agenix identity key (git-crypt inside the repo) +``` + +**Never read or write files under `secrets/`.** They are encrypted at rest (git-crypt for plaintext, agenix for `.age`). The git-crypt key is delivered to `muffin` at runtime as `/run/agenix/git-crypt-key-nixos.age`. + +## Build & Deploy + +```sh +# --- from any host --- +nix fmt # nixfmt-tree +nix flake update # bump both channels + inputs +nix flake update --input-name nixpkgs # bump just desktops' channel +nix flake update --input-name nixpkgs-stable # bump just muffin's channel + +# --- per-host eval / build (add -L for verbose logs) --- +nix build .#nixosConfigurations.mreow.config.system.build.toplevel -L +nix build .#nixosConfigurations.yarn.config.system.build.toplevel -L +nix build .#nixosConfigurations.muffin.config.system.build.toplevel -L + +# --- quick eval without building --- +nix eval .#nixosConfigurations.muffin.config.system.build.toplevel --no-build 2>&1 | head -5 + +# --- activate on current host (mreow / yarn only) --- +./deploy.sh # boot (default; next reboot) +./deploy.sh switch # apply immediately +./deploy.sh test # apply without boot entry +./deploy.sh build # build only + +# --- deploy to muffin from anywhere --- +./deploy.sh muffin +# equivalent to: +nix run .#deploy -- .#muffin + +# --- tests (muffin) --- +nix build .#packages.x86_64-linux.tests -L # all tests (slow) +nix build .#test-zfsTest -L # one test by name +# test names are the keys of tests/tests.nix; pattern is test- +``` + +No unit tests for desktop configs. Validation is the `nix build` exit code plus the successful `nix-diff` against the previous generation. + +If Nix complains about a missing file, `git add` it first — flakes only see tracked files. + +## Module naming + +| Prefix | Meaning | Example | +|--------|---------|---------| +| `common-` | imported by ALL hosts | `common-doas.nix`, `common-nix.nix`, `common-shell-fish.nix` | +| `desktop-` | imported by mreow + yarn only | `desktop-common.nix`, `desktop-steam.nix`, `desktop-networkmanager.nix` | +| `server-` | imported by muffin only | `server-security.nix`, `server-power.nix`, `server-impermanence.nix`, `server-lanzaboote-agenix.nix` | +| *(none)* | host-specific filename-scoped; see file contents | `age-secrets.nix`, `zfs.nix`, `no-rgb.nix` (yarn + muffin) | + +New modules: pick the narrowest prefix that's true, then add the import explicitly in the host's `default.nix` (there is no auto-discovery). + +## Code style + +- **Formatter**: `nixfmt-tree` (declared in `flake.nix`). Run `nix fmt` before every commit. +- **Indentation**: 2 spaces, enforced by the formatter. +- **Function args**: one per line, trailing comma, always end with `...`: + ```nix + { + config, + lib, + pkgs, + username, + ... + }: + ``` +- **Imports**: relative paths, one per line. Use the `../../modules/` style from `hosts/`; do not invent new aggregator modules unless more than one host uses the aggregation. +- **Package paths**: `lib.getExe pkgs.foo` over `"${pkgs.foo}/bin/foo"` when the derivation declares `meta.mainProgram`. +- **Unfree packages**: allowlisted per-module via `nixpkgs.config.allowUnfreePredicate`. Do not add a global permit. +- **Comments**: lowercase, `#` style. Use `# TODO!` / `# BUG!` / `# FIX:` prefixes for known issues that should be searchable. +- **No trailing commas** (Nix syntax forbids them). +- **`lib.mkDefault` / `lib.mkForce`**: prefer `mkDefault` in shared modules so hosts can override without fighting priority; use `mkForce` only to beat inherited defaults you can't reach any other way. + +## Secrets + +- **git-crypt** covers `secrets/**` per the root `.gitattributes`. Initialized with a single symmetric key checked into `secrets/server/git-crypt-key-nixos.age` (agenix-encrypted to the USB SSH identity). +- **agenix** decrypts `secrets/server/*.age` at activation into `/run/agenix/` on muffin. +- **USB identity**: `/mnt/usb-secrets/usb-secrets-key` on muffin; the age identity path is wired in `modules/usb-secrets.nix`. +- **Encrypting a new agenix secret** uses the SSH public key directly with `age -R`: + ```sh + age -R <(ssh-keygen -y -f secrets/usb-secrets/usb-secrets-key) \ + -o secrets/server/.age \ + /path/to/plaintext + ``` +- **DO NOT use `ssh-to-age`**. It produces `X25519` recipient stanzas, which the SSH private key on muffin cannot decrypt (it only decrypts `ssh-ed25519` stanzas produced by `age -R` against the SSH pubkey). Mismatched stanzas show up as `age: error: no identity matched any of the recipients` at deploy time. +- Never read or commit plaintext secrets. Never log secret values. + +## Service pattern (muffin) + +Each file under `services/` follows this shape: + +1. `imports` block with `lib.serviceMountWithZpool` and (optionally) `lib.serviceFilePerms`. +2. The service configuration (`services. = { … }`). +3. Caddy reverse-proxy vhost (usually via `lib.mkCaddyReverseProxy` in `lib/default.nix`). +4. Firewall rules (`networking.firewall.allowed{TCP,UDP}Ports`) if externally reachable. +5. `services.fail2ban.jails.` if the service authenticates users. + +Custom lib helpers (in `lib/default.nix`) to prefer over reinventing: + +- `lib.serviceMountWithZpool [dirs]` +- `lib.serviceFilePerms [tmpfilesRules]` +- `lib.optimizePackage ` — applies `-O3 -march=znver3 -mtune=znver3` +- `lib.vpnNamespaceOpenPort ` — confines service to the WireGuard namespace +- `lib.mkCaddyReverseProxy { subdomain|domain, port, auth ? false, vpn ? false }` +- `lib.mkFail2banJail { name, unitName ? "${name}.service", failregex }` +- `lib.mkGrafanaAnnotationService { name, description, script, after ? [], environment ? {}, loadCredential ? null }` +- `lib.extractArrApiKey ` — shell snippet to read the `` element + +Hard requirements that are asserted at eval time: + +- **Port uniqueness**: every port in `hosts/muffin/service-configs.nix` `ports.{public,private}` must be unique. The flake asserts this. +- **Public/private segregation**: public ports must appear in the firewall allow-list; private ports must not. The flake asserts both directions. +- **Hugepages**: services that need 2 MiB hugepages declare their budget in `service-configs.nix` under `hugepages_2m.services`. The `vm.nr_hugepages` sysctl is derived from the total. +- **PostgreSQL-first**: any service that supports PostgreSQL uses it (via peer-auth Unix socket when possible). Do not reach for embedded H2/SQLite. + +## Technical details + +- **Privilege escalation**: `doas` everywhere; `sudo` is disabled on every host. +- **Shell**: fish. `bash` login shells re-exec into fish via `programs.bash.interactiveShellInit` (see `modules/common-shell-fish.nix`). +- **Secure boot**: lanzaboote. Desktops extract keys from `secrets/desktop/secureboot.tar`; muffin extracts from an agenix-decrypted tar (see `modules/server-lanzaboote-agenix.nix`). +- **Impermanence**: muffin is tmpfs-root with `/persistent` surviving reboots (`modules/server-impermanence.nix`); yarn binds `/home/primary` from `/persistent` (`hosts/yarn/impermanence.nix`). +- **Disks**: disko. +- **Binary cache**: muffin runs harmonia; desktops consume it at `https://nix-cache.sigkill.computer`. +- **Kernel**: + - Desktops: `linux-cachyos-bore-lto`, `processorOpt = "x86_64-v3"` (see `modules/desktop-common.nix` — also trims ~80 legacy subsystems). + - muffin: `linuxPackages_6_12` (pinned; 6.18 has a ZFS deadlock in `dbuf_evict`). +- **Domain**: `sigkill.computer`. The old `gardling.com` redirects automatically. + +## Agent-specific instructions + +- If instructed to commit, **disable GPG signing** (`git commit --no-gpg-sign`). The author's GPG key is not available in this environment. +- Use `nix-shell -p ` if a tool is missing from the environment. +- For `nix build`, always append `-L` for verbose logs. +- If Nix reports a missing file, run `git add ` first — flakes only see git-tracked files. +- Do not read files under `secrets/`. +- Run `nix fmt` after editing any `.nix` file. +- Validate every change with `nix build .#nixosConfigurations..config.system.build.toplevel -L`. +- Commit messages are terse, lowercase; prefix with `:` when narrowly scoped (`caddy: add redirect`, `zfs: remove unneeded options`, `mreow: bump kernel`). Generic changes use `update` or a short description. diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..40dc5d7 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Wrapper around nixos-rebuild and deploy-rs for the three hosts. +# +# Usage: +# ./deploy.sh # nixos-rebuild boot on current host (mreow/yarn) +# ./deploy.sh switch # apply immediately on current host +# ./deploy.sh test # apply without adding boot entry +# ./deploy.sh build # build only, no activation +# ./deploy.sh muffin # build + deploy to muffin via deploy-rs +# +# muffin cannot be rebuilt locally from another host — this script only issues +# the remote deploy via deploy-rs when explicitly named. + +set -eu + +host="$(hostname -s)" +arg="${1:-boot}" + +case "$arg" in + muffin) + exec nix run .#deploy -- .#muffin "$@" + ;; + boot | switch | test | build) + exec nixos-rebuild "$arg" --flake ".#$host" --use-remote-sudo + ;; + *) + echo "usage: $0 [muffin | boot | switch | test | build]" >&2 + exit 2 + ;; +esac