Add 'legacy/server-config/' from commit '4bc5d57fa69a393877e7019d7673ceb33c3ab4b4'

git-subtree-dir: legacy/server-config
git-subtree-mainline: dc481c24b0
git-subtree-split: 4bc5d57fa6

legacy/server-config/modules/age-secrets.nix

@@ -0,0 +1,203 @@
+{
+  config,
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.agenix.nixosModules.default
+  ];
+
+  # Configure all agenix secrets
+  age.secrets = {
+    # ZFS encryption key
+    # path is set to /etc/zfs-key to match the ZFS dataset keylocation property
+    zfs-key = {
+      file = ../secrets/zfs-key.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+      path = "/etc/zfs-key";
+    };
+
+    # Secureboot keys archive
+    secureboot-tar = {
+      file = ../secrets/secureboot.tar.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # System passwords
+    hashedPass = {
+      file = ../secrets/hashedPass.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Service authentication
+    caddy_auth = {
+      file = ../secrets/caddy_auth.age;
+      mode = "0400";
+      owner = "caddy";
+      group = "caddy";
+    };
+
+    # Njalla API token (NJALLA_API_TOKEN=...) for Caddy DNS-01 challenge
+    njalla-api-token-env = {
+      file = ../secrets/njalla-api-token-env.age;
+      mode = "0400";
+      owner = "caddy";
+      group = "caddy";
+    };
+
+    # ddns-updater config.json with Njalla provider credentials
+    ddns-updater-config = {
+      file = ../secrets/ddns-updater-config.age;
+      mode = "0400";
+      owner = "ddns-updater";
+      group = "ddns-updater";
+    };
+
+    jellyfin-api-key = {
+      file = ../secrets/jellyfin-api-key.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    slskd_env = {
+      file = ../secrets/slskd_env.age;
+      mode = "0500";
+      owner = config.services.slskd.user;
+      group = config.services.slskd.group;
+    };
+
+    # Network configuration
+    wg0-conf = {
+      file = ../secrets/wg0.conf.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # ntfy-alerts secrets (group-readable for CI runner notifications)
+    ntfy-alerts-topic = {
+      file = ../secrets/ntfy-alerts-topic.age;
+      mode = "0440";
+      owner = "root";
+      group = "gitea-runner";
+    };
+
+    ntfy-alerts-token = {
+      file = ../secrets/ntfy-alerts-token.age;
+      mode = "0440";
+      owner = "root";
+      group = "gitea-runner";
+    };
+
+    # Firefox Sync server secrets (SYNC_MASTER_SECRET)
+    firefox-syncserver-env = {
+      file = ../secrets/firefox-syncserver-env.age;
+      mode = "0400";
+    };
+
+    # MollySocket env (MOLLY_VAPID_PRIVKEY + MOLLY_ALLOWED_UUIDS)
+    mollysocket-env = {
+      file = ../secrets/mollysocket-env.age;
+      mode = "0400";
+    };
+
+    # Murmur (Mumble) server password
+    murmur-password-env = {
+      file = ../secrets/murmur-password-env.age;
+      mode = "0400";
+      owner = "murmur";
+      group = "murmur";
+    };
+
+    # Coturn static auth secret
+    coturn-auth-secret = {
+      file = ../secrets/coturn-auth-secret.age;
+      mode = "0400";
+      owner = "turnserver";
+      group = "turnserver";
+    };
+
+    # Matrix (continuwuity) registration token
+    matrix-reg-token = {
+      file = ../secrets/matrix-reg-token.age;
+      mode = "0400";
+      owner = "continuwuity";
+      group = "continuwuity";
+    };
+
+    # Matrix (continuwuity) TURN secret — same secret as coturn-auth-secret,
+    # decrypted separately so continuwuity can read it with its own ownership
+    matrix-turn-secret = {
+      file = ../secrets/coturn-auth-secret.age;
+      mode = "0400";
+      owner = "continuwuity";
+      group = "continuwuity";
+    };
+
+    # CI deploy SSH key
+    ci-deploy-key = {
+      file = ../secrets/ci-deploy-key.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Git-crypt symmetric key for dotfiles repo
+    git-crypt-key-dotfiles = {
+      file = ../secrets/git-crypt-key-dotfiles.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Git-crypt symmetric key for server-config repo
+    git-crypt-key-server-config = {
+      file = ../secrets/git-crypt-key-server-config.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # Gitea Actions runner registration token
+    gitea-runner-token = {
+      file = ../secrets/gitea-runner-token.age;
+      mode = "0400";
+      owner = "gitea-runner";
+      group = "gitea-runner";
+    };
+
+    # llama-cpp API key for bearer token auth
+    llama-cpp-api-key = {
+      file = ../secrets/llama-cpp-api-key.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Harmonia binary cache signing key
+    harmonia-sign-key = {
+      file = ../secrets/harmonia-sign-key.age;
+      mode = "0400";
+      owner = "harmonia";
+      group = "harmonia";
+    };
+
+    # Caddy basic auth for nix binary cache (separate from main caddy_auth)
+    nix-cache-auth = {
+      file = ../secrets/nix-cache-auth.age;
+      mode = "0400";
+      owner = "caddy";
+      group = "caddy";
+    };
+  };
+}

legacy/server-config/modules/hardware.nix

@@ -0,0 +1,62 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  ...
+}:
+let
+  hddTuneIosched = pkgs.writeShellScript "hdd-tune-iosched" ''
+    # Called by udev with the partition kernel name (e.g. sdb1).
+    # Derives the parent disk and applies mq-deadline iosched params.
+    parent=''${1%%[0-9]*}
+    dev="/sys/block/$parent"
+    [ -d "$dev/queue/iosched" ] || exit 0
+    echo 500 > "$dev/queue/iosched/read_expire"
+    echo 15000 > "$dev/queue/iosched/write_expire"
+    echo 128 > "$dev/queue/iosched/fifo_batch"
+    echo 16 > "$dev/queue/iosched/writes_starved"
+    echo 4096 > "$dev/queue/max_sectors_kb" 2>/dev/null || true
+  '';
+in
+{
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "ahci"
+    "usb_storage"
+    "usbhid"
+    "sd_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  swapDevices = [ ];
+
+  hardware.cpu.amd.updateMicrocode = true;
+  hardware.enableRedistributableFirmware = true;
+
+  # HDD I/O tuning for torrent seeding workload (high-concurrency random reads)
+  # sharing the pool with latency-sensitive sequential reads (Jellyfin playback).
+  #
+  # mq-deadline sorts requests into elevator sweeps, reducing seek distance.
+  # read_expire=500ms keeps reads bounded so a Jellyfin segment can't queue for
+  # seconds behind a torrent burst; write_expire=15s lets the scheduler batch
+  # writes for coalescence (torrent writes are async and tolerate delay).
+  # The bulk of read coalescence already happens above the scheduler via ZFS
+  # aggregation (zfs_vdev_aggregation_limit=4M, read_gap_limit=128K,
+  # async_read_max=32), so the scheduler deadline only needs to be large enough
+  # to keep the elevator sweep coherent -- 500ms is plenty on rotational disks.
+  # fifo_batch=128 keeps sweeps long; writes_starved=16 heavily favors reads.
+  # 4 MiB readahead matches libtorrent piece extent affinity for sequential prefetch.
+  #
+  # The NixOS ZFS module hardcodes a udev rule that forces scheduler="none" on all
+  # ZFS member partitions' parent disks (on both add AND change events). We counter
+  # it with lib.mkAfter so our rule appears after theirs in 99-local.rules — our
+  # rule matches the same partition events and sets mq-deadline back, then a RUN
+  # script applies the iosched params. Only targets rotational, non-removable disks
+  # (i.e. HDDs, not SSDs or USB).
+  services.udev.extraRules = lib.mkAfter ''
+    ACTION=="add|change", KERNEL=="sd[a-z]*[0-9]*", ENV{ID_FS_TYPE}=="zfs_member", ATTR{../queue/rotational}=="1", ATTR{../removable}=="0", ATTR{../queue/scheduler}="mq-deadline", ATTR{../queue/read_ahead_kb}="4096", ATTR{../queue/nr_requests}="512", RUN+="${hddTuneIosched} %k"
+  '';
+}

legacy/server-config/modules/home.nix

@@ -0,0 +1,31 @@
+{
+  pkgs,
+  lib,
+  ...
+}:
+{
+  home.stateVersion = "24.11";
+  programs.fish = {
+    enable = true;
+
+    interactiveShellInit = ''
+      # disable greeting
+      set fish_greeting
+
+      # pfetch on shell start (disable pkgs because of execution time)
+      PF_INFO="ascii title os host kernel uptime memory editor wm" ${lib.getExe pkgs.pfetch-rs}
+    '';
+
+    shellAliases =
+      let
+        eza = "${lib.getExe pkgs.eza} --color=always --group-directories-first";
+      in
+      {
+        # from DistroTube's dot files: Changing "ls" to "eza"
+        ls = "${eza} -al";
+        la = "${eza} -a";
+        ll = "${eza} -l";
+        lt = "${eza} -aT";
+      };
+  };
+}

legacy/server-config/modules/impermanence.nix

@@ -0,0 +1,71 @@
+{
+  config,
+  lib,
+  pkgs,
+  username,
+  service_configs,
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.impermanence.nixosModules.impermanence
+  ];
+
+  environment.persistence."/persistent" = {
+    hideMounts = true;
+    directories = [
+      "/var/log"
+      "/var/lib/systemd/coredump"
+      "/var/lib/nixos"
+
+      "/var/lib/systemd/timers"
+
+      # ZFS cache directory - persisting the directory instead of the file
+      # avoids "device busy" errors when ZFS atomically updates the cache
+      "/etc/zfs"
+      "/var/lib/gitea-runner"
+    ];
+
+    files = [
+      # Machine ID
+      "/etc/machine-id"
+    ];
+
+    users.${username} = {
+      files = [
+        ".local/share/fish/fish_history"
+      ];
+    };
+
+    users.root = {
+      files = [
+        ".local/share/fish/fish_history"
+      ];
+    };
+  };
+
+  # Store SSH host keys directly in /persistent to survive tmpfs root wipes.
+  # This is more reliable than bind mounts for service-generated files.
+  services.openssh.hostKeys = [
+    {
+      path = "/persistent/etc/ssh/ssh_host_ed25519_key";
+      type = "ed25519";
+    }
+    {
+      path = "/persistent/etc/ssh/ssh_host_rsa_key";
+      type = "rsa";
+      bits = 4096;
+    }
+  ];
+
+  # Enforce root ownership on /persistent/etc. The impermanence activation
+  # script copies ownership from /persistent/etc to /etc via
+  # `chown --reference`. If /persistent/etc ever gets non-root ownership,
+  # sshd StrictModes rejects /etc/ssh/authorized_keys.d/root and root SSH
+  # breaks while non-root users still work.
+  # Use "z" (set ownership, non-recursive) not "d" (create only, no-op on existing).
+  systemd.tmpfiles.rules = [
+    "z /persistent/etc 0755 root root"
+  ];
+}

legacy/server-config/modules/lib.nix

@@ -0,0 +1,287 @@
+{
+  inputs,
+  pkgs,
+  service_configs,
+  ...
+}:
+inputs.nixpkgs.lib.extend (
+  final: prev:
+  let
+    lib = prev;
+  in
+  {
+    optimizeWithFlags =
+      pkg: flags:
+      pkg.overrideAttrs (old: {
+        env = (old.env or { }) // {
+          NIX_CFLAGS_COMPILE =
+            (old.env.NIX_CFLAGS_COMPILE or old.NIX_CFLAGS_COMPILE or "")
+            + " "
+            + (lib.concatStringsSep " " flags);
+        };
+      });
+
+    optimizePackage =
+      pkg:
+      final.optimizeWithFlags pkg [
+        "-O3"
+        "-march=${service_configs.cpu_arch}"
+        "-mtune=${service_configs.cpu_arch}"
+      ];
+
+    vpnNamespaceOpenPort =
+      port: service:
+      { ... }:
+      {
+        vpnNamespaces.wg = {
+          portMappings = [
+            {
+              from = port;
+              to = port;
+            }
+          ];
+
+          openVPNPorts = [
+            {
+              port = port;
+              protocol = "both";
+            }
+          ];
+        };
+        systemd.services.${service}.vpnConfinement = {
+          enable = true;
+          vpnNamespace = "wg";
+        };
+      };
+
+    serviceMountWithZpool =
+      serviceName: zpool: dirs:
+      { pkgs, config, ... }:
+      {
+        systemd.services."${serviceName}-mounts" = {
+          wants = [
+            "zfs.target"
+            "zfs-mount.service"
+          ]
+          ++ lib.optionals (zpool != "") [ "zfs-import-${zpool}.service" ];
+          after = [ "zfs-mount.service" ] ++ lib.optionals (zpool != "") [ "zfs-import-${zpool}.service" ];
+          before = [ "${serviceName}.service" ];
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = [
+              (lib.getExe (
+                pkgs.writeShellApplication {
+                  name = "ensure-zfs-mounts-with-pool-${serviceName}-${zpool}";
+                  runtimeInputs = with pkgs; [
+                    gawk
+                    coreutils
+                    config.boot.zfs.package
+                  ];
+
+                  text = ''
+                    set -euo pipefail
+
+                    echo "Ensuring ZFS mounts for service: ${serviceName} (pool: ${zpool})"
+                    echo "Directories: ${lib.strings.concatStringsSep ", " dirs}"
+
+                    # Validate mounts exist (ensureZfsMounts already has proper PATH)
+                    ${lib.getExe pkgs.ensureZfsMounts} ${lib.strings.concatStringsSep " " dirs}
+
+                    # Additional runtime check: verify paths are on correct zpool
+                    ${lib.optionalString (zpool != "") ''
+                      echo "Verifying ZFS mountpoints are on pool '${zpool}'..."
+
+                      if ! zfs_list_output=$(zfs list -H -o name,mountpoint 2>&1); then
+                        echo "ERROR: Failed to query ZFS datasets: $zfs_list_output" >&2
+                        exit 1
+                      fi
+
+                      # shellcheck disable=SC2043
+                      for target in ${lib.strings.concatStringsSep " " dirs}; do
+                        echo "Checking: $target"
+
+                        # Find dataset that has this mountpoint
+                        dataset=$(echo "$zfs_list_output" | awk -v target="$target" '$2 == target {print $1; exit}')
+
+                        if [ -z "$dataset" ]; then
+                          echo "ERROR: No ZFS dataset found for mountpoint: $target" >&2
+                          exit 1
+                        fi
+
+                        # Extract pool name from dataset (first part before /)
+                        actual_pool=$(echo "$dataset" | cut -d'/' -f1)
+
+                        if [ "$actual_pool" != "${zpool}" ]; then
+                          echo "ERROR: ZFS pool mismatch for $target" >&2
+                          echo " Expected pool: ${zpool}" >&2
+                          echo " Actual pool:   $actual_pool" >&2
+                          echo " Dataset:       $dataset" >&2
+                          exit 1
+                        fi
+
+                        echo "$target is on $dataset (pool: $actual_pool)"
+                      done
+
+                      echo "All paths verified successfully on pool '${zpool}'"
+                    ''}
+
+                    echo "Mount validation completed for ${serviceName} (pool: ${zpool})"
+                  '';
+                }
+              ))
+            ];
+          };
+        };
+
+        systemd.services.${serviceName} = {
+          wants = [
+            "${serviceName}-mounts.service"
+          ];
+          after = [
+            "${serviceName}-mounts.service"
+          ];
+          requires = [
+            "${serviceName}-mounts.service"
+          ];
+        };
+
+        # assert that the pool is even enabled
+        #assertions = lib.optionals (zpool != "") [
+        #  {
+        #    assertion = builtins.elem zpool config.boot.zfs.extraPools;
+        #    message = "${zpool} is not enabled in `boot.zfs.extraPools`";
+        #  }
+        #];
+      };
+
+    serviceFilePerms =
+      serviceName: tmpfilesRules:
+      { pkgs, ... }:
+      let
+        confFile = pkgs.writeText "${serviceName}-file-perms.conf" (
+          lib.concatStringsSep "\n" tmpfilesRules
+        );
+      in
+      {
+        systemd.services."${serviceName}-file-perms" = {
+          after = [ "${serviceName}-mounts.service" ];
+          before = [ "${serviceName}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = "${pkgs.systemd}/bin/systemd-tmpfiles --create ${confFile}";
+          };
+        };
+
+        systemd.services.${serviceName} = {
+          wants = [ "${serviceName}-file-perms.service" ];
+          after = [ "${serviceName}-file-perms.service" ];
+        };
+      };
+    # Creates a Caddy virtualHost with reverse_proxy to a local or VPN-namespaced port.
+    # Use `subdomain` for "<name>.${domain}" or `domain` for a full custom domain.
+    # Exactly one of `subdomain` or `domain` must be provided.
+    mkCaddyReverseProxy =
+      {
+        subdomain ? null,
+        domain ? null,
+        port,
+        auth ? false,
+        vpn ? false,
+      }:
+      assert (subdomain != null) != (domain != null);
+      { config, ... }:
+      let
+        vhostDomain = if domain != null then domain else "${subdomain}.${service_configs.https.domain}";
+        upstream =
+          if vpn then
+            "${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString port}"
+          else
+            ":${builtins.toString port}";
+      in
+      {
+        services.caddy.virtualHosts."${vhostDomain}".extraConfig = lib.concatStringsSep "\n" (
+          lib.optional auth "import ${config.age.secrets.caddy_auth.path}" ++ [ "reverse_proxy ${upstream}" ]
+        );
+      };
+
+    # Creates a fail2ban jail with systemd journal backend.
+    # Covers the common pattern: journal-based detection, http/https ports, default thresholds.
+    mkFail2banJail =
+      {
+        name,
+        unitName ? "${name}.service",
+        failregex,
+      }:
+      { ... }:
+      {
+        services.fail2ban.jails.${name} = {
+          enabled = true;
+          settings = {
+            backend = "systemd";
+            port = "http,https";
+            # defaults: maxretry=5, findtime=10m, bantime=10m
+          };
+          filter.Definition = {
+            inherit failregex;
+            ignoreregex = "";
+            journalmatch = "_SYSTEMD_UNIT=${unitName}";
+          };
+        };
+      };
+
+    # Creates a hardened Grafana annotation daemon service.
+    # Provides DynamicUser, sandboxing, state directory, and GRAFANA_URL/STATE_FILE automatically.
+    mkGrafanaAnnotationService =
+      {
+        name,
+        description,
+        script,
+        after ? [ ],
+        environment ? { },
+        loadCredential ? null,
+      }:
+      {
+        systemd.services."${name}-annotations" = {
+          inherit description;
+          after = [
+            "network.target"
+            "grafana.service"
+          ]
+          ++ after;
+          wantedBy = [ "multi-user.target" ];
+          serviceConfig = {
+            ExecStart = "${pkgs.python3}/bin/python3 ${script}";
+            Restart = "always";
+            RestartSec = "10s";
+            DynamicUser = true;
+            StateDirectory = "${name}-annotations";
+            NoNewPrivileges = true;
+            ProtectSystem = "strict";
+            ProtectHome = true;
+            PrivateTmp = true;
+            RestrictAddressFamilies = [
+              "AF_INET"
+              "AF_INET6"
+            ];
+            MemoryDenyWriteExecute = true;
+          }
+          // lib.optionalAttrs (loadCredential != null) {
+            LoadCredential = loadCredential;
+          };
+          environment = {
+            GRAFANA_URL = "http://127.0.0.1:${toString service_configs.ports.private.grafana.port}";
+            STATE_FILE = "/var/lib/${name}-annotations/state.json";
+          }
+          // environment;
+        };
+      };
+
+    # Shell command to extract an API key from an *arr config.xml file.
+    # Returns a string suitable for $() command substitution in shell scripts.
+    extractArrApiKey =
+      configXmlPath: "${lib.getExe pkgs.gnugrep} -oP '(?<=<ApiKey>)[^<]+' ${configXmlPath}";
+  }
+)

legacy/server-config/modules/no-rgb.nix

@@ -0,0 +1,66 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  systemd.services.no-rgb =
+    let
+      no-rgb = (
+        pkgs.writeShellApplication {
+          name = "no-rgb";
+          runtimeInputs = with pkgs; [
+            openrgb
+            coreutils
+            gnugrep
+          ];
+
+          text = ''
+            # Retry loop to wait for hardware to be ready
+            NUM_DEVICES=0
+            for attempt in 1 2 3 4 5; do
+              DEVICE_LIST=$(openrgb --noautoconnect --list-devices 2>/dev/null) || DEVICE_LIST=""
+              NUM_DEVICES=$(echo "$DEVICE_LIST" | grep -cE '^[0-9]+: ') || NUM_DEVICES=0
+              if [ "$NUM_DEVICES" -gt 0 ]; then
+                break
+              fi
+              if [ "$attempt" -lt 5 ]; then
+                sleep 2
+              fi
+            done
+
+            # If no devices found after retries, exit gracefully
+            if [ "$NUM_DEVICES" -eq 0 ]; then
+              exit 0
+            fi
+
+            # Disable RGB on each device
+            for i in $(seq 0 $((NUM_DEVICES - 1))); do
+              openrgb --noautoconnect --device "$i" --mode direct --color 000000 || true
+            done
+          '';
+        }
+      );
+    in
+    {
+      description = "disable rgb";
+      after = [ "systemd-udev-settle.service" ];
+      serviceConfig = {
+        ExecStart = lib.getExe no-rgb;
+        Type = "oneshot";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+
+  services.hardware.openrgb = {
+    enable = true;
+    package = pkgs.openrgb-with-all-plugins;
+    motherboard = "amd";
+  };
+
+  services.udev.packages = [ pkgs.openrgb-with-all-plugins ];
+  hardware.i2c.enable = true;
+}

legacy/server-config/modules/ntfy-alerts.nix

@@ -0,0 +1,132 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.ntfyAlerts;
+
+  curl = "${pkgs.curl}/bin/curl";
+  hostname = config.networking.hostName;
+
+  # Build the curl auth args as a proper bash array fragment
+  authCurlArgs =
+    if cfg.tokenFile != null then
+      ''
+        if [ -f "${cfg.tokenFile}" ]; then
+          TOKEN=$(cat "${cfg.tokenFile}" 2>/dev/null || echo "")
+          if [ -n "$TOKEN" ]; then
+            AUTH_ARGS=(-H "Authorization: Bearer $TOKEN")
+          fi
+        fi
+      ''
+    else
+      "";
+
+  # Systemd failure alert script
+  systemdAlertScript = pkgs.writeShellScript "ntfy-systemd-alert" ''
+    set -euo pipefail
+
+    UNIT_NAME="$1"
+    SERVER_URL="${cfg.serverUrl}"
+    TOPIC=$(cat "${cfg.topicFile}" 2>/dev/null | tr -d '[:space:]')
+    if [ -z "$TOPIC" ]; then
+      echo "ERROR: Could not read topic from ${cfg.topicFile}"
+      exit 1
+    fi
+
+    # Get journal output for context
+    JOURNAL_OUTPUT=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 15 --no-pager 2>/dev/null || echo "No journal output available")
+
+    # Build auth args
+    AUTH_ARGS=()
+    ${authCurlArgs}
+
+    # Send notification
+    ${curl} -sf --max-time 15 -X POST \
+      "$SERVER_URL/$TOPIC" \
+      -H "Title: [${hostname}] Service failed: $UNIT_NAME" \
+      -H "Priority: high" \
+      -H "Tags: warning" \
+      "''${AUTH_ARGS[@]}" \
+      -d "$JOURNAL_OUTPUT" || true
+  '';
+
+in
+{
+  options.services.ntfyAlerts = {
+    enable = lib.mkEnableOption "ntfy push notifications for system alerts";
+
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "The ntfy server URL (e.g. https://ntfy.example.com)";
+      example = "https://ntfy.example.com";
+    };
+
+    topicFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to a file containing the ntfy topic name to publish alerts to.";
+      example = "/run/agenix/ntfy-alerts-topic";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Path to a file containing the ntfy auth token.
+        If set, uses Authorization: Bearer header for authentication.
+      '';
+      example = "/run/secrets/ntfy-token";
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    # Per-service OnFailure for monitored services
+    systemd.services = {
+      "ntfy-alert@" = {
+        description = "Send ntfy notification for failed service %i";
+
+        unitConfig.OnFailure = lib.mkForce "";
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${systemdAlertScript} %i";
+          TimeoutSec = 30;
+        };
+      };
+
+      # TODO: sanoid's ExecStartPre runs `zfs allow` which blocks on TXG sync;
+      # on the hdds pool (slow spinning disks + large async frees) this causes
+      # 30+ minute hangs and guaranteed timeouts. Suppress until we fix sanoid
+      # to run as root without `zfs allow`. See: nixpkgs#72060, openzfs/zfs#14180
+      "sanoid".unitConfig.OnFailure = lib.mkForce "";
+    };
+
+    # Global OnFailure drop-in for all services
+    systemd.packages = [
+      (pkgs.writeTextDir "etc/systemd/system/service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=ntfy-alert@%p.service
+      '')
+
+      # Sanoid-specific drop-in to override the global OnFailure (see TODO above)
+      (pkgs.writeTextDir "etc/systemd/system/sanoid.service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=
+      '')
+    ];
+    # ZED (ZFS Event Daemon) ntfy notification settings
+    services.zfs.zed = {
+      enableMail = false;
+      settings = {
+        ZED_NTFY_URL = cfg.serverUrl;
+        ZED_NTFY_TOPIC = "$(cat ${cfg.topicFile} | tr -d '[:space:]')";
+        ZED_NTFY_ACCESS_TOKEN = lib.mkIf (cfg.tokenFile != null) "$(cat ${cfg.tokenFile})";
+        ZED_NOTIFY_VERBOSE = true;
+      };
+    };
+
+  };
+}

legacy/server-config/modules/overlays.nix

@@ -0,0 +1,78 @@
+final: prev: {
+  ensureZfsMounts = prev.writeShellApplication {
+    name = "zfsEnsureMounted";
+    runtimeInputs = with prev; [
+      zfs
+      gawk
+      coreutils
+    ];
+
+    text = ''
+       #!/bin/sh
+
+       if [[ "$#" -eq "0" ]]; then
+          echo "no arguments passed"
+          exit 1
+       fi
+
+      MOUNTED=$(zfs list -o mountpoint,mounted -H | awk '$NF == "yes" {NF--; print}')
+
+       MISSING=""
+       for target in "$@"; do
+          if ! grep -Fxq "$target" <<< "$MOUNTED"; then
+             MISSING="$MISSING $target"
+          fi
+       done
+
+       if [[ -n "$MISSING" ]]; then
+          echo "FAILURE, missing:$MISSING" 1>&2
+          exit 1
+       fi
+    '';
+  };
+
+  reflac = prev.writeShellApplication {
+    name = "reflac";
+    runtimeInputs = with prev; [ flac ];
+    excludeShellChecks = [ "2086" ];
+
+    text = builtins.readFile (
+      prev.fetchurl {
+        url = "https://raw.githubusercontent.com/chungy/reflac/refs/heads/master/reflac";
+        sha256 = "61c6cc8be3d276c6714e68b55e5de0e6491f50bbf195233073dbce14a1e278a7";
+      }
+    );
+  };
+
+  jellyfin-exporter = prev.buildGoModule rec {
+    pname = "jellyfin-exporter";
+    version = "unstable-2025-03-27";
+    src = prev.fetchFromGitHub {
+      owner = "rebelcore";
+      repo = "jellyfin_exporter";
+      rev = "8e3970cb1bdf3cb21fac099c13072bb7c1b20cf9";
+      hash = "sha256-wDnhepYj1MyLRZlwKfmwf4xiEEL3mgQY6V+7TnBd0MY=";
+    };
+    vendorHash = "sha256-e08u10e/wNapNZSsD/fGVN9ybMHe3sW0yDIOqI8ZcYs=";
+    # upstream tests require a running Jellyfin instance
+    doCheck = false;
+    meta.mainProgram = "jellyfin_exporter";
+  };
+
+  igpu-exporter = prev.buildGoModule rec {
+    pname = "igpu-exporter";
+    version = "unstable-2025-03-27";
+    src = prev.fetchFromGitHub {
+      owner = "mike1808";
+      repo = "igpu-exporter";
+      rev = "db2dace1a895c2b950f6d3ba1a2e46729251d124";
+      hash = "sha256-xWTiu26UzTZIK/6jeda+x6VePUgoWTS0AekejFdgFWs=";
+    };
+    vendorHash = "sha256-oeCSKwDKVwvYQ1fjXXTwQSXNl/upDE3WAAk680vqh3U=";
+    subPackages = [ "cmd" ];
+    postInstall = ''
+      mv $out/bin/cmd $out/bin/igpu-exporter
+    '';
+    meta.mainProgram = "igpu-exporter";
+  };
+}

legacy/server-config/modules/power.nix

@@ -0,0 +1,41 @@
+{
+  ...
+}:
+{
+  powerManagement = {
+    enable = true;
+    cpuFreqGovernor = "powersave";
+  };
+
+  # Always-on server: disable all sleep targets.
+  systemd.targets = {
+    sleep.enable = false;
+    suspend.enable = false;
+    hibernate.enable = false;
+    hybrid-sleep.enable = false;
+  };
+
+  boot.kernelParams = [
+    # Disable NMI watchdog at boot. Eliminates periodic perf-counter interrupts
+    # across all cores (~1 W). Safe: apcupsd provides hardware hang detection
+    # via UPS, and softlockup watchdog remains active.
+    "nmi_watchdog=0"
+
+    # Route kernel work items to already-busy CPUs rather than waking idle ones.
+    # Reduces C-state exit frequency at the cost of slightly higher latency on
+    # work items -- irrelevant for a server whose latency-sensitive paths are
+    # all in userspace (caddy, jellyfin).
+    "workqueue.power_efficient=1"
+  ];
+
+  boot.kernel.sysctl = {
+    # Belt-and-suspenders: also set via boot param, but sysctl ensures it
+    # stays off if anything re-enables it at runtime.
+    "kernel.nmi_watchdog" = 0;
+  };
+
+  # Server has no audio consumers. Power-gate the HDA codec at module load.
+  boot.extraModprobeConfig = ''
+    options snd_hda_intel power_save=1 power_save_controller=Y
+  '';
+}

legacy/server-config/modules/secureboot.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  boot = {
+    loader.systemd-boot.enable = lib.mkForce false;
+
+    lanzaboote = {
+      enable = true;
+      # needed to be in `/etc/secureboot` for sbctl to work
+      pkiBundle = "/etc/secureboot";
+    };
+
+  };
+  system.activationScripts = {
+    # extract secureboot keys from agenix-decrypted tar
+    "secureboot-keys" = {
+      deps = [ "agenix" ];
+      text = ''
+        #!/bin/sh
+        (
+          umask 077
+          # Check if keys already exist (e.g., from disko-install)
+          if [[ -d ${config.boot.lanzaboote.pkiBundle} && -f ${config.boot.lanzaboote.pkiBundle}/db.key ]]; then
+            echo "Secureboot keys already present, skipping extraction"
+          else
+            echo "Extracting secureboot keys from agenix"
+            rm -fr ${config.boot.lanzaboote.pkiBundle} || true
+            install -d -o root -g wheel -m 0500 ${config.boot.lanzaboote.pkiBundle}
+            ${pkgs.gnutar}/bin/tar xf ${config.age.secrets.secureboot-tar.path} -C ${config.boot.lanzaboote.pkiBundle}
+          fi
+          chown -R root:wheel ${config.boot.lanzaboote.pkiBundle}
+          chmod -R 500 ${config.boot.lanzaboote.pkiBundle}
+        )
+      '';
+    };
+  };
+}

legacy/server-config/modules/security.nix

@@ -0,0 +1,120 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  # memory allocator
+  # BREAKS REDIS-IMMICH
+  # environment.memoryAllocator.provider = "graphene-hardened";
+
+  # disable coredumps
+  systemd.coredump.enable = false;
+
+  # Needed for Nix sandbox UID/GID mapping inside derivation builds.
+  # See https://github.com/NixOS/nixpkgs/issues/287194
+  security.unprivilegedUsernsClone = true;
+
+  # Disable kexec to prevent replacing the running kernel at runtime.
+  security.protectKernelImage = true;
+
+  # Kernel hardening boot parameters. These recover most of the runtime-
+  # configurable protections that the linux-hardened patchset provided.
+  boot.kernelParams = [
+    # Zero all page allocator pages on free / alloc. Prevents info leaks
+    # and use-after-free from seeing stale data. Modest CPU overhead.
+    "init_on_alloc=1"
+    "init_on_free=1"
+
+    # Prevent SLUB allocator from merging caches with similar size/flags.
+    # Keeps different kernel object types in separate slabs, making heap
+    # exploitation (type confusion, spray, use-after-free) significantly harder.
+    "slab_nomerge"
+
+    # Randomize order of pages returned by the buddy allocator.
+    "page_alloc.shuffle=1"
+
+    # Disable debugfs entirely (exposes kernel internals).
+    "debugfs=off"
+
+    # Disable legacy vsyscall emulation (unused by any modern glibc).
+    "vsyscall=none"
+
+    # Strict IOMMU TLB invalidation (no batching). Prevents DMA-capable
+    # devices from accessing stale mappings after unmap.
+    "iommu.strict=1"
+  ];
+
+  boot.kernel.sysctl = {
+    # Immediately reboot on kernel oops (don't leave a compromised
+    # kernel running). Negative value = reboot without delay.
+    "kernel.panic" = -1;
+
+    # Hide kernel pointers from all processes, including CAP_SYSLOG.
+    # Prevents info leaks used to defeat KASLR.
+    "kernel.kptr_restrict" = 2;
+
+    # Disable bpf() JIT compiler (eliminates JIT spray attack vector).
+    "net.core.bpf_jit_enable" = false;
+
+    # Disable ftrace (kernel function tracer) at runtime.
+    "kernel.ftrace_enabled" = false;
+
+    # Strict reverse-path filtering: drop packets arriving on an interface
+    # where the source address isn't routable back via that interface.
+    "net.ipv4.conf.all.rp_filter" = 1;
+    "net.ipv4.conf.default.rp_filter" = 1;
+    "net.ipv4.conf.all.log_martians" = true;
+    "net.ipv4.conf.default.log_martians" = true;
+
+    # Ignore ICMP redirects (prevents route table poisoning).
+    "net.ipv4.conf.all.accept_redirects" = false;
+    "net.ipv4.conf.all.secure_redirects" = false;
+    "net.ipv4.conf.default.accept_redirects" = false;
+    "net.ipv4.conf.default.secure_redirects" = false;
+    "net.ipv6.conf.all.accept_redirects" = false;
+    "net.ipv6.conf.default.accept_redirects" = false;
+
+    # Don't send ICMP redirects (we are not a router).
+    "net.ipv4.conf.all.send_redirects" = false;
+    "net.ipv4.conf.default.send_redirects" = false;
+
+    # Ignore broadcast ICMP (SMURF amplification mitigation).
+    "net.ipv4.icmp_echo_ignore_broadcasts" = true;
+
+    # Filesystem hardening: prevent hardlink/symlink-based attacks.
+    # protected_hardlinks/symlinks: block unprivileged creation of hard/symlinks
+    # to files the user doesn't own (prevents TOCTOU privilege escalation).
+    # protected_fifos/regular (level 2): restrict opening FIFOs and regular files
+    # in world-writable sticky directories to owner/group match only.
+    # Also required for systemd-tmpfiles to chmod hardlinked files.
+    "fs.protected_hardlinks" = true;
+    "fs.protected_symlinks" = true;
+    "fs.protected_fifos" = 2;
+    "fs.protected_regular" = 2;
+  };
+
+  services = {
+    dbus.implementation = "broker";
+    /*
+      logrotate.enable = true;
+      journald = {
+        storage = "volatile"; # Store logs in memory
+        upload.enable = false; # Disable remote log upload (the default)
+        extraConfig = ''
+          SystemMaxUse=500M
+          SystemMaxFileSize=50M
+        '';
+      };
+    */
+  };
+
+  services.fail2ban = {
+    enable = true;
+    # Use iptables actions for compatibility
+    banaction = "iptables-multiport";
+    banaction-allports = "iptables-allports";
+  };
+}

legacy/server-config/modules/usb-secrets.nix

@@ -0,0 +1,22 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  # Mount USB secrets drive via fileSystems
+  fileSystems."/mnt/usb-secrets" = {
+    device = "/dev/disk/by-label/SECRETS";
+    fsType = "vfat";
+    options = [
+      "ro"
+      "uid=root"
+      "gid=root"
+      "umask=377"
+    ];
+    neededForBoot = true;
+  };
+
+  age.identityPaths = [ "/mnt/usb-secrets/usb-secrets-key" ];
+}

legacy/server-config/modules/zfs.nix

@@ -0,0 +1,127 @@
+{
+  config,
+  lib,
+  service_configs,
+  pkgs,
+  ...
+}:
+let
+  # Total RAM in bytes (from /proc/meminfo: 65775836 KiB).
+  totalRamBytes = 65775836 * 1024;
+
+  # Hugepage reservations that the kernel carves out before ZFS can use them.
+  hugepages2mBytes = service_configs.hugepages_2m.total_pages * 2 * 1024 * 1024;
+  hugepages1gBytes = 3 * 1024 * 1024 * 1024; # 3x 1G pages for RandomX (xmrig.nix)
+  totalHugepageBytes = hugepages2mBytes + hugepages1gBytes;
+
+  # ARC max: 60% of RAM remaining after hugepages. Leaves headroom for
+  # application RSS (PostgreSQL, qBittorrent, Jellyfin, Grafana, etc.),
+  # kernel slabs, and page cache.
+  arcMaxBytes = (totalRamBytes - totalHugepageBytes) * 60 / 100;
+in
+{
+  boot.zfs.package = pkgs.zfs_2_4;
+  boot.initrd.kernelModules = [ "zfs" ];
+
+  boot.kernelParams = [
+    # 120s TXG timeout: batch more dirty data per transaction group so the
+    # HDD pool (hdds) writes larger, sequential I/Os instead of many small syncs.
+    # This is a global setting (no per-pool control); the SSD pool (tank) syncs
+    # infrequently but handles it fine since SSDs don't suffer from seek overhead.
+    "zfs.zfs_txg_timeout=120"
+
+    # Cap ARC to prevent it from claiming memory reserved for hugepages.
+    # Without this, ZFS auto-sizes c_max to ~62 GiB on a 64 GiB system,
+    # ignoring the 11.5 GiB of hugepage reservations.
+    "zfs.zfs_arc_max=${toString arcMaxBytes}"
+
+    # vdev I/O scheduler: feed more concurrent reads to the block scheduler so
+    # mq-deadline has a larger pool of requests to sort and merge into elevator sweeps.
+    # Default async_read_max is 3 — far too few for effective coalescence.
+    # 32 was empirically optimal (64 overwhelmed the drives, 3 gave near-zero merges).
+    "zfs.zfs_vdev_async_read_max_active=32"
+    "zfs.zfs_vdev_async_read_min_active=4"
+
+    # Merge reads within 128 KiB of each other (default 32 KiB). On HDDs, reading a
+    # 128 KiB gap is far cheaper than a mechanical seek (~8 ms).
+    "zfs.zfs_vdev_read_gap_limit=131072"
+
+    # Allow ZFS to aggregate I/Os up to 4 MiB (default 1 MiB), matching the
+    # libtorrent piece extent size for larger sequential disk operations.
+    "zfs.zfs_vdev_aggregation_limit=4194304"
+  ];
+
+  boot.supportedFilesystems = [ "zfs" ];
+  boot.zfs.extraPools = [
+    service_configs.zpool_ssds
+    service_configs.zpool_hdds
+  ];
+
+  services.sanoid = {
+    enable = true;
+    datasets."${service_configs.zpool_ssds}" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 5;
+      daily = 7;
+      monthly = 3;
+      yearly = 0;
+    };
+
+    datasets."${service_configs.zpool_ssds}/services/sql" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 12;
+      daily = 2;
+      monthly = 0;
+      yearly = 0;
+    };
+
+    datasets."${service_configs.zpool_ssds}/services/jellyfin/cache" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 0;
+      daily = 0;
+      monthly = 0;
+      yearly = 0;
+    };
+
+    datasets."${service_configs.zpool_ssds}/services/monero" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 0;
+      daily = 0;
+      monthly = 0;
+      yearly = 0;
+    };
+
+    datasets."${service_configs.zpool_ssds}/services/p2pool" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 0;
+      daily = 0;
+      monthly = 0;
+      yearly = 0;
+    };
+
+    datasets."${service_configs.zpool_hdds}" = {
+      recursive = true;
+      autoprune = true;
+      autosnap = true;
+      hourly = 0;
+      daily = 0;
+      monthly = 0;
+      yearly = 0;
+    };
+  };
+
+  services.zfs = {
+    autoScrub.enable = true;
+    trim.enable = true;
+  };
+}