From d4b679d1a52ed0f70d685122ba7c808095185697 Mon Sep 17 00:00:00 2001 From: Simon Gardling Date: Tue, 3 Mar 2026 19:21:31 -0500 Subject: [PATCH] cleanup --- configuration.nix | 39 +-------- flake.nix | 137 +------------------------------- modules/zfs.nix | 17 ++-- service-configs.nix | 140 +++++++++++++++++++++++++++++++++ services/minecraft.nix | 81 ++++++++++--------- services/wg.nix | 32 ++++++++ tests/fail2ban-gitea.nix | 10 +-- tests/fail2ban-immich.nix | 15 ++-- tests/fail2ban-jellyfin.nix | 11 +-- tests/fail2ban-ssh.nix | 5 -- tests/fail2ban-vaultwarden.nix | 13 +-- tests/minecraft.nix | 21 +++-- 12 files changed, 256 insertions(+), 265 deletions(-) create mode 100644 service-configs.nix diff --git a/configuration.nix b/configuration.nix index c4c2b28..4181355 100644 --- a/configuration.nix +++ b/configuration.nix @@ -133,41 +133,6 @@ compressor = "zstd"; supportedFilesystems = [ "f2fs" ]; }; - - # BBR congestion control handles variable-latency VPN connections much - # better than CUBIC by probing bandwidth continuously rather than - # reacting to packet loss. - kernelModules = [ "tcp_bbr" ]; - - kernel.sysctl = { - # Use BBR + fair queuing for smooth throughput through the WireGuard VPN - "net.core.default_qdisc" = "fq"; - "net.ipv4.tcp_congestion_control" = "bbr"; - - # Disable slow-start after idle: prevents TCP from resetting window - # size on each burst cycle (the primary cause of the 0 -> 40 MB/s spikes) - "net.ipv4.tcp_slow_start_after_idle" = 0; - - # Larger socket buffers to accommodate the VPN bandwidth-delay product - # (22ms RTT * target throughput). Current 2.5MB max is too small. - "net.core.rmem_max" = 16777216; - "net.core.wmem_max" = 16777216; - "net.ipv4.tcp_rmem" = "4096 87380 16777216"; - "net.ipv4.tcp_wmem" = "4096 65536 16777216"; - - # Higher backlog for the large number of concurrent torrent connections - "net.core.netdev_max_backlog" = 5000; - # Faster cleanup of dead connections from torrent peer churn - "net.ipv4.tcp_fin_timeout" = 15; # default 60 - "net.ipv4.tcp_tw_reuse" = 1; - - # Minecraft server optimizations - # Disable autogroup for better scheduling of game server threads - "kernel.sched_autogroup_enabled" = 0; - # Huge pages for Minecraft JVM (ZGC ZGenerational needs heap + ~15% overhead) - # 4000MB heap = 2000 pages, plus ~285 for ZGC metadata = ~2285 needed - "vm.nr_hugepages" = 2600; - }; }; environment.etc = { @@ -237,8 +202,10 @@ hostName = hostname; hostId = "0f712d56"; firewall.enable = true; - firewall.trustedInterfaces = [ "wg-br" ]; + useDHCP = false; + + # Disabled because of Jellyfin (various issues) enableIPv6 = false; interfaces.${eth_interface} = { diff --git a/flake.nix b/flake.nix index 1a0fbd7..4fee0c2 100644 --- a/flake.nix +++ b/flake.nix @@ -97,142 +97,7 @@ eth_interface = "enp4s0"; system = "x86_64-linux"; - service_configs = rec { - zpool_ssds = "tank"; - zpool_hdds = "hdds"; - torrents_path = "/torrents"; - services_dir = "/services"; - music_dir = "/${zpool_ssds}/music"; - media_group = "media"; - - cpu_arch = "znver3"; - - ports = { - http = 80; - https = 443; - jellyfin = 8096; # no services.jellyfin option for this - torrent = 6011; - bitmagnet = 3333; - gitea = 2283; - immich = 2284; - soulseek_web = 5030; - soulseek_listen = 50300; - llama_cpp = 8991; - vaultwarden = 8222; - syncthing_gui = 8384; - syncthing_protocol = 22000; - syncthing_discovery = 21027; - minecraft = 25565; - matrix = 6167; - matrix_federation = 8448; - coturn = 3478; - coturn_tls = 5349; - ntfy = 2586; - livekit = 7880; - lk_jwt = 8081; - prowlarr = 9696; - sonarr = 8989; - radarr = 7878; - bazarr = 6767; - jellyseerr = 5055; - }; - - https = { - certs = services_dir + "/http_certs"; - domain = "gardling.com"; - }; - - gitea = { - dir = services_dir + "/gitea"; - domain = "git.${https.domain}"; - }; - - postgres = { - socket = "/run/postgresql"; - dataDir = services_dir + "/sql"; - }; - - immich = { - dir = services_dir + "/immich"; - }; - - minecraft = { - parent_dir = services_dir + "/minecraft"; - server_name = "main"; - }; - - torrent = { - SavePath = torrents_path; - TempPath = torrents_path + "/incomplete"; - }; - - jellyfin = { - dataDir = services_dir + "/jellyfin"; - cacheDir = services_dir + "/jellyfin_cache"; - }; - - slskd = rec { - base = "/var/lib/slskd"; - downloads = base + "/downloads"; - incomplete = base + "/incomplete"; - }; - - vaultwarden = { - path = "/var/lib/vaultwarden"; - }; - - monero = { - dataDir = services_dir + "/monero"; - }; - - matrix = { - dataDir = "/var/lib/continuwuity"; - domain = "matrix.${https.domain}"; - }; - - ntfy = { - domain = "ntfy.${https.domain}"; - }; - - livekit = { - domain = "livekit.${https.domain}"; - }; - - syncthing = { - dataDir = services_dir + "/syncthing"; - signalBackupDir = "/${zpool_ssds}/bak/signal"; - grayjayBackupDir = "/${zpool_ssds}/bak/grayjay"; - }; - - prowlarr = { - dataDir = services_dir + "/prowlarr"; - }; - - sonarr = { - dataDir = services_dir + "/sonarr"; - }; - - radarr = { - dataDir = services_dir + "/radarr"; - }; - - bazarr = { - dataDir = services_dir + "/bazarr"; - }; - - jellyseerr = { - configDir = services_dir + "/jellyseerr"; - }; - - recyclarr = { - dataDir = services_dir + "/recyclarr"; - }; - - media = { - moviesDir = torrents_path + "/media/movies"; - tvDir = torrents_path + "/media/tv"; - }; - }; + service_configs = import ./service-configs.nix; pkgs = import nixpkgs { inherit system; diff --git a/modules/zfs.nix b/modules/zfs.nix index ae377bc..1df7971 100644 --- a/modules/zfs.nix +++ b/modules/zfs.nix @@ -27,15 +27,20 @@ in boot.kernelParams = let - gb = 32; - mb = gb * 1000; - kb = mb * 1000; - b = kb * 1000; + arc_gb = 32; + arc_mb = arc_gb * 1000; + arc_kb = arc_mb * 1000; + arc_b = arc_kb * 1000; + + dirty_gb = 8; # Default value is 4GB, helps smooth writes + dirty_mb = dirty_gb * 1000; + dirty_kb = dirty_mb * 1000; + dirty_b = dirty_kb * 1000; in [ - "zfs.zfs_arc_max=${builtins.toString b}" + "zfs.zfs_arc_max=${builtins.toString arc_b}" "zfs.zfs_txg_timeout=120" # longer TXG open time = larger sequential writes - "zfs.zfs_dirty_data_max=8589934592" # 8GB dirty data buffer (default 4GB) for USB HDD write smoothing + "zfs.zfs_dirty_data_max=${builtins.toString dirty_b}" "zfs.zfs_delay_min_dirty_percent=80" # delay write throttling until 80% dirty (default 60%) "zfs.zfs_vdev_async_write_max_active=30" # more concurrent async writes to vdevs (default 10) ]; diff --git a/service-configs.nix b/service-configs.nix new file mode 100644 index 0000000..d53ed7c --- /dev/null +++ b/service-configs.nix @@ -0,0 +1,140 @@ +rec { + zpool_ssds = "tank"; + zpool_hdds = "hdds"; + torrents_path = "/torrents"; + services_dir = "/services"; + music_dir = "/${zpool_ssds}/music"; + media_group = "media"; + + cpu_arch = "znver3"; + + ports = { + http = 80; + https = 443; + jellyfin = 8096; # no services.jellyfin option for this + torrent = 6011; + bitmagnet = 3333; + gitea = 2283; + immich = 2284; + soulseek_web = 5030; + soulseek_listen = 50300; + llama_cpp = 8991; + vaultwarden = 8222; + syncthing_gui = 8384; + syncthing_protocol = 22000; + syncthing_discovery = 21027; + minecraft = 25565; + matrix = 6167; + matrix_federation = 8448; + coturn = 3478; + coturn_tls = 5349; + ntfy = 2586; + livekit = 7880; + lk_jwt = 8081; + prowlarr = 9696; + sonarr = 8989; + radarr = 7878; + bazarr = 6767; + jellyseerr = 5055; + }; + + https = { + certs = services_dir + "/http_certs"; + domain = "gardling.com"; + }; + + gitea = { + dir = services_dir + "/gitea"; + domain = "git.${https.domain}"; + }; + + postgres = { + socket = "/run/postgresql"; + dataDir = services_dir + "/sql"; + }; + + immich = { + dir = services_dir + "/immich"; + }; + + minecraft = { + parent_dir = services_dir + "/minecraft"; + server_name = "main"; + memory = rec { + heap_size_m = 4000; + large_page_size_m = 2; + }; + }; + + torrent = { + SavePath = torrents_path; + TempPath = torrents_path + "/incomplete"; + }; + + jellyfin = { + dataDir = services_dir + "/jellyfin"; + cacheDir = services_dir + "/jellyfin_cache"; + }; + + slskd = rec { + base = "/var/lib/slskd"; + downloads = base + "/downloads"; + incomplete = base + "/incomplete"; + }; + + vaultwarden = { + path = "/var/lib/vaultwarden"; + }; + + monero = { + dataDir = services_dir + "/monero"; + }; + + matrix = { + dataDir = "/var/lib/continuwuity"; + domain = "matrix.${https.domain}"; + }; + + ntfy = { + domain = "ntfy.${https.domain}"; + }; + + livekit = { + domain = "livekit.${https.domain}"; + }; + + syncthing = { + dataDir = services_dir + "/syncthing"; + signalBackupDir = "/${zpool_ssds}/bak/signal"; + grayjayBackupDir = "/${zpool_ssds}/bak/grayjay"; + }; + + prowlarr = { + dataDir = services_dir + "/prowlarr"; + }; + + sonarr = { + dataDir = services_dir + "/sonarr"; + }; + + radarr = { + dataDir = services_dir + "/radarr"; + }; + + bazarr = { + dataDir = services_dir + "/bazarr"; + }; + + jellyseerr = { + configDir = services_dir + "/jellyseerr"; + }; + + recyclarr = { + dataDir = services_dir + "/recyclarr"; + }; + + media = { + moviesDir = torrents_path + "/media/movies"; + tvDir = torrents_path + "/media/tv"; + }; +} diff --git a/services/minecraft.nix b/services/minecraft.nix index bc2750a..6a733ea 100644 --- a/services/minecraft.nix +++ b/services/minecraft.nix @@ -21,6 +21,19 @@ ]) ]; + boot.kernel.sysctl = { + # Disable autogroup for better scheduling of game server threads + "kernel.sched_autogroup_enabled" = 0; + + # We want to determine the number of hugepages based on how many minecraft needs. + # This can be determined by dividing the heap size by the size of a large page. + # Doing this gives us how many large pages are needed. + # Then we add 300 to give some headroom. + "vm.nr_hugepages" = + (service_configs.minecraft.memory.heap_size_m / service_configs.minecraft.memory.large_page_size_m) + + 300; + }; + services.minecraft-servers = { enable = true; eula = true; @@ -31,42 +44,38 @@ enable = true; package = pkgs.fabricServers.fabric-1_21_11; - jvmOpts = - let - heap_size = "4000M"; - in - lib.concatStringsSep " " [ - # Memory - "-Xmx${heap_size}" - "-Xms${heap_size}" - # GC - "-XX:+UseZGC" - "-XX:+ZGenerational" - # Base JVM optimizations (brucethemoose/Minecraft-Performance-Flags-Benchmarks) - "-XX:+UnlockExperimentalVMOptions" - "-XX:+UnlockDiagnosticVMOptions" - "-XX:+AlwaysActAsServerClassMachine" - "-XX:+AlwaysPreTouch" - "-XX:+DisableExplicitGC" - "-XX:+UseNUMA" - "-XX:+PerfDisableSharedMem" - "-XX:+UseFastUnorderedTimeStamps" - "-XX:+UseCriticalJavaThreadPriority" - "-XX:ThreadPriorityPolicy=1" - "-XX:AllocatePrefetchStyle=3" - "-XX:-DontCompileHugeMethods" - "-XX:MaxNodeLimit=240000" - "-XX:NodeLimitFudgeFactor=8000" - "-XX:ReservedCodeCacheSize=400M" - "-XX:NonNMethodCodeHeapSize=12M" - "-XX:ProfiledCodeHeapSize=194M" - "-XX:NonProfiledCodeHeapSize=194M" - "-XX:NmethodSweepActivity=1" - "-XX:+UseVectorCmov" - # Large pages (requires vm.nr_hugepages sysctl) - "-XX:+UseLargePages" - "-XX:LargePageSizeInBytes=2m" - ]; + jvmOpts = lib.concatStringsSep " " [ + # Memory + "-Xmx${builtins.toString service_configs.minecraft.memory.heap_size_m}M" + "-Xms${builtins.toString service_configs.minecraft.memory.heap_size_m}M" + # GC + "-XX:+UseZGC" + "-XX:+ZGenerational" + # Base JVM optimizations (brucethemoose/Minecraft-Performance-Flags-Benchmarks) + "-XX:+UnlockExperimentalVMOptions" + "-XX:+UnlockDiagnosticVMOptions" + "-XX:+AlwaysActAsServerClassMachine" + "-XX:+AlwaysPreTouch" + "-XX:+DisableExplicitGC" + "-XX:+UseNUMA" + "-XX:+PerfDisableSharedMem" + "-XX:+UseFastUnorderedTimeStamps" + "-XX:+UseCriticalJavaThreadPriority" + "-XX:ThreadPriorityPolicy=1" + "-XX:AllocatePrefetchStyle=3" + "-XX:-DontCompileHugeMethods" + "-XX:MaxNodeLimit=240000" + "-XX:NodeLimitFudgeFactor=8000" + "-XX:ReservedCodeCacheSize=400M" + "-XX:NonNMethodCodeHeapSize=12M" + "-XX:ProfiledCodeHeapSize=194M" + "-XX:NonProfiledCodeHeapSize=194M" + "-XX:NmethodSweepActivity=1" + "-XX:+UseVectorCmov" + # Large pages (requires vm.nr_hugepages sysctl) + "-XX:+UseLargePages" + "-XX:LargePageSizeInBytes=${builtins.toString service_configs.minecraft.memory.large_page_size_m}M" + ]; serverProperties = { server-port = service_configs.ports.minecraft; diff --git a/services/wg.nix b/services/wg.nix index 283e381..05d67ce 100644 --- a/services/wg.nix +++ b/services/wg.nix @@ -17,4 +17,36 @@ # "192.168.0.0/24" ]; }; + + boot = { + # BBR congestion control handles variable-latency VPN connections much + # better than CUBIC by probing bandwidth continuously rather than + # reacting to packet loss. + kernelModules = [ "tcp_bbr" ]; + + kernel.sysctl = { + # Use BBR + fair queuing for smooth throughput through the WireGuard VPN + "net.core.default_qdisc" = "fq"; + "net.ipv4.tcp_congestion_control" = "bbr"; + + # Disable slow-start after idle: prevents TCP from resetting window + # size on each burst cycle (the primary cause of the 0 -> 40 MB/s spikes) + "net.ipv4.tcp_slow_start_after_idle" = 0; + + # Larger socket buffers to accommodate the VPN bandwidth-delay product + # (22ms RTT * target throughput). Current 2.5MB max is too small. + "net.core.rmem_max" = 16777216; + "net.core.wmem_max" = 16777216; + "net.ipv4.tcp_rmem" = "4096 87380 16777216"; + "net.ipv4.tcp_wmem" = "4096 65536 16777216"; + + # Higher backlog for the large number of concurrent torrent connections + "net.core.netdev_max_backlog" = 5000; + # Faster cleanup of dead connections from torrent peer churn + "net.ipv4.tcp_fin_timeout" = 15; # default 60 + "net.ipv4.tcp_tw_reuse" = 1; + }; + }; + + networking.firewall.trustedInterfaces = [ "wg-br" ]; } diff --git a/tests/fail2ban-gitea.nix b/tests/fail2ban-gitea.nix index 82b3595..0022c06 100644 --- a/tests/fail2ban-gitea.nix +++ b/tests/fail2ban-gitea.nix @@ -5,18 +5,14 @@ ... }: let - testServiceConfigs = { + baseServiceConfigs = import ../service-configs.nix; + testServiceConfigs = lib.recursiveUpdate baseServiceConfigs { zpool_ssds = ""; gitea = { dir = "/var/lib/gitea"; domain = "git.test.local"; }; - postgres = { - socket = "/run/postgresql"; - }; - ports = { - gitea = 3000; - }; + ports.gitea = 3000; }; testLib = lib.extend ( diff --git a/tests/fail2ban-immich.nix b/tests/fail2ban-immich.nix index 64052d5..2fefa0a 100644 --- a/tests/fail2ban-immich.nix +++ b/tests/fail2ban-immich.nix @@ -5,17 +5,12 @@ ... }: let - testServiceConfigs = { + baseServiceConfigs = import ../service-configs.nix; + testServiceConfigs = lib.recursiveUpdate baseServiceConfigs { zpool_ssds = ""; - https = { - domain = "test.local"; - }; - ports = { - immich = 2283; - }; - immich = { - dir = "/var/lib/immich"; - }; + https.domain = "test.local"; + ports.immich = 2283; + immich.dir = "/var/lib/immich"; }; testLib = lib.extend ( diff --git a/tests/fail2ban-jellyfin.nix b/tests/fail2ban-jellyfin.nix index 8c1d53e..165dcd0 100644 --- a/tests/fail2ban-jellyfin.nix +++ b/tests/fail2ban-jellyfin.nix @@ -5,19 +5,14 @@ ... }: let - testServiceConfigs = { + baseServiceConfigs = import ../service-configs.nix; + testServiceConfigs = lib.recursiveUpdate baseServiceConfigs { zpool_ssds = ""; - https = { - domain = "test.local"; - }; - ports = { - jellyfin = 8096; - }; + https.domain = "test.local"; jellyfin = { dataDir = "/var/lib/jellyfin"; cacheDir = "/var/cache/jellyfin"; }; - media_group = "media"; }; testLib = lib.extend ( diff --git a/tests/fail2ban-ssh.nix b/tests/fail2ban-ssh.nix index 9730fff..758820f 100644 --- a/tests/fail2ban-ssh.nix +++ b/tests/fail2ban-ssh.nix @@ -5,11 +5,6 @@ ... }: let - testServiceConfigs = { - zpool_ssds = ""; - zpool_hdds = ""; - }; - securityModule = import ../modules/security.nix; sshModule = diff --git a/tests/fail2ban-vaultwarden.nix b/tests/fail2ban-vaultwarden.nix index fc871f2..5869a71 100644 --- a/tests/fail2ban-vaultwarden.nix +++ b/tests/fail2ban-vaultwarden.nix @@ -5,17 +5,10 @@ ... }: let - testServiceConfigs = { + baseServiceConfigs = import ../service-configs.nix; + testServiceConfigs = lib.recursiveUpdate baseServiceConfigs { zpool_ssds = ""; - https = { - domain = "test.local"; - }; - ports = { - vaultwarden = 8222; - }; - vaultwarden = { - path = "/var/lib/vaultwarden"; - }; + https.domain = "test.local"; }; testLib = lib.extend ( diff --git a/tests/minecraft.nix b/tests/minecraft.nix index 7a27d21..40dbfa9 100644 --- a/tests/minecraft.nix +++ b/tests/minecraft.nix @@ -6,18 +6,14 @@ ... }: let - testServiceConfigs = { - minecraft = { - server_name = "main"; - parent_dir = "/var/lib/minecraft"; - }; - https = { - domain = "test.local"; - }; - ports = { - minecraft = 25565; - }; + baseServiceConfigs = import ../service-configs.nix; + testServiceConfigs = lib.recursiveUpdate baseServiceConfigs { zpool_ssds = ""; + https.domain = "test.local"; + minecraft.parent_dir = "/var/lib/minecraft"; + minecraft.memory = rec { + heap_size_m = 1000; + }; }; # Create pkgs with nix-minecraft overlay and unfree packages allowed @@ -46,6 +42,9 @@ testPkgs.testers.runNixOSTest { ../services/minecraft.nix ]; + # Force to 0 because no huge pages in vms ? + boot.kernel.sysctl."vm.nr_hugepages" = lib.mkForce 0; + # Enable caddy service (required by minecraft service) services.caddy.enable = true;