titaniumtown/server-config
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix xmrig pause

Browse Source
This commit is contained in:
Simon Gardling
2026-04-03 14:39:20 -04:00
parent d4d01d63f1
commit daf82c16ba
8 changed files with 263 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
tests/llama-cpp-xmrig-pause.nix
View File

@@ -1,162 +0,0 @@
{
pkgs,
...
}:
let
script = ../services/llama-cpp/llama-cpp-xmrig-pause.py;
python = pkgs.python3;
# SmolLM-135M Q2_K: 85MB, modern GGUFv3, generates ~30 tok/s on one CPU
# thread — slow enough that a 200-token request keeps the process busy for
# several seconds, fast enough that tests don't crawl.
tinyModel = pkgs.fetchurl {
url = "https://huggingface.co/QuantFactory/SmolLM-135M-GGUF/resolve/main/SmolLM-135M.Q2_K.gguf";
hash = "sha256-DX46drPNJILNba21xfY2tyE0/yPWgOhz43gJdeSYKh4=";
};
in
pkgs.testers.runNixOSTest {
name = "llama-cpp-xmrig-pause";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [
pkgs.python3
pkgs.procps
pkgs.curl
pkgs.llama-cpp
];
# Mock xmrig as a simple sleep process that can be stopped/started.
systemd.services.xmrig = {
description = "Mock xmrig miner";
serviceConfig = {
ExecStart = "${pkgs.coreutils}/bin/sleep infinity";
Type = "simple";
};
wantedBy = [ "multi-user.target" ];
};
};
testScript = ''
import time
PORT = 18088
MODEL = "${tinyModel}"
PYTHON = "${python}/bin/python3"
SCRIPT = "${script}"
# Tuned for test speed while remaining realistic.
# POLL_INTERVAL=1 keeps detection latency low.
# GRACE_PERIOD=5 is long enough to verify "stays stopped" but short enough
# that the full test completes in ~2 minutes.
# CPU_THRESHOLD=10 is low because the VM has limited cores and the model
# is small — but any active inference still saturates a core.
POLL_INTERVAL = "1"
GRACE_PERIOD = "5"
CPU_THRESHOLD = "10"
infer_counter = 0
def send_completion(n_predict=200):
"""Fire a completion request in the background via a transient systemd unit."""
global infer_counter
infer_counter += 1
name = f"infer-{infer_counter}"
machine.succeed(
f"systemd-run --unit={name} --property=Type=exec "
f"curl -sf -X POST http://127.0.0.1:{PORT}/completion "
f"-H 'Content-Type: application/json' "
f"-d '{{\"prompt\": \"Once upon a time in a land far away there lived\", \"n_predict\": {n_predict}}}'"
)
return name
def wait_inference_done(unit_name, timeout=60):
"""Wait for a background inference request to finish."""
machine.wait_until_fails(
f"systemctl is-active {unit_name}",
timeout=timeout,
)
start_all()
machine.wait_for_unit("multi-user.target")
machine.wait_for_unit("xmrig.service")
with subtest("Start llama-server"):
machine.succeed(
f"systemd-run --unit=llama-server "
# Single inference thread to maximise per-core CPU%, which is
# what the monitor measures. Keeps token generation slow enough
# (~30 tok/s) that a 200-token request sustains load for seconds.
f"llama-server --model {MODEL} --port {PORT} --ctx-size 512 -t 1 -np 1"
)
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:{PORT}/health",
timeout=30,
)
machine.succeed("pgrep -x llama-server")
with subtest("Start pause monitor"):
machine.succeed(
f"systemd-run --unit=llama-xmrig-pause "
f"--setenv=POLL_INTERVAL={POLL_INTERVAL} "
f"--setenv=GRACE_PERIOD={GRACE_PERIOD} "
f"--setenv=CPU_THRESHOLD={CPU_THRESHOLD} "
f"{PYTHON} {SCRIPT}"
)
# The monitor needs two consecutive polls to compute a CPU delta.
# Wait for baseline to stabilise.
time.sleep(3)
with subtest("xmrig stays running while llama-server is idle"):
machine.succeed("systemctl is-active xmrig")
with subtest("xmrig stopped during prompt processing"):
unit = send_completion(n_predict=200)
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
with subtest("xmrig remains stopped during grace period after inference ends"):
wait_inference_done(unit)
# Inference just finished. The monitor will need 1-2 polls to detect
# idle, then the grace period starts. Checking 2s after completion
# is well within the 5s grace window.
time.sleep(2)
machine.fail("systemctl is-active xmrig")
with subtest("xmrig resumes after grace period expires"):
# Already idle since previous subtest. Grace period (5s) plus
# detection delay (~2 polls) means xmrig should restart within ~8s.
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
with subtest("Sequential prompts do not cause xmrig flapping"):
# First prompt — stop xmrig
unit1 = send_completion(n_predict=200)
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
wait_inference_done(unit1)
# Brief idle gap — shorter than grace period
time.sleep(2)
# Second prompt arrives before grace period expires, resetting it
unit2 = send_completion(n_predict=200)
time.sleep(3)
# xmrig must still be stopped
machine.fail("systemctl is-active xmrig")
wait_inference_done(unit2)
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
with subtest("xmrig stays stopped during sustained inference"):
unit = send_completion(n_predict=500)
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
# Stay busy longer than the grace period to prove continuous
# activity keeps xmrig stopped indefinitely.
time.sleep(8)
machine.fail("systemctl is-active xmrig")
wait_inference_done(unit)
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
'';
}

4
tests/tests.nix
View File

@@ -30,7 +30,9 @@ in
# llama-cpp tests
llamaCppAnnotationsTest = handleTest ./llama-cpp-annotations.nix;
llamaCppXmrigPauseTest = handleTest ./llama-cpp-xmrig-pause.nix;
# xmrig auto-pause test
xmrigAutoPauseTest = handleTest ./xmrig-auto-pause.nix;
# ntfy alerts test
ntfyAlertsTest = handleTest ./ntfy-alerts.nix;

121
tests/xmrig-auto-pause.nix Normal file
View File

@@ -0,0 +1,121 @@
{
pkgs,
...
}:
let
script = ../services/xmrig-auto-pause.py;
python = pkgs.python3;
in
pkgs.testers.runNixOSTest {
name = "xmrig-auto-pause";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [
pkgs.python3
pkgs.procps
];
# Mock xmrig as a nice'd sleep process that can be stopped/started.
systemd.services.xmrig = {
description = "Mock xmrig miner";
serviceConfig = {
ExecStart = "${pkgs.coreutils}/bin/sleep infinity";
Type = "simple";
Nice = 19;
};
wantedBy = [ "multi-user.target" ];
};
};
testScript = ''
import time
PYTHON = "${python}/bin/python3"
SCRIPT = "${script}"
# Tuned for test VMs (1-2 cores).
# POLL_INTERVAL=1 keeps detection latency low.
# GRACE_PERIOD=5 is long enough to verify "stays stopped" but short
# enough that the full test completes in reasonable time.
# CPU_THRESHOLD=10 catches a single busy-loop on a 1-2 core VM.
POLL_INTERVAL = "1"
GRACE_PERIOD = "5"
CPU_THRESHOLD = "10"
def start_cpu_load(name):
"""Start a non-nice CPU burn as a transient systemd unit."""
machine.succeed(
f"systemd-run --unit={name} --property=Type=exec "
f"bash -c 'while true; do :; done'"
)
def stop_cpu_load(name):
machine.succeed(f"systemctl stop {name}")
start_all()
machine.wait_for_unit("multi-user.target")
machine.wait_for_unit("xmrig.service")
with subtest("Start auto-pause monitor"):
machine.succeed(
f"systemd-run --unit=xmrig-auto-pause "
f"--setenv=POLL_INTERVAL={POLL_INTERVAL} "
f"--setenv=GRACE_PERIOD={GRACE_PERIOD} "
f"--setenv=CPU_THRESHOLD={CPU_THRESHOLD} "
f"{PYTHON} {SCRIPT}"
)
# Monitor needs two consecutive polls to compute a CPU delta.
time.sleep(3)
with subtest("xmrig stays running while system is idle"):
machine.succeed("systemctl is-active xmrig")
with subtest("xmrig stopped when CPU load appears"):
start_cpu_load("cpu-load")
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
with subtest("xmrig remains stopped during grace period after load ends"):
stop_cpu_load("cpu-load")
# Load just stopped. Grace period is 5s. Check at 2s — well within.
time.sleep(2)
machine.fail("systemctl is-active xmrig")
with subtest("xmrig resumes after grace period expires"):
# Already idle since previous subtest. Grace period (5s) plus
# detection delay (~2 polls) means xmrig should restart within ~8s.
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
with subtest("Intermittent load does not cause flapping"):
# First load — stop xmrig
start_cpu_load("cpu-load-1")
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
stop_cpu_load("cpu-load-1")
# Brief idle gap — shorter than grace period
time.sleep(2)
# Second load arrives before grace period expires
start_cpu_load("cpu-load-2")
time.sleep(3)
# xmrig must still be stopped
machine.fail("systemctl is-active xmrig")
stop_cpu_load("cpu-load-2")
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
with subtest("Sustained load keeps xmrig stopped"):
start_cpu_load("cpu-load-3")
machine.wait_until_fails("systemctl is-active xmrig", timeout=20)
# Stay busy longer than the grace period to prove continuous
# activity keeps xmrig stopped indefinitely.
time.sleep(8)
machine.fail("systemctl is-active xmrig")
stop_cpu_load("cpu-load-3")
machine.wait_until_succeeds("systemctl is-active xmrig", timeout=15)
'';
}