grafana: more things
1. Smoothed out power draw - UPS only reports on 9 watt intervals, so smoothing it out gives more relative detail on trends 2. Add jellyfin integration - Good for seeing correlations between statistics and jellyfin streams 3. intel gpu stats - Provides info on utilization of the gpu
This commit is contained in:
78
services/intel-gpu-collector.py
Normal file
78
services/intel-gpu-collector.py
Normal file
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
TEXTFILE = os.environ.get(
|
||||
"TEXTFILE",
|
||||
"/var/lib/prometheus-node-exporter-textfiles/intel-gpu.prom",
|
||||
)
|
||||
|
||||
|
||||
def read_one_sample():
|
||||
proc = subprocess.Popen(
|
||||
["intel_gpu_top", "-J", "-s", "1000"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
buf = b""
|
||||
depth = 0
|
||||
in_obj = False
|
||||
deadline = time.monotonic() + 5.0
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
byte = proc.stdout.read(1)
|
||||
if not byte:
|
||||
break
|
||||
if byte == b"{":
|
||||
in_obj = True
|
||||
depth += 1
|
||||
if in_obj:
|
||||
buf += byte
|
||||
if in_obj and byte == b"}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
finally:
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
return json.loads(buf) if buf else None
|
||||
|
||||
|
||||
def write_metrics(sample):
|
||||
lines = [
|
||||
"# HELP intel_gpu_engine_busy_percent Intel GPU engine busy percentage",
|
||||
"# TYPE intel_gpu_engine_busy_percent gauge",
|
||||
]
|
||||
for engine, data in sample.get("engines", {}).items():
|
||||
lines.append(
|
||||
f'intel_gpu_engine_busy_percent{{engine="{engine}"}} {data.get("busy", 0)}'
|
||||
)
|
||||
freq = sample.get("frequency", {})
|
||||
lines += [
|
||||
"# HELP intel_gpu_frequency_mhz Intel GPU actual frequency in MHz",
|
||||
"# TYPE intel_gpu_frequency_mhz gauge",
|
||||
f'intel_gpu_frequency_mhz {freq.get("actual", 0)}',
|
||||
"# HELP intel_gpu_rc6_percent Intel GPU RC6 power-saving state percentage",
|
||||
"# TYPE intel_gpu_rc6_percent gauge",
|
||||
f'intel_gpu_rc6_percent {sample.get("rc6", {}).get("value", 0)}',
|
||||
]
|
||||
|
||||
tmp = TEXTFILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
f.write("\n".join(lines) + "\n")
|
||||
os.replace(tmp, TEXTFILE)
|
||||
|
||||
|
||||
def main():
|
||||
sample = read_one_sample()
|
||||
if sample is None:
|
||||
print("Failed to read intel_gpu_top sample", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
write_metrics(sample)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
40
services/jellyfin-annotations.nix
Normal file
40
services/jellyfin-annotations.nix
Normal file
@@ -0,0 +1,40 @@
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
service_configs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{
|
||||
systemd.services.jellyfin-annotations = {
|
||||
description = "Jellyfin stream annotation service for Grafana";
|
||||
after = [
|
||||
"network.target"
|
||||
"grafana.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.python3}/bin/python3 ${./jellyfin-annotations.py}";
|
||||
Restart = "always";
|
||||
RestartSec = "10s";
|
||||
LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
|
||||
DynamicUser = true;
|
||||
StateDirectory = "jellyfin-annotations";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
MemoryDenyWriteExecute = true;
|
||||
};
|
||||
environment = {
|
||||
JELLYFIN_URL = "http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port}";
|
||||
GRAFANA_URL = "http://127.0.0.1:${toString service_configs.ports.private.grafana.port}";
|
||||
STATE_FILE = "/var/lib/jellyfin-annotations/state.json";
|
||||
POLL_INTERVAL = "30";
|
||||
};
|
||||
};
|
||||
}
|
||||
233
services/jellyfin-annotations.py
Normal file
233
services/jellyfin-annotations.py
Normal file
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
JELLYFIN_URL = os.environ.get("JELLYFIN_URL", "http://127.0.0.1:8096")
|
||||
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000")
|
||||
STATE_FILE = os.environ.get("STATE_FILE", "/var/lib/jellyfin-annotations/state.json")
|
||||
POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "30"))
|
||||
|
||||
|
||||
def get_api_key():
|
||||
cred_dir = os.environ.get("CREDENTIALS_DIRECTORY")
|
||||
if cred_dir:
|
||||
return Path(cred_dir, "jellyfin-api-key").read_text().strip()
|
||||
for p in ["/run/agenix/jellyfin-api-key"]:
|
||||
if Path(p).exists():
|
||||
return Path(p).read_text().strip()
|
||||
sys.exit("ERROR: Cannot find jellyfin-api-key")
|
||||
|
||||
|
||||
def http_json(method, url, body=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
||||
method=method,
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def get_active_sessions(api_key):
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{JELLYFIN_URL}/Sessions?api_key={api_key}",
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
sessions = json.loads(resp.read())
|
||||
return [s for s in sessions if s.get("NowPlayingItem")]
|
||||
except Exception as e:
|
||||
print(f"Error fetching sessions: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def _codec(name):
|
||||
if not name:
|
||||
return ""
|
||||
aliases = {"h264": "H.264", "h265": "H.265", "hevc": "H.265", "av1": "AV1",
|
||||
"vp9": "VP9", "vp8": "VP8", "mpeg4": "MPEG-4", "mpeg2video": "MPEG-2",
|
||||
"aac": "AAC", "ac3": "AC3", "eac3": "EAC3", "dts": "DTS",
|
||||
"truehd": "TrueHD", "mp3": "MP3", "opus": "Opus", "flac": "FLAC",
|
||||
"vorbis": "Vorbis"}
|
||||
return aliases.get(name.lower(), name.upper())
|
||||
|
||||
|
||||
def _res(width, height):
|
||||
if not height:
|
||||
return ""
|
||||
common = {2160: "4K", 1440: "1440p", 1080: "1080p", 720: "720p",
|
||||
480: "480p", 360: "360p"}
|
||||
return common.get(height, f"{height}p")
|
||||
|
||||
|
||||
def _channels(n):
|
||||
labels = {1: "Mono", 2: "Stereo", 6: "5.1", 7: "6.1", 8: "7.1"}
|
||||
return labels.get(n, f"{n}ch") if n else ""
|
||||
|
||||
|
||||
def format_label(session):
|
||||
user = session.get("UserName", "Unknown")
|
||||
item = session.get("NowPlayingItem", {}) or {}
|
||||
transcode = session.get("TranscodingInfo") or {}
|
||||
play_state = session.get("PlayState") or {}
|
||||
client = session.get("Client", "")
|
||||
device = session.get("DeviceName", "")
|
||||
|
||||
name = item.get("Name", "Unknown")
|
||||
series = item.get("SeriesName", "")
|
||||
season = item.get("ParentIndexNumber")
|
||||
episode = item.get("IndexNumber")
|
||||
media_type = item.get("Type", "")
|
||||
|
||||
if series and season and episode:
|
||||
title = f"{series} S{season:02d}E{episode:02d} \u2013 {name}"
|
||||
elif series:
|
||||
title = f"{series} \u2013 {name}"
|
||||
elif media_type == "Movie":
|
||||
title = f"{name} (movie)"
|
||||
else:
|
||||
title = name
|
||||
|
||||
play_method = play_state.get("PlayMethod", "")
|
||||
if play_method == "DirectPlay":
|
||||
method = "Direct Play"
|
||||
elif play_method == "DirectStream":
|
||||
method = "Direct Stream"
|
||||
elif play_method == "Transcode" or transcode:
|
||||
method = "Transcode"
|
||||
else:
|
||||
method = "Direct Play"
|
||||
|
||||
media_streams = item.get("MediaStreams") or []
|
||||
video_streams = [s for s in media_streams if s.get("Type") == "Video"]
|
||||
audio_streams = [s for s in media_streams if s.get("Type") == "Audio"]
|
||||
default_audio = next((s for s in audio_streams if s.get("IsDefault")), None)
|
||||
audio_stream = default_audio or (audio_streams[0] if audio_streams else {})
|
||||
video_stream = video_streams[0] if video_streams else {}
|
||||
|
||||
src_vcodec = _codec(video_stream.get("Codec", ""))
|
||||
src_res = _res(video_stream.get("Width") or item.get("Width"),
|
||||
video_stream.get("Height") or item.get("Height"))
|
||||
src_acodec = _codec(audio_stream.get("Codec", ""))
|
||||
src_channels = _channels(audio_stream.get("Channels"))
|
||||
|
||||
is_video_direct = transcode.get("IsVideoDirect", True)
|
||||
is_audio_direct = transcode.get("IsAudioDirect", True)
|
||||
|
||||
if transcode and not is_video_direct:
|
||||
dst_vcodec = _codec(transcode.get("VideoCodec", ""))
|
||||
dst_res = _res(transcode.get("Width"), transcode.get("Height")) or src_res
|
||||
if src_vcodec and dst_vcodec and src_vcodec != dst_vcodec:
|
||||
video_part = f"{src_vcodec}\u2192{dst_vcodec} {dst_res}".strip()
|
||||
else:
|
||||
video_part = f"{dst_vcodec or src_vcodec} {dst_res}".strip()
|
||||
else:
|
||||
video_part = f"{src_vcodec} {src_res}".strip()
|
||||
|
||||
if transcode and not is_audio_direct:
|
||||
dst_acodec = _codec(transcode.get("AudioCodec", ""))
|
||||
dst_channels = _channels(transcode.get("AudioChannels")) or src_channels
|
||||
if src_acodec and dst_acodec and src_acodec != dst_acodec:
|
||||
audio_part = f"{src_acodec}\u2192{dst_acodec} {dst_channels}".strip()
|
||||
else:
|
||||
audio_part = f"{dst_acodec or src_acodec} {dst_channels}".strip()
|
||||
else:
|
||||
audio_part = f"{src_acodec} {src_channels}".strip()
|
||||
|
||||
bitrate = transcode.get("Bitrate") or item.get("Bitrate")
|
||||
bitrate_part = f"{bitrate / 1_000_000:.1f} Mbps" if bitrate else ""
|
||||
|
||||
reasons = transcode.get("TranscodeReasons") or []
|
||||
reason_part = f"[{', '.join(reasons)}]" if reasons else ""
|
||||
|
||||
stream_parts = [p for p in [method, video_part, audio_part, bitrate_part, reason_part] if p]
|
||||
client_str = " \u00b7 ".join(filter(None, [client, device]))
|
||||
|
||||
lines = [f"{user}: {title}", " | ".join(stream_parts)]
|
||||
if client_str:
|
||||
lines.append(client_str)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def load_state():
|
||||
try:
|
||||
with open(STATE_FILE) as f:
|
||||
return json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(state):
|
||||
os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
|
||||
tmp = STATE_FILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(state, f)
|
||||
os.replace(tmp, STATE_FILE)
|
||||
|
||||
|
||||
def grafana_post(label, start_ms):
|
||||
try:
|
||||
result = http_json(
|
||||
"POST",
|
||||
f"{GRAFANA_URL}/api/annotations",
|
||||
{"time": start_ms, "text": label, "tags": ["jellyfin"]},
|
||||
)
|
||||
return result.get("id")
|
||||
except Exception as e:
|
||||
print(f"Error posting annotation: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def grafana_close(grafana_id, end_ms):
|
||||
try:
|
||||
http_json(
|
||||
"PATCH",
|
||||
f"{GRAFANA_URL}/api/annotations/{grafana_id}",
|
||||
{"timeEnd": end_ms},
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error closing annotation {grafana_id}: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
api_key = get_api_key()
|
||||
state = load_state()
|
||||
|
||||
while True:
|
||||
now_ms = int(time.time() * 1000)
|
||||
sessions = get_active_sessions(api_key)
|
||||
|
||||
if sessions is not None:
|
||||
current_ids = {s["Id"] for s in sessions}
|
||||
|
||||
for s in sessions:
|
||||
sid = s["Id"]
|
||||
if sid not in state:
|
||||
label = format_label(s)
|
||||
grafana_id = grafana_post(label, now_ms)
|
||||
if grafana_id is not None:
|
||||
state[sid] = {
|
||||
"grafana_id": grafana_id,
|
||||
"label": label,
|
||||
"start_ms": now_ms,
|
||||
}
|
||||
save_state(state)
|
||||
|
||||
for sid in [k for k in state if k not in current_ids]:
|
||||
info = state.pop(sid)
|
||||
grafana_close(info["grafana_id"], now_ms)
|
||||
save_state(state)
|
||||
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -38,6 +38,17 @@ let
|
||||
'';
|
||||
};
|
||||
|
||||
intelGpuCollector = pkgs.writeShellApplication {
|
||||
name = "intel-gpu-collector";
|
||||
runtimeInputs = with pkgs; [
|
||||
python3
|
||||
intel-gpu-tools
|
||||
];
|
||||
text = ''
|
||||
exec python3 ${./intel-gpu-collector.py}
|
||||
'';
|
||||
};
|
||||
|
||||
dashboard = {
|
||||
editable = true;
|
||||
graphTooltip = 1;
|
||||
@@ -54,6 +65,21 @@ let
|
||||
title = "System Overview";
|
||||
uid = "system-overview";
|
||||
|
||||
annotations.list = [
|
||||
{
|
||||
name = "Jellyfin Streams";
|
||||
datasource = {
|
||||
type = "grafana";
|
||||
uid = "-- Grafana --";
|
||||
};
|
||||
enable = true;
|
||||
iconColor = "green";
|
||||
showIn = 0;
|
||||
type = "tags";
|
||||
tags = [ "jellyfin" ];
|
||||
}
|
||||
];
|
||||
|
||||
panels = [
|
||||
# -- Row 1: UPS --
|
||||
{
|
||||
@@ -415,6 +441,42 @@ let
|
||||
graphMode = "area";
|
||||
};
|
||||
}
|
||||
|
||||
# -- Row 3: Intel GPU --
|
||||
{
|
||||
id = 8;
|
||||
type = "timeseries";
|
||||
title = "Intel GPU Utilization";
|
||||
gridPos = {
|
||||
h = 8;
|
||||
w = 24;
|
||||
x = 0;
|
||||
y = 16;
|
||||
};
|
||||
datasource = promDs;
|
||||
targets = [
|
||||
{
|
||||
datasource = promDs;
|
||||
expr = "intel_gpu_engine_busy_percent";
|
||||
legendFormat = "{{engine}}";
|
||||
refId = "A";
|
||||
}
|
||||
];
|
||||
fieldConfig = {
|
||||
defaults = {
|
||||
unit = "percent";
|
||||
min = 0;
|
||||
max = 100;
|
||||
color.mode = "palette-classic";
|
||||
custom = {
|
||||
lineWidth = 2;
|
||||
fillOpacity = 10;
|
||||
spanNulls = true;
|
||||
};
|
||||
};
|
||||
overrides = [ ];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
in
|
||||
@@ -500,7 +562,6 @@ in
|
||||
root_url = "https://${service_configs.grafana.domain}";
|
||||
};
|
||||
|
||||
# Caddy handles auth -- disable Grafana login entirely
|
||||
"auth.anonymous" = {
|
||||
enabled = true;
|
||||
org_role = "Admin";
|
||||
@@ -539,21 +600,17 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Provision dashboard JSON
|
||||
environment.etc."grafana-dashboards/system-overview.json" = {
|
||||
text = builtins.toJSON dashboard;
|
||||
mode = "0444";
|
||||
};
|
||||
|
||||
# Caddy reverse proxy with auth
|
||||
services.caddy.virtualHosts."${service_configs.grafana.domain}".extraConfig = ''
|
||||
import ${config.age.secrets.caddy_auth.path}
|
||||
reverse_proxy :${builtins.toString service_configs.ports.private.grafana.port}
|
||||
'';
|
||||
|
||||
# -- Jellyfin metrics collector --
|
||||
# Queries the Jellyfin API for active streams and writes a .prom file
|
||||
# for the node_exporter textfile collector.
|
||||
# -- Jellyfin active-stream prometheus textfile collector --
|
||||
systemd.services.jellyfin-metrics-collector = {
|
||||
description = "Collect Jellyfin metrics for Prometheus";
|
||||
after = [ "network.target" ];
|
||||
@@ -572,7 +629,24 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Ensure textfile collector directory exists (tmpfs root -- recreated on boot)
|
||||
# -- Intel GPU textfile collector --
|
||||
systemd.services.intel-gpu-collector = {
|
||||
description = "Collect Intel GPU metrics for Prometheus";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = lib.getExe intelGpuCollector;
|
||||
};
|
||||
environment.TEXTFILE = "${textfileDir}/intel-gpu.prom";
|
||||
};
|
||||
|
||||
systemd.timers.intel-gpu-collector = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = "*:*:0/30";
|
||||
RandomizedDelaySec = "10s";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${textfileDir} 0755 root root -"
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user