grafana: re-organize

configuration.nix

@@ -48,13 +48,11 @@
     ./services/soulseek.nix
 
     ./services/llama-cpp.nix
-    ./services/llama-cpp-annotations.nix
     ./services/trilium.nix
 
     ./services/ups.nix
-    ./services/monitoring.nix
+
-    ./services/jellyfin-annotations.nix
+    ./services/grafana
-    ./services/zfs-scrub-annotations.nix
 
     ./services/bitwarden.nix
     ./services/firefox-syncserver.nix

services/grafana/dashboard.nix

@@ -1,95 +1,12 @@
 {
-  config,
-  pkgs,
-  service_configs,
-  lib,
   ...
 }:
 let
-  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
-
   promDs = {
     type = "prometheus";
     uid = "prometheus";
   };
 
-  jellyfinCollector = pkgs.writeShellApplication {
-    name = "jellyfin-metrics-collector";
-    runtimeInputs = with pkgs; [
-      curl
-      jq
-    ];
-    text = ''
-      API_KEY=$(cat "$CREDENTIALS_DIRECTORY/jellyfin-api-key")
-      JELLYFIN="http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port}"
-
-      if response=$(curl -sf --max-time 5 "''${JELLYFIN}/Sessions?api_key=''${API_KEY}"); then
-        active_streams=$(echo "$response" | jq '[.[] | select(.NowPlayingItem != null)] | length')
-      else
-        active_streams=0
-      fi
-
-      {
-        echo '# HELP jellyfin_active_streams Number of currently active Jellyfin streams'
-        echo '# TYPE jellyfin_active_streams gauge'
-        echo "jellyfin_active_streams $active_streams"
-      } > "${textfileDir}/jellyfin.prom.$$.tmp"
-      mv "${textfileDir}/jellyfin.prom.$$.tmp" "${textfileDir}/jellyfin.prom"
-    '';
-  };
-
-  intelGpuCollector = pkgs.writeShellApplication {
-    name = "intel-gpu-collector";
-    runtimeInputs = with pkgs; [
-      python3
-      intel-gpu-tools
-    ];
-    text = ''
-      exec python3 ${./intel-gpu-collector.py}
-    '';
-  };
-
-  qbittorrentCollector = pkgs.writeShellApplication {
-    name = "qbittorrent-collector";
-    runtimeInputs = with pkgs; [
-      curl
-      jq
-    ];
-    text = ''
-      QBIT="http://${config.vpnNamespaces.wg.namespaceAddress}:${toString config.services.qbittorrent.webuiPort}"
-      OUT="${textfileDir}/qbittorrent.prom"
-
-      if info=$(curl -sf --max-time 5 "''${QBIT}/api/v2/transfer/info"); then
-        dl=$(echo "$info" | jq '.dl_info_speed')
-        ul=$(echo "$info" | jq '.up_info_speed')
-      else
-        dl=0
-        ul=0
-      fi
-
-      {
-        echo '# HELP qbittorrent_download_bytes_per_second Current download speed in bytes/s'
-        echo '# TYPE qbittorrent_download_bytes_per_second gauge'
-        echo "qbittorrent_download_bytes_per_second $dl"
-        echo '# HELP qbittorrent_upload_bytes_per_second Current upload speed in bytes/s'
-        echo '# TYPE qbittorrent_upload_bytes_per_second gauge'
-        echo "qbittorrent_upload_bytes_per_second $ul"
-      } > "''${OUT}.tmp"
-      mv "''${OUT}.tmp" "$OUT"
-    '';
-  };
-
-  diskUsageCollector = pkgs.writeShellApplication {
-    name = "disk-usage-collector";
-    runtimeInputs = with pkgs; [
-      coreutils
-      gawk
-      config.boot.zfs.package
-      util-linux # for mountpoint
-    ];
-    text = builtins.readFile ./disk-usage-collector.sh;
-  };
-
   dashboard = {
     editable = true;
     graphTooltip = 1;
@@ -772,227 +689,8 @@ let
   };
 in
 {
-  imports = [
-    (lib.serviceMountWithZpool "grafana" service_configs.zpool_ssds [
-      service_configs.grafana.dir
-    ])
-    (lib.serviceFilePerms "grafana" [
-      "Z ${service_configs.grafana.dir} 0700 grafana grafana"
-    ])
-    (lib.serviceMountWithZpool "prometheus" service_configs.zpool_ssds [
-      "/var/lib/prometheus"
-    ])
-    (lib.serviceFilePerms "prometheus" [
-      "Z /var/lib/prometheus 0700 prometheus prometheus"
-    ])
-  ];
-
-  # -- Prometheus --
-  services.prometheus = {
-    enable = true;
-    port = service_configs.ports.private.prometheus.port;
-    listenAddress = "127.0.0.1";
-    stateDir = "prometheus";
-    retentionTime = "90d";
-
-    exporters = {
-      node = {
-        enable = true;
-        port = service_configs.ports.private.prometheus_node.port;
-        listenAddress = "127.0.0.1";
-        enabledCollectors = [
-          "hwmon"
-          "systemd"
-          "textfile"
-        ];
-        extraFlags = [
-          "--collector.textfile.directory=${textfileDir}"
-        ];
-      };
-
-      apcupsd = {
-        enable = true;
-        port = service_configs.ports.private.prometheus_apcupsd.port;
-        listenAddress = "127.0.0.1";
-        apcupsdAddress = "127.0.0.1:3551";
-      };
-    };
-
-    scrapeConfigs = [
-      {
-        job_name = "prometheus";
-        static_configs = [
-          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus.port}" ]; }
-        ];
-      }
-      {
-        job_name = "node";
-        static_configs = [
-          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_node.port}" ]; }
-        ];
-      }
-      {
-        job_name = "apcupsd";
-        static_configs = [
-          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_apcupsd.port}" ]; }
-        ];
-      }
-    ];
-  };
-
-  # -- Grafana --
-  services.grafana = {
-    enable = true;
-    dataDir = service_configs.grafana.dir;
-
-    settings = {
-      server = {
-        http_addr = "127.0.0.1";
-        http_port = service_configs.ports.private.grafana.port;
-        domain = service_configs.grafana.domain;
-        root_url = "https://${service_configs.grafana.domain}";
-      };
-
-      "auth.anonymous" = {
-        enabled = true;
-        org_role = "Admin";
-      };
-      "auth.basic".enabled = false;
-      "auth".disable_login_form = true;
-
-      analytics.reporting_enabled = false;
-
-      feature_toggles.enable = "dataConnectionsConsole=false";
-
-      users.default_theme = "dark";
-
-      # Disable unused built-in integrations
-      alerting.enabled = false;
-      "unified_alerting".enabled = false;
-      explore.enabled = false;
-      news.news_feed_enabled = false;
-
-      plugins = {
-        enable_alpha = false;
-        plugin_admin_enabled = false;
-      };
-    };
-
-    provision = {
-      datasources.settings = {
-        apiVersion = 1;
-        datasources = [
-          {
-            name = "Prometheus";
-            type = "prometheus";
-            url = "http://127.0.0.1:${toString service_configs.ports.private.prometheus.port}";
-            access = "proxy";
-            isDefault = true;
-            editable = false;
-            uid = "prometheus";
-          }
-        ];
-      };
-
-      dashboards.settings.providers = [
-        {
-          name = "system";
-          type = "file";
-          options.path = "/etc/grafana-dashboards";
-          disableDeletion = true;
-          updateIntervalSeconds = 60;
-        }
-      ];
-    };
-  };
-
   environment.etc."grafana-dashboards/system-overview.json" = {
     text = builtins.toJSON dashboard;
     mode = "0444";
   };
-
-  services.caddy.virtualHosts."${service_configs.grafana.domain}".extraConfig = ''
-    import ${config.age.secrets.caddy_auth.path}
-    reverse_proxy :${builtins.toString service_configs.ports.private.grafana.port}
-  '';
-
-  # -- Jellyfin active-stream prometheus textfile collector --
-  systemd.services.jellyfin-metrics-collector = {
-    description = "Collect Jellyfin metrics for Prometheus";
-    after = [ "network.target" ];
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = lib.getExe jellyfinCollector;
-      LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
-    };
-  };
-
-  systemd.timers.jellyfin-metrics-collector = {
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*:*:0/30";
-      RandomizedDelaySec = "5s";
-    };
-  };
-
-  # -- Intel GPU textfile collector --
-  systemd.services.intel-gpu-collector = {
-    description = "Collect Intel GPU metrics for Prometheus";
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = lib.getExe intelGpuCollector;
-    };
-    environment.TEXTFILE = "${textfileDir}/intel-gpu.prom";
-  };
-
-  systemd.timers.intel-gpu-collector = {
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*:*:0/30";
-      RandomizedDelaySec = "10s";
-    };
-  };
-
-  # -- qBittorrent speed textfile collector --
-  systemd.services.qbittorrent-collector = {
-    description = "Collect qBittorrent transfer metrics for Prometheus";
-    after = [
-      "network.target"
-      "qbittorrent.service"
-    ];
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = lib.getExe qbittorrentCollector;
-    };
-  };
-
-  systemd.timers.qbittorrent-collector = {
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*:*:0/15";
-      RandomizedDelaySec = "3s";
-    };
-  };
-
-  # -- Disk/pool usage textfile collector --
-  systemd.services.disk-usage-collector = {
-    description = "Collect ZFS pool and partition usage metrics for Prometheus";
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = lib.getExe diskUsageCollector;
-    };
-    environment.TEXTFILE = "${textfileDir}/disk-usage.prom";
-  };
-
-  systemd.timers.disk-usage-collector = {
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "minutely";
-      RandomizedDelaySec = "10s";
-    };
-  };
-
-  systemd.tmpfiles.rules = [
-    "d ${textfileDir} 0755 root root -"
-  ];
 }

services/grafana/default.nix

@@ -0,0 +1,14 @@
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+    ./dashboard.nix
+    ./jellyfin-collector.nix
+    ./jellyfin-annotations.nix
+    ./qbittorrent-collector.nix
+    ./intel-gpu-collector.nix
+    ./disk-usage-collector.nix
+    ./llama-cpp-annotations.nix
+    ./zfs-scrub-annotations.nix
+  ];
+}

services/grafana/disk-usage-collector.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+
+  diskUsageCollector = pkgs.writeShellApplication {
+    name = "disk-usage-collector";
+    runtimeInputs = with pkgs; [
+      coreutils
+      gawk
+      config.boot.zfs.package
+      util-linux # for mountpoint
+    ];
+    text = builtins.readFile ./disk-usage-collector.sh;
+  };
+in
+lib.mkIf config.services.grafana.enable {
+  systemd.services.disk-usage-collector = {
+    description = "Collect ZFS pool and partition usage metrics for Prometheus";
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe diskUsageCollector;
+    };
+    environment.TEXTFILE = "${textfileDir}/disk-usage.prom";
+  };
+
+  systemd.timers.disk-usage-collector = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "minutely";
+      RandomizedDelaySec = "10s";
+    };
+  };
+}

services/grafana/grafana.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "grafana" service_configs.zpool_ssds [
+      service_configs.grafana.dir
+    ])
+    (lib.serviceFilePerms "grafana" [
+      "Z ${service_configs.grafana.dir} 0700 grafana grafana"
+    ])
+  ];
+
+  services.grafana = {
+    enable = true;
+    dataDir = service_configs.grafana.dir;
+
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = service_configs.ports.private.grafana.port;
+        domain = service_configs.grafana.domain;
+        root_url = "https://${service_configs.grafana.domain}";
+      };
+
+      "auth.anonymous" = {
+        enabled = true;
+        org_role = "Admin";
+      };
+      "auth.basic".enabled = false;
+      "auth".disable_login_form = true;
+
+      analytics.reporting_enabled = false;
+
+      feature_toggles.enable = "dataConnectionsConsole=false";
+
+      users.default_theme = "dark";
+
+      # Disable unused built-in integrations
+      alerting.enabled = false;
+      "unified_alerting".enabled = false;
+      explore.enabled = false;
+      news.news_feed_enabled = false;
+
+      plugins = {
+        enable_alpha = false;
+        plugin_admin_enabled = false;
+      };
+    };
+
+    provision = {
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://127.0.0.1:${toString service_configs.ports.private.prometheus.port}";
+            access = "proxy";
+            isDefault = true;
+            editable = false;
+            uid = "prometheus";
+          }
+        ];
+      };
+
+      dashboards.settings.providers = [
+        {
+          name = "system";
+          type = "file";
+          options.path = "/etc/grafana-dashboards";
+          disableDeletion = true;
+          updateIntervalSeconds = 60;
+        }
+      ];
+    };
+  };
+
+  services.caddy.virtualHosts."${service_configs.grafana.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${toString service_configs.ports.private.grafana.port}
+  '';
+}

services/grafana/intel-gpu-collector.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+
+  intelGpuCollector = pkgs.writeShellApplication {
+    name = "intel-gpu-collector";
+    runtimeInputs = with pkgs; [
+      python3
+      intel-gpu-tools
+    ];
+    text = ''
+      exec python3 ${./intel-gpu-collector.py}
+    '';
+  };
+in
+lib.mkIf config.services.grafana.enable {
+  systemd.services.intel-gpu-collector = {
+    description = "Collect Intel GPU metrics for Prometheus";
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe intelGpuCollector;
+    };
+    environment.TEXTFILE = "${textfileDir}/intel-gpu.prom";
+  };
+
+  systemd.timers.intel-gpu-collector = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*:*:0/30";
+      RandomizedDelaySec = "10s";
+    };
+  };
+}

services/grafana/jellyfin-annotations.nix

@@ -5,7 +5,7 @@
   lib,
   ...
 }:
-{
+lib.mkIf (config.services.grafana.enable && config.services.jellyfin.enable) {
   systemd.services.jellyfin-annotations = {
     description = "Jellyfin stream annotation service for Grafana";
     after = [

services/grafana/jellyfin-collector.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  pkgs,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+
+  jellyfinCollector = pkgs.writeShellApplication {
+    name = "jellyfin-metrics-collector";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+    ];
+    text = ''
+      API_KEY=$(cat "$CREDENTIALS_DIRECTORY/jellyfin-api-key")
+      JELLYFIN="http://127.0.0.1:${toString service_configs.ports.private.jellyfin.port}"
+
+      if response=$(curl -sf --max-time 5 "''${JELLYFIN}/Sessions?api_key=''${API_KEY}"); then
+        active_streams=$(echo "$response" | jq '[.[] | select(.NowPlayingItem != null)] | length')
+      else
+        active_streams=0
+      fi
+
+      {
+        echo '# HELP jellyfin_active_streams Number of currently active Jellyfin streams'
+        echo '# TYPE jellyfin_active_streams gauge'
+        echo "jellyfin_active_streams $active_streams"
+      } > "${textfileDir}/jellyfin.prom.$$.tmp"
+      mv "${textfileDir}/jellyfin.prom.$$.tmp" "${textfileDir}/jellyfin.prom"
+    '';
+  };
+in
+lib.mkIf (config.services.grafana.enable && config.services.jellyfin.enable) {
+  systemd.services.jellyfin-metrics-collector = {
+    description = "Collect Jellyfin metrics for Prometheus";
+    after = [ "network.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe jellyfinCollector;
+      LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
+    };
+  };
+
+  systemd.timers.jellyfin-metrics-collector = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*:*:0/30";
+      RandomizedDelaySec = "5s";
+    };
+  };
+}

services/grafana/llama-cpp-annotations.nix

@@ -1,9 +1,11 @@
 {
+  config,
   pkgs,
   service_configs,
+  lib,
   ...
 }:
-{
+lib.mkIf (config.services.grafana.enable && config.services.llama-cpp.enable) {
   systemd.services.llama-cpp-annotations = {
     description = "LLM request annotation service for Grafana";
     after = [

services/grafana/prometheus.nix

@@ -0,0 +1,74 @@
+{
+  service_configs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+in
+{
+  imports = [
+    (lib.serviceMountWithZpool "prometheus" service_configs.zpool_ssds [
+      "/var/lib/prometheus"
+    ])
+    (lib.serviceFilePerms "prometheus" [
+      "Z /var/lib/prometheus 0700 prometheus prometheus"
+    ])
+  ];
+
+  services.prometheus = {
+    enable = true;
+    port = service_configs.ports.private.prometheus.port;
+    listenAddress = "127.0.0.1";
+    stateDir = "prometheus";
+    retentionTime = "90d";
+
+    exporters = {
+      node = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_node.port;
+        listenAddress = "127.0.0.1";
+        enabledCollectors = [
+          "hwmon"
+          "systemd"
+          "textfile"
+        ];
+        extraFlags = [
+          "--collector.textfile.directory=${textfileDir}"
+        ];
+      };
+
+      apcupsd = {
+        enable = true;
+        port = service_configs.ports.private.prometheus_apcupsd.port;
+        listenAddress = "127.0.0.1";
+        apcupsdAddress = "127.0.0.1:3551";
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "prometheus";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus.port}" ]; }
+        ];
+      }
+      {
+        job_name = "node";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_node.port}" ]; }
+        ];
+      }
+      {
+        job_name = "apcupsd";
+        static_configs = [
+          { targets = [ "127.0.0.1:${toString service_configs.ports.private.prometheus_apcupsd.port}" ]; }
+        ];
+      }
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d ${textfileDir} 0755 root root -"
+  ];
+}

services/grafana/qbittorrent-collector.nix

@@ -0,0 +1,60 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  textfileDir = "/var/lib/prometheus-node-exporter-textfiles";
+
+  qbittorrentCollector = pkgs.writeShellApplication {
+    name = "qbittorrent-collector";
+    runtimeInputs = with pkgs; [
+      curl
+      jq
+    ];
+    text = ''
+      QBIT="http://${config.vpnNamespaces.wg.namespaceAddress}:${toString config.services.qbittorrent.webuiPort}"
+      OUT="${textfileDir}/qbittorrent.prom"
+
+      if info=$(curl -sf --max-time 5 "''${QBIT}/api/v2/transfer/info"); then
+        dl=$(echo "$info" | jq '.dl_info_speed')
+        ul=$(echo "$info" | jq '.up_info_speed')
+      else
+        dl=0
+        ul=0
+      fi
+
+      {
+        echo '# HELP qbittorrent_download_bytes_per_second Current download speed in bytes/s'
+        echo '# TYPE qbittorrent_download_bytes_per_second gauge'
+        echo "qbittorrent_download_bytes_per_second $dl"
+        echo '# HELP qbittorrent_upload_bytes_per_second Current upload speed in bytes/s'
+        echo '# TYPE qbittorrent_upload_bytes_per_second gauge'
+        echo "qbittorrent_upload_bytes_per_second $ul"
+      } > "''${OUT}.tmp"
+      mv "''${OUT}.tmp" "$OUT"
+    '';
+  };
+in
+lib.mkIf (config.services.grafana.enable && config.services.qbittorrent.enable) {
+  systemd.services.qbittorrent-collector = {
+    description = "Collect qBittorrent transfer metrics for Prometheus";
+    after = [
+      "network.target"
+      "qbittorrent.service"
+    ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe qbittorrentCollector;
+    };
+  };
+
+  systemd.timers.qbittorrent-collector = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*:*:0/15";
+      RandomizedDelaySec = "3s";
+    };
+  };
+}

services/grafana/zfs-scrub-annotations.nix

@@ -21,7 +21,7 @@ let
     text = builtins.readFile ./zfs-scrub-annotations.sh;
   };
 in
-{
+lib.mkIf (config.services.grafana.enable && config.services.zfs.autoScrub.enable) {
   systemd.services.zfs-scrub = {
     environment = {
       GRAFANA_URL = grafanaUrl;

tests/jellyfin-annotations.nix

@@ -6,7 +6,7 @@
 let
   jfLib = import ./jellyfin-test-lib.nix { inherit pkgs lib; };
   mockGrafana = ./mock-grafana-server.py;
-  script = ../services/jellyfin-annotations.py;
+  script = ../services/grafana/jellyfin-annotations.py;
   python = pkgs.python3;
 in
 pkgs.testers.runNixOSTest {

tests/llama-cpp-annotations.nix

@@ -4,7 +4,7 @@
 }:
 let
   mockGrafana = ./mock-grafana-server.py;
-  script = ../services/llama-cpp-annotations.py;
+  script = ../services/grafana/llama-cpp-annotations.py;
   python = pkgs.python3;
 
   mockLlamaProcess = ./mock-llama-server-proc.py;

tests/zfs-scrub-annotations.nix

@@ -23,7 +23,7 @@ let
     esac
   '';
 
-  script = ../services/zfs-scrub-annotations.sh;
+  script = ../services/grafana/zfs-scrub-annotations.sh;
   python = pkgs.python3;
 in
 pkgs.testers.runNixOSTest {