Set up grafana & prometheus

hosts/nixos/lapetus/default.nix

@@ -13,6 +13,8 @@
     ./services/actual.nix
     ./services/homer.nix
     ./services/zfs.nix
+    ./services/prometheus.nix
+    ./services/grafana.nix
     ./filesystems
     ./hardware
   ];

hosts/nixos/lapetus/secrets.yaml

@@ -1,5 +1,7 @@
 tilde_irc_pass: ENC[AES256_GCM,data:+pw/g0pffo1zF++1H/+iFXQDCDw=,iv:zTBvaUCwt78dgv1jF9EmrTuHMnM2S+GUGpQZWY828tA=,tag:umqaQOWqy8aMOxWR0CNGHQ==,type:str]
 vaultwarden_env: ENC[AES256_GCM,data:39gY2J+AFTwIRar7tbF6D9WadTzw1xiqPE9T204Z,iv:k9m6wQIPh1qScCjgLnULjVxVmDxxmotd/xzVuH6ju/w=,tag:+xIkwguOwYryO4rgsyMOsQ==,type:str]
+grafana_smtp_pass: ENC[AES256_GCM,data:PudFnWOS6LR69FMhlMs=,iv:4oKSiW0Xgu539w3QQBOW/ay/8w5HrbxRoPGBh/0wST4=,tag:jat8wA3JQlC7WbOwNQ4Ctw==,type:str]
+grafana_discord_webhook: ENC[AES256_GCM,data:y17UjlnfNmtvim9REkop4abcU6BX0P5JnJY1Mk7mNoE6mhyN7cEOrikTbehT+IOylG6rd+VtKIEj0X86qjx59qEo/NMbXqCrqxy6nhWD2NIDxQ5ZSQOUMVYGVLv7VKx3YG5mMvGgMHZEuJrobc0t6WejKAZ3LT/nqQ==,iv:2XtCnuirsXx2R2X7FozDczi4trAbnP5d8dXV7aJMWzE=,tag:a/dxsRuyye5ChaLGV+P6Zw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -24,8 +26,8 @@ sops:
             RHZ6alYrUU5BZ2xlMkdGR1dWRG5aeGMKJdsdtVZ6Mk9Vo3a+tS+rzAgaF2wpH+8U
             lWhA+c0Kbe8EJT8hm7Vr8PqBmElz4V9AnXSCTp7D+Cu4pfWsHopLUQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-02-12T16:10:16Z"
+    lastmodified: "2024-02-24T05:29:55Z"
-    mac: ENC[AES256_GCM,data:kYPlAH/LZiA6UJPgRgj6MBWHDWx21unyWj/qtJ1dmaoW8UXi8AnZt+/PT53rvRgzPYGnDgaxTugFH+kYflMQ7wOJpmie/VcsA0kJ+KVAg1Z7awjCBeqSQn+yuS+/ngqLRxHd3gBjmV32NOg6hlmBCJPhWUzqn9WiItq5ut3Da2w=,iv:W0Bg6PBiFtdwN6xuu8kE9x860T2LuTRv+ARF/EOUf4g=,tag:r30t606ttqT9qEjresPKbA==,type:str]
+    mac: ENC[AES256_GCM,data:Ckpg3qO7nrxouIuUlWsV4lPHqMGGgDf5rfVOvAOBqaoJyWqBL+kp24cakWLKEIfwNQKti7sGGUnz4zNvFE+brZaQv3HZqy9LwkCrvow6wLjZ4aHKms8o8MM8klZ2kydTXkORV8efoYja7FovQPIS99I9NNjlRe7+RpsnEs4AxHc=,iv:CP2m83RsiLd+qk+48/Js7NqPeFAc6r+PcvNuuUKQHbc=,tag:zaln4IigusRt9pogQERKFQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

hosts/nixos/lapetus/services/grafana.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, ... }:
+let secret = name: "$__file(${config.sops.secrets.${name}.path})";
+in
+{
+  imports = [
+    ../../common/optional/services/nginx.nix
+    ./prometheus.nix
+  ];
+
+  sops.secrets.grafana_smtp_pass.sopsFile = ../secrets.yaml;
+  sops.secrets.grafana_discord_webhook.sopsFile = ../secrets.yaml;
+
+  # {{{ Main config
+  services.grafana = {
+    enable = true;
+
+    settings = {
+      domain = "grafana.moonythm.dev";
+      port = 8409;
+      addr = "127.0.0.1";
+
+      # {{{ Smtp
+      smtp = {
+        enabled = true;
+        host = "smtp.migadu.com";
+        port = 465;
+        from_name = "Grafana";
+        from_address = "grafana@orbit.moonythm.dev";
+        password = secret "grafana_smtp_pass";
+      };
+      # }}}
+    };
+
+    # {{{ Provisoning
+    provision = {
+      enable = true;
+      notifiers = {
+        email.type = "email";
+
+        discord = {
+          type = "discord";
+          settings.webhook_url = secret "grafana_discord_webhook";
+        };
+      };
+
+      datasources.settings.datasources.prometheus = {
+        name = "Prometheus";
+        type = "prometheus";
+        access = "proxy";
+        url = "prometheus.moonythm.dev";
+      };
+    };
+    # }}}
+  };
+  # }}}
+  # {{{ Networking & storage
+  services.nginx.virtualHosts.${config.services.grafana.domain} =
+    config.satellite.proxy config.services.grafana.port { };
+
+  environment.persistence."/persist/state".directories = [
+    config.services.grafana.dataDir
+  ];
+  # }}}
+}

hosts/nixos/lapetus/services/homer.nix

@@ -18,6 +18,8 @@ let
   icon = file: "assets/${iconPath}/${file}";
 in
 {
+  imports = [ ../../common/optional/services/nginx.nix ];
+
   services.nginx.virtualHosts."lab.moonythm.dev" = {
     enableACME = true;
     acmeRoot = null;

hosts/nixos/lapetus/services/prometheus.nix

@@ -0,0 +1,41 @@
+{ config, pkgs, ... }:
+let host = "prometheus.moonythm.dev";
+in
+{
+  imports = [ ../../common/optional/services/nginx.nix ];
+
+  # {{{ Main config
+  services.prometheus = {
+    enable = true;
+    port = 8410;
+    webExternalUrl = host;
+
+    # {{{ Node exporter (system info)
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 8411;
+      };
+    };
+
+    scrapeConfigs = [{
+      job_name = "lapetus";
+      static_configs = [{
+        targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
+      }];
+    }];
+    # }}}
+  };
+  # }}}
+  # {{{ Networking & storage
+  services.nginx.virtualHosts.${host} =
+    config.satellite.proxy
+      config.services.grafana.port
+      { proxyWebsockets = true; };
+
+  environment.persistence."/persist/state".directories = [
+    "/var/lib/prometheus2"
+  ];
+  # }}}
+}