feat: add monitoring module for infrastructure

* VictoriaMetrics for polling/collecting metrics
* Grafana for UI
* Multiple prometheus-exporter modules for gathering data

Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com>

modules/nixos/monitoring/default.nix

@@ -0,0 +1,23 @@
+{
+  config,
+  lib,
+  ...
+}: {
+  options.snowflake.monitoring.enable = lib.mkEnableOption "Enable the base monitoring stack configuration";
+
+  config = lib.mkIf config.snowflake.monitoring.enable {
+    # Enable base snowflake monitoring modules.
+    snowflake.monitoring = {
+      victoriametrics.enable = lib.mkDefault true;
+      grafana.enable = lib.mkDefault true;
+      exporter.collectd.enable = lib.mkDefault true;
+      exporter.node.enable = lib.mkDefault true;
+      exporter.systemd.enable = lib.mkDefault true;
+      # NOTE: Extra modules such as unifi-unpoller can be
+      # enabled in the system configuration manually.
+      # For example:
+      # exporter.unifi = true;
+      # Check exporter/default.nix for more details.
+    };
+  };
+}

modules/nixos/monitoring/exporter/default.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  ...
+}: {
+  options.snowflake.monitoring.exporter = {
+    collectd.enable = lib.mkEnableOption "Enable collectd exporter service";
+    node.enable = lib.mkEnableOption "Enable node-exporter service";
+    systemd.enable = lib.mkEnableOption "Enable systemd exporter service";
+  };
+
+  config = let
+    cfg = config.snowflake.monitoring.exporter;
+  in {
+    services.prometheus.exporters = {
+      collectd.enable = cfg.collectd.enable;
+      node.enable = cfg.node.enable;
+      systemd.enable = cfg.systemd.enable;
+      # NOTE: These are the base monitoring modules meant to
+      # be enabled by default as sane defaults.
+      # Extra options for the defined exporters or custom exporters
+      # can be added to machine configuration manually.
+      # For example:
+      #   services.prometheus.exporters.unifi = {
+      #     enable = true;
+      #     unifiUsername = "username";
+      #     unifiPassword = "password";
+      #     unifiInsecure = true;
+      #   };
+      # This can then be added to the vmagent configuration as extraConfig.
+    };
+  };
+}

modules/nixos/monitoring/grafana/default.nix

@@ -0,0 +1,90 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: {
+  options.snowflake.monitoring.grafana = let
+    settingsFormat = pkgs.formats.yaml {};
+  in {
+    enable = lib.mkEnableOption "Enable grafana for monitoring stack";
+
+    domain = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "Configuration domain to use for the grafana service";
+    };
+
+    adminPasswordFile = lib.mkOption {
+      description = "Age module containing the administrator password to use for grafana";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 3010;
+      description = "Configuration port to use for the grafana service";
+    };
+
+    extraDatasourceConfig = lib.mkOption {
+      description = "Extra datasource configuration for grafana";
+      type = lib.types.listOf (lib.types.submodule {freeformType = settingsFormat.type;});
+      default = [];
+    };
+  };
+
+  config = let
+    cfg = config.snowflake.monitoring.grafana;
+  in
+    lib.mkIf cfg.enable {
+      age.secrets.grafana = {
+        inherit (cfg.adminPasswordFile) file;
+        owner = "grafana";
+        group = "grafana";
+      };
+
+      services.grafana = {
+        enable = true;
+
+        settings = {
+          server = {
+            http_addr = "127.0.0.1";
+            http_port = cfg.port;
+          };
+
+          analytics = {
+            reporting_enabled = false;
+            feedback_links_enabled = false;
+          };
+          security.admin_password = "$__file{${config.age.secrets.grafana.path}}";
+        };
+
+        provision = {
+          enable = true;
+
+          datasources.settings.datasources =
+            []
+            ++ lib.optional config.services.victoriametrics.enable {
+              name = "Victoriametrics";
+              type = "prometheus";
+              access = "proxy";
+              url = "http://127.0.0.1:${toString config.snowflake.monitoring.victoriametrics.port}";
+            }
+            ++ cfg.extraDatasourceConfig;
+        };
+      };
+
+      # Requires services.nginx.enable.
+      services.nginx = {
+        virtualHosts = {
+          "${cfg.domain}" = {
+            serverName = "${cfg.domain}";
+            enableACME = true;
+            forceSSL = true;
+            locations."/" = {
+              proxyPass = "http://${config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/";
+            };
+          };
+        };
+      };
+    };
+}

modules/nixos/monitoring/victoriametrics/default.nix

@@ -0,0 +1,81 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: {
+  options.snowflake.monitoring.victoriametrics = let
+    settingsFormat = pkgs.formats.json {};
+  in {
+    enable = lib.mkEnableOption "Enable victoriametrics and vmagent stack";
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 8428;
+      description = "Port to listen on for victoriametrics";
+    };
+
+    extraPrometheusConfig = lib.mkOption {
+      description = "Extra prometheus scrape config for vmagent";
+      type = lib.types.listOf (lib.types.submodule {freeformType = settingsFormat.type;});
+      default = [];
+    };
+  };
+
+  config = let
+    cfg = config.snowflake.monitoring.victoriametrics;
+    exporterCfg = config.services.prometheus.exporters;
+  in
+    lib.mkIf cfg.enable {
+      services.victoriametrics = {
+        enable = cfg.enable;
+        listenAddress = "127.0.0.1:${toString cfg.port}";
+        retentionPeriod = 3;
+      };
+      services.vmagent = {
+        enable = cfg.enable;
+        remoteWrite.url = "http://${config.services.victoriametrics.listenAddress}/api/v1/write";
+        prometheusConfig = {
+          global = {
+            scrape_interval = "1m";
+            scrape_timeout = "30s";
+          };
+          scrape_configs =
+            []
+            ++ lib.optional exporterCfg.node.enable {
+              job_name = "node";
+              static_configs = [
+                {
+                  targets = ["127.0.0.1:${toString exporterCfg.node.port}"];
+                }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = ["__address__"];
+                  target_label = "instance";
+                  regex = "([^:]+)(:[0-9]+)?";
+                  replacement = config.networking.hostName;
+                }
+              ];
+            }
+            ++ lib.optional exporterCfg.collectd.enable {
+              job_name = "collectd";
+              static_configs = [
+                {
+                  targets = ["127.0.0.1:${toString exporterCfg.collectd.port}"];
+                }
+              ];
+            }
+            ++ lib.optional exporterCfg.systemd.enable {
+              job_name = "systemd";
+              static_configs = [
+                {
+                  targets = ["127.0.0.1:${toString exporterCfg.systemd.port}"];
+                }
+              ];
+            }
+            ++ cfg.extraPrometheusConfig;
+        };
+      };
+    };
+}