From f433953ebd3ee554a8658c17bf09550250b37103 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Mon, 2 Sep 2024 19:16:01 +0530 Subject: [PATCH] feat: add monitoring module for infrastructure * VictoriaMetrics for polling/collecting metrics * Grafana for UI * Multiple prometheus-exporter modules for gathering data Signed-off-by: Chinmay D. Pai --- modules/nixos/monitoring/default.nix | 23 +++++ modules/nixos/monitoring/exporter/default.nix | 33 +++++++ modules/nixos/monitoring/grafana/default.nix | 90 +++++++++++++++++++ .../monitoring/victoriametrics/default.nix | 81 +++++++++++++++++ 4 files changed, 227 insertions(+) create mode 100644 modules/nixos/monitoring/default.nix create mode 100644 modules/nixos/monitoring/exporter/default.nix create mode 100644 modules/nixos/monitoring/grafana/default.nix create mode 100644 modules/nixos/monitoring/victoriametrics/default.nix diff --git a/modules/nixos/monitoring/default.nix b/modules/nixos/monitoring/default.nix new file mode 100644 index 0000000..20d829a --- /dev/null +++ b/modules/nixos/monitoring/default.nix @@ -0,0 +1,23 @@ +{ + config, + lib, + ... +}: { + options.snowflake.monitoring.enable = lib.mkEnableOption "Enable the base monitoring stack configuration"; + + config = lib.mkIf config.snowflake.monitoring.enable { + # Enable base snowflake monitoring modules. + snowflake.monitoring = { + victoriametrics.enable = lib.mkDefault true; + grafana.enable = lib.mkDefault true; + exporter.collectd.enable = lib.mkDefault true; + exporter.node.enable = lib.mkDefault true; + exporter.systemd.enable = lib.mkDefault true; + # NOTE: Extra modules such as unifi-unpoller can be + # enabled in the system configuration manually. + # For example: + # exporter.unifi = true; + # Check exporter/default.nix for more details. + }; + }; +} diff --git a/modules/nixos/monitoring/exporter/default.nix b/modules/nixos/monitoring/exporter/default.nix new file mode 100644 index 0000000..7f745a6 --- /dev/null +++ b/modules/nixos/monitoring/exporter/default.nix @@ -0,0 +1,33 @@ +{ + config, + lib, + ... +}: { + options.snowflake.monitoring.exporter = { + collectd.enable = lib.mkEnableOption "Enable collectd exporter service"; + node.enable = lib.mkEnableOption "Enable node-exporter service"; + systemd.enable = lib.mkEnableOption "Enable systemd exporter service"; + }; + + config = let + cfg = config.snowflake.monitoring.exporter; + in { + services.prometheus.exporters = { + collectd.enable = cfg.collectd.enable; + node.enable = cfg.node.enable; + systemd.enable = cfg.systemd.enable; + # NOTE: These are the base monitoring modules meant to + # be enabled by default as sane defaults. + # Extra options for the defined exporters or custom exporters + # can be added to machine configuration manually. + # For example: + # services.prometheus.exporters.unifi = { + # enable = true; + # unifiUsername = "username"; + # unifiPassword = "password"; + # unifiInsecure = true; + # }; + # This can then be added to the vmagent configuration as extraConfig. + }; + }; +} diff --git a/modules/nixos/monitoring/grafana/default.nix b/modules/nixos/monitoring/grafana/default.nix new file mode 100644 index 0000000..4663438 --- /dev/null +++ b/modules/nixos/monitoring/grafana/default.nix @@ -0,0 +1,90 @@ +{ + config, + lib, + pkgs, + ... +}: { + options.snowflake.monitoring.grafana = let + settingsFormat = pkgs.formats.yaml {}; + in { + enable = lib.mkEnableOption "Enable grafana for monitoring stack"; + + domain = lib.mkOption { + type = lib.types.str; + default = ""; + description = "Configuration domain to use for the grafana service"; + }; + + adminPasswordFile = lib.mkOption { + description = "Age module containing the administrator password to use for grafana"; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 3010; + description = "Configuration port to use for the grafana service"; + }; + + extraDatasourceConfig = lib.mkOption { + description = "Extra datasource configuration for grafana"; + type = lib.types.listOf (lib.types.submodule {freeformType = settingsFormat.type;}); + default = []; + }; + }; + + config = let + cfg = config.snowflake.monitoring.grafana; + in + lib.mkIf cfg.enable { + age.secrets.grafana = { + inherit (cfg.adminPasswordFile) file; + owner = "grafana"; + group = "grafana"; + }; + + services.grafana = { + enable = true; + + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = cfg.port; + }; + + analytics = { + reporting_enabled = false; + feedback_links_enabled = false; + }; + security.admin_password = "$__file{${config.age.secrets.grafana.path}}"; + }; + + provision = { + enable = true; + + datasources.settings.datasources = + [] + ++ lib.optional config.services.victoriametrics.enable { + name = "Victoriametrics"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:${toString config.snowflake.monitoring.victoriametrics.port}"; + } + ++ cfg.extraDatasourceConfig; + }; + }; + + # Requires services.nginx.enable. + services.nginx = { + virtualHosts = { + "${cfg.domain}" = { + serverName = "${cfg.domain}"; + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://${config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/"; + }; + }; + }; + }; + }; +} diff --git a/modules/nixos/monitoring/victoriametrics/default.nix b/modules/nixos/monitoring/victoriametrics/default.nix new file mode 100644 index 0000000..7648dfe --- /dev/null +++ b/modules/nixos/monitoring/victoriametrics/default.nix @@ -0,0 +1,81 @@ +{ + config, + lib, + pkgs, + ... +}: { + options.snowflake.monitoring.victoriametrics = let + settingsFormat = pkgs.formats.json {}; + in { + enable = lib.mkEnableOption "Enable victoriametrics and vmagent stack"; + + port = lib.mkOption { + type = lib.types.port; + default = 8428; + description = "Port to listen on for victoriametrics"; + }; + + extraPrometheusConfig = lib.mkOption { + description = "Extra prometheus scrape config for vmagent"; + type = lib.types.listOf (lib.types.submodule {freeformType = settingsFormat.type;}); + default = []; + }; + }; + + config = let + cfg = config.snowflake.monitoring.victoriametrics; + exporterCfg = config.services.prometheus.exporters; + in + lib.mkIf cfg.enable { + services.victoriametrics = { + enable = cfg.enable; + listenAddress = "127.0.0.1:${toString cfg.port}"; + retentionPeriod = 3; + }; + services.vmagent = { + enable = cfg.enable; + remoteWrite.url = "http://${config.services.victoriametrics.listenAddress}/api/v1/write"; + prometheusConfig = { + global = { + scrape_interval = "1m"; + scrape_timeout = "30s"; + }; + scrape_configs = + [] + ++ lib.optional exporterCfg.node.enable { + job_name = "node"; + static_configs = [ + { + targets = ["127.0.0.1:${toString exporterCfg.node.port}"]; + } + ]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "instance"; + regex = "([^:]+)(:[0-9]+)?"; + replacement = config.networking.hostName; + } + ]; + } + ++ lib.optional exporterCfg.collectd.enable { + job_name = "collectd"; + static_configs = [ + { + targets = ["127.0.0.1:${toString exporterCfg.collectd.port}"]; + } + ]; + } + ++ lib.optional exporterCfg.systemd.enable { + job_name = "systemd"; + static_configs = [ + { + targets = ["127.0.0.1:${toString exporterCfg.systemd.port}"]; + } + ]; + } + ++ cfg.extraPrometheusConfig; + }; + }; + }; +}