mirror/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2024-09-29 15:43:00 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #316727 from jpds/nixos-tests-prometheus

Browse Source 
nixos/prometheus: Revamp tests
This commit is contained in:
Franz Pletz 2024-06-23 23:05:06 +02:00 committed by GitHub
commit 3c033186ee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 865 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
maintainers/maintainer-list.nix
View File

@ -9848,6 +9848,11 @@
githubId = 1918771;
name = "Joe Doyle";
};
jpds = {
github = "jpds";
githubId = 29158971;
name = "Jonathan Davies";
};
jpentland = {
email = "joe.pentland@gmail.com";
github = "jpentland";

1
nixos/modules/module-list.nix
View File

@ -880,6 +880,7 @@
./services/monitoring/osquery.nix
./services/monitoring/parsedmarc.nix
./services/monitoring/prometheus/alertmanager-irc-relay.nix
./services/monitoring/prometheus/alertmanager-webhook-logger.nix
./services/monitoring/prometheus/alertmanager.nix
./services/monitoring/prometheus/default.nix
./services/monitoring/prometheus/exporters.nix

70
nixos/modules/services/monitoring/prometheus/alertmanager-webhook-logger.nix Normal file
View File

@ -0,0 +1,70 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.prometheus.alertmanagerWebhookLogger;
in
{
options.services.prometheus.alertmanagerWebhookLogger = {
enable = mkEnableOption "Alertmanager Webhook Logger";
package = mkPackageOption pkgs "alertmanager-webhook-logger" { };
extraFlags = mkOption {
type = types.listOf types.str;
default = [];
description = "Extra command line options to pass to alertmanager-webhook-logger.";
};
};
config = mkIf cfg.enable {
systemd.services.alertmanager-webhook-logger = {
description = "Alertmanager Webhook Logger";
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
serviceConfig = {
ExecStart = ''
${cfg.package}/bin/alertmanager-webhook-logger \
${escapeShellArgs cfg.extraFlags}
'';
DynamicUser = true;
NoNewPrivileges = true;
ProtectProc = "invisible";
ProtectSystem = "strict";
ProtectHome = "tmpfs";
PrivateTmp = true;
PrivateDevices = true;
PrivateIPC = true;
ProtectHostname = true;
ProtectClock = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallFilter = [
"@system-service"
"~@cpu-emulation"
"~@privileged"
"~@reboot"
"~@setuid"
"~@swap"
];
};
};
};
meta.maintainers = [ maintainers.jpds ];
}

3
nixos/tests/all-tests.nix
View File

@ -775,7 +775,7 @@ in {
printing-service = handleTest ./printing.nix { socket = false; };
private-gpt = handleTest ./private-gpt.nix {};
privoxy = handleTest ./privoxy.nix {};
prometheus = handleTest ./prometheus.nix {};
prometheus = handleTest ./prometheus {};
prometheus-exporters = handleTest ./prometheus-exporters.nix {};
prosody = handleTest ./xmpp/prosody.nix {};
prosody-mysql = handleTest ./xmpp/prosody-mysql.nix {};
@ -968,6 +968,7 @@ in {
teleport = handleTest ./teleport.nix {};
thelounge = handleTest ./thelounge.nix {};
terminal-emulators = handleTest ./terminal-emulators.nix {};
thanos = handleTest ./thanos.nix {};
tiddlywiki = handleTest ./tiddlywiki.nix {};
tigervnc = handleTest ./tigervnc.nix {};
timescaledb = handleTest ./timescaledb.nix {};

148
nixos/tests/prometheus/alertmanager.nix Normal file
View File

@ -0,0 +1,148 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-alertmanager";
nodes = {
prometheus = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"alertmanager:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [
''
groups:
- name: test
rules:
- alert: InstanceDown
expr: up == 0
for: 5s
labels:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
''
];
scrapeConfigs = [
{
job_name = "alertmanager";
static_configs = [
{
targets = [
"alertmanager:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
{
job_name = "node";
static_configs = [
{
targets = [
"node:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
];
};
};
alertmanager = { config, pkgs, ... }: {
services.prometheus.alertmanager = {
enable = true;
openFirewall = true;
configuration = {
global = {
resolve_timeout = "1m";
};
route = {
# Root route node
receiver = "test";
group_by = ["..."];
continue = false;
group_wait = "1s";
group_interval = "15s";
repeat_interval = "24h";
};
receivers = [
{
name = "test";
webhook_configs = [
{
url = "http://logger:6725";
send_resolved = true;
max_alerts = 0;
}
];
}
];
};
};
};
logger = { config, pkgs, ... }: {
networking.firewall.allowedTCPPorts = [ 6725 ];
services.prometheus.alertmanagerWebhookLogger.enable = true;
};
};
testScript = ''
alertmanager.wait_for_unit("alertmanager")
alertmanager.wait_for_open_port(9093)
alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
#alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")
logger.wait_for_unit("alertmanager-webhook-logger")
logger.wait_for_open_port(6725)
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
+ "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
)
logger.wait_until_succeeds(
"journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
)
'';
})

116
nixos/tests/prometheus/config-reload.nix Normal file
View File

@ -0,0 +1,116 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-config-reload";
nodes = {
prometheus = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
enableReload = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus:${toString config.services.prometheus.port}"
];
}
];
}
];
};
specialisation = {
"prometheus-config-change" = {
configuration = {
environment.systemPackages = [ pkgs.yq ];
# This configuration just adds a new prometheus job
# to scrape the node_exporter metrics of the s3 machine.
services.prometheus = {
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ];
}
];
}
];
};
};
};
};
};
};
testScript = ''
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
# Check if switching to a NixOS configuration that changes the prometheus
# configuration reloads (instead of restarts) prometheus before the switch
# finishes successfully:
with subtest("config change reloads prometheus"):
import json
# We check if prometheus has finished reloading by looking for the message
# "Completed loading of configuration file" in the journal between the start
# and finish of switching to the new NixOS configuration.
#
# To mark the start we record the journal cursor before starting the switch:
cursor_before_switching = json.loads(
prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
)["__CURSOR"]
# Now we switch:
prometheus_config_change = prometheus.succeed(
"readlink /run/current-system/specialisation/prometheus-config-change"
).strip()
prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
# Next we retrieve all logs since the start of switching:
logs_after_starting_switching = prometheus.succeed(
"""
journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
""".format(
cursor_before_switching=cursor_before_switching
)
)
# Finally we check if the message "Completed loading of configuration file"
# occurs before the "finished switching to system configuration" message:
finished_switching_msg = (
"finished switching to system configuration " + prometheus_config_change
)
reloaded_before_switching_finished = False
finished_switching = False
for log_line in logs_after_starting_switching.split("\n"):
msg = json.loads(log_line)["MESSAGE"]
if "Completed loading of configuration file" in msg:
reloaded_before_switching_finished = True
if msg == finished_switching_msg:
finished_switching = True
break
assert reloaded_before_switching_finished
assert finished_switching
# Check if the reloaded config includes the new node job:
prometheus.succeed(
"""
curl -sf http://127.0.0.1:9090/api/v1/status/config \
| jq -r .data.yaml \
| yq '.scrape_configs | any(.job_name == "node")' \
| grep true
"""
)
'';
})

13
nixos/tests/prometheus/default.nix Normal file
View File

@ -0,0 +1,13 @@
{ system ? builtins.currentSystem
, config ? { }
, pkgs ? import ../../.. { inherit system config; }
}:
{
alertmanager = import ./alertmanager.nix { inherit system pkgs; };
config-reload = import ./config-reload.nix { inherit system pkgs; };
federation = import ./federation.nix { inherit system pkgs; };
prometheus-pair = import ./prometheus-pair.nix { inherit system pkgs; };
pushgateway = import ./pushgateway.nix { inherit system pkgs; };
remote-write = import ./remote-write.nix { inherit system pkgs; };
}

213
nixos/tests/prometheus/federation.nix Normal file
View File

@ -0,0 +1,213 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-federation";
nodes = {
global1 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
];
};
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
global2 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
];
};
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
prometheus1 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node1:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
prometheus2 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node2:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
node1 = { config, pkgs, ... }: {
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
node2 = { config, pkgs, ... }: {
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
};
testScript = ''
for machine in node1, node2:
machine.wait_for_unit("prometheus-node-exporter")
machine.wait_for_open_port(9100)
for machine in prometheus1, prometheus2, global1, global2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
# Verify both servers got the same data from the exporter
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
for machine in global1, global2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"4\"'"
)
'';
})

87
nixos/tests/prometheus/prometheus-pair.nix Normal file
View File

@ -0,0 +1,87 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-pair";
nodes = {
prometheus1 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
prometheus2 = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
};
testScript = ''
for machine in prometheus1, prometheus2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'")
machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy")
# Prometheii ready - run some queries
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'"))
'';
})

94
nixos/tests/prometheus/pushgateway.nix Normal file
View File

@ -0,0 +1,94 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-pushgateway";
nodes = {
prometheus = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "pushgateway";
static_configs = [
{
targets = [
"pushgateway:9091"
];
}
];
}
];
};
};
pushgateway = { config, pkgs, ... }: {
networking.firewall.allowedTCPPorts = [ 9091 ];
services.prometheus.pushgateway = {
enable = true;
};
};
client = { config, pkgs, ... }: {
};
};
testScript = ''
pushgateway.wait_for_unit("pushgateway")
pushgateway.wait_for_open_port(9091)
pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready")
pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'")
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'"
)
client.wait_for_unit("network-online.target")
# Add a metric and check in Prometheus
client.wait_until_succeeds(
"echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep 'null'"
)
# Delete the metric, check not in Prometheus
client.wait_until_succeeds(
"curl -X DELETE http://pushgateway:9091/metrics/job/some_job"
)
prometheus.wait_until_fails(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
'';
})

73
nixos/tests/prometheus/remote-write.nix Normal file
View File

@ -0,0 +1,73 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
{
name = "prometheus-remote-write";
nodes = {
receiver = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
extraFlags = [ "--web.enable-remote-write-receiver" ];
};
};
prometheus = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
remoteWrite = [
{
url = "http://receiver:9090/api/v1/write";
}
];
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
];
};
};
node = { config, pkgs, ... }: {
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
};
testScript = ''
node.wait_for_unit("prometheus-node-exporter")
node.wait_for_open_port(9100)
for machine in prometheus, receiver:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
# Verify both servers got the same data from the exporter
for machine in prometheus, receiver:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
'';
})

58
nixos/tests/prometheus.nix → nixos/tests/thanos.nix
View File

@ -212,8 +212,6 @@ in import ./make-test-python.nix {
};
testScript = { nodes, ... } : ''
import json
# Before starting the other machines we first make sure that our S3 service is online
# and has a bucket added for thanos:
s3.start()
@ -289,61 +287,5 @@ in import ./make-test-python.nix {
+ "jq .thanos.labels.some_label | "
+ "grep 'required by thanos'"
)
# Check if switching to a NixOS configuration that changes the prometheus
# configuration reloads (instead of restarts) prometheus before the switch
# finishes successfully:
with subtest("config change reloads prometheus"):
# We check if prometheus has finished reloading by looking for the message
# "Completed loading of configuration file" in the journal between the start
# and finish of switching to the new NixOS configuration.
#
# To mark the start we record the journal cursor before starting the switch:
cursor_before_switching = json.loads(
prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
)["__CURSOR"]
# Now we switch:
prometheus_config_change = prometheus.succeed(
"readlink /run/current-system/specialisation/prometheus-config-change"
).strip()
prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
# Next we retrieve all logs since the start of switching:
logs_after_starting_switching = prometheus.succeed(
"""
journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
""".format(
cursor_before_switching=cursor_before_switching
)
)
# Finally we check if the message "Completed loading of configuration file"
# occurs before the "finished switching to system configuration" message:
finished_switching_msg = (
"finished switching to system configuration " + prometheus_config_change
)
reloaded_before_switching_finished = False
finished_switching = False
for log_line in logs_after_starting_switching.split("\n"):
msg = json.loads(log_line)["MESSAGE"]
if "Completed loading of configuration file" in msg:
reloaded_before_switching_finished = True
if msg == finished_switching_msg:
finished_switching = True
break
assert reloaded_before_switching_finished
assert finished_switching
# Check if the reloaded config includes the new s3-node_exporter job:
prometheus.succeed(
"""
curl -sf http://127.0.0.1:${toString queryPort}/api/v1/status/config \
| jq -r .data.yaml \
| yq '.scrape_configs | any(.job_name == "s3-node_exporter")' \
| grep true
"""
)
'';
}

33
pkgs/by-name/al/alertmanager-webhook-logger/package.nix Normal file
View File

@ -0,0 +1,33 @@
{ lib
, stdenv
, buildGoModule
, fetchFromGitHub
, nixosTests
}:
buildGoModule rec {
pname = "alertmanager-webhook-logger";
version = "1.0";
rev = "${version}";
src = fetchFromGitHub {
inherit rev;
owner = "tomtom-international";
repo = "alertmanager-webhook-logger";
hash = "sha256-mJbpDiTwUsFm0lDKz8UE/YF6sBvcSSR6WWLrfKvtri4=";
};
vendorHash = "sha256-gKtOoM9TuEIHgvSjZhqWmdexG2zDjlPuM0HjjP52DOI=";
doCheck = true;
passthru.tests = { inherit (nixosTests.prometheus) alertmanager; };
meta = with lib; {
description = "Generates (structured) log messages from Prometheus AlertManager webhook notifier";
mainProgram = "alertmanager-webhook-logger";
homepage = "https://github.com/tomtom-international/alertmanager-webhook-logger";
license = licenses.asl20;
maintainers = with maintainers; [ jpds ];
};
}

3
pkgs/servers/monitoring/prometheus/alertmanager.nix
View File

@ -3,6 +3,7 @@
, buildGoModule
, fetchFromGitHub
, installShellFiles
, nixosTests
}:
buildGoModule rec {
@ -39,6 +40,8 @@ buildGoModule rec {
installShellCompletion amtool.zsh
'';
passthru.tests = { inherit (nixosTests.prometheus) alertmanager; };
meta = with lib; {
description = "Alert dispatcher for the Prometheus monitoring system";
homepage = "https://github.com/prometheus/alertmanager";

9
pkgs/servers/monitoring/prometheus/pushgateway.nix
View File

@ -1,4 +1,4 @@
{ lib, buildGoModule, fetchFromGitHub, testers, prometheus-pushgateway }:
{ lib, buildGoModule, fetchFromGitHub, nixosTests, testers, prometheus-pushgateway }:
buildGoModule rec {
pname = "pushgateway";
@ -23,8 +23,11 @@ buildGoModule rec {
"-X github.com/prometheus/common/version.BuildDate=19700101-00:00:00"
];
passthru.tests.version = testers.testVersion {
package = prometheus-pushgateway;
passthru.tests = {
inherit (nixosTests.prometheus) pushgateway;
version = testers.testVersion {
package = prometheus-pushgateway;
};
};
meta = with lib; {

2
pkgs/servers/monitoring/thanos/default.nix
View File

@ -37,7 +37,7 @@ buildGoModule rec {
passthru = {
updateScript = nix-update-script { };
tests = {
inherit (nixosTests) prometheus;
inherit (nixosTests) thanos;
version = testers.testVersion {
command = "thanos --version";
package = thanos;