diff --git a/.env.sample.holesky b/.env.sample.holesky index 84887bf5..c3019e7b 100644 --- a/.env.sample.holesky +++ b/.env.sample.holesky @@ -259,6 +259,12 @@ LIDODVEXIT_EXIT_EPOCH=256 # See available tags https://github.com/prometheus/prometheus/releases. #PROMETHEUS_VERSION= +# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below. +# Enable developer mode on discord with User Settings > Advanced. +# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. +# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`) +#ALERT_DISCORD_IDS="" + # Uncomment these if you have log exporting with Promtail # and want to disable log export on a particular container. #EL_NETHERMIND_PROMTAIL_MONITORED=false diff --git a/.env.sample.hoodi b/.env.sample.hoodi index 3329a1f7..78b800a8 100644 --- a/.env.sample.hoodi +++ b/.env.sample.hoodi @@ -266,6 +266,12 @@ LIDODVEXIT_EXIT_EPOCH=256 # See available tags https://github.com/prometheus/prometheus/releases. #PROMETHEUS_VERSION= +# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below. +# Enable developer mode on discord with User Settings > Advanced. +# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. +# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`) +#ALERT_DISCORD_IDS="" + # Uncomment these if you have log exporting with Promtail # and want to disable log export on a particular container. #EL_NETHERMIND_PROMTAIL_MONITORED=false diff --git a/.env.sample.mainnet b/.env.sample.mainnet index 709ce0da..b3eebe94 100644 --- a/.env.sample.mainnet +++ b/.env.sample.mainnet @@ -267,6 +267,12 @@ LIDODVEXIT_EXIT_EPOCH=194048 # See available tags https://github.com/prometheus/prometheus/releases. #PROMETHEUS_VERSION= +# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below. +# Enable developer mode on discord with User Settings > Advanced. +# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. +# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`) +#ALERT_DISCORD_IDS="" + # Uncomment these if you have log exporting with Promtail # and want to disable log export on a particular container. #EL_NETHERMIND_PROMTAIL_MONITORED=false diff --git a/.gitignore b/.gitignore index b9c601f4..d76459fa 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ teku/validator/ # Teku directory for storing logs teku/logs/ commit-boost/config.toml +prometheus/prometheus.yml diff --git a/docker-compose.yml b/docker-compose.yml index bab6221a..dd397336 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -198,6 +198,7 @@ services: - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro - ./grafana/dashboards:/etc/dashboards - ./data/grafana:/var/lib/grafana + entrypoint: /etc/prometheus/run.sh restart: unless-stopped loki: diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index a5866c81..cceb57ad 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -21,6 +21,9 @@ scrape_configs: - job_name: "charon" static_configs: - targets: ["charon:3620"] + relabel_configs: + - target_label: alert_discord_ids + replacement: "$ALERT_DISCORD_IDS" - job_name: "lodestar" static_configs: - targets: [ "lodestar:5064" ] diff --git a/prometheus/prometheus.yml.example b/prometheus/prometheus.yml.example new file mode 100644 index 00000000..87ff64de --- /dev/null +++ b/prometheus/prometheus.yml.example @@ -0,0 +1,29 @@ +global: + scrape_interval: 30s # Set the scrape interval to every 30 seconds. + evaluation_interval: 30s # Evaluate rules every 30 seconds. + +remote_write: + - url: https://vm.monitoring.gcp.obol.tech/write + authorization: + credentials: $PROM_REMOTE_WRITE_TOKEN + write_relabel_configs: + - source_labels: [job] + regex: "charon|nethermind|lighthouse|lodestar" + action: keep # Keeps charon metrics and drop metrics from other containers. + +scrape_configs: + - job_name: "nethermind" + static_configs: + - targets: ["nethermind:8008"] + - job_name: "lighthouse" + static_configs: + - targets: ["lighthouse:5054"] + - job_name: "charon" + static_configs: + - targets: ["charon:3620"] + - job_name: "lodestar" + static_configs: + - targets: [ "lodestar:5064" ] + - job_name: "validator-ejector" + static_configs: + - targets: [ "validator-ejector:8989" ] \ No newline at end of file diff --git a/prometheus/run.sh b/prometheus/run.sh new file mode 100644 index 00000000..000941ba --- /dev/null +++ b/prometheus/run.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ -z "$SERVICE_OWNER" ] +then + echo "\$SERVICE_OWNER variable is empty" >&2 + exit 1 +fi + +if [ -z "$PROM_REMOTE_WRITE_TOKEN" ] +then + echo "\$PROM_REMOTE_WRITE_TOKEN variable is empty" >&2 + exit 1 +fi + +sed -e "s|\$PROM_REMOTE_WRITE_TOKEN|${PROM_REMOTE_WRITE_TOKEN}|g" \ + -e "s|\$SERVICE_OWNER|${SERVICE_OWNER}|g" \ + -e "s|\$ALERT_DISCORD_IDS|${ALERT_DISCORD_IDS}|g" \ + /etc/prometheus/prometheus.yml.example > /etc/prometheus/prometheus.yml + +/bin/prometheus \ + --config.file=/etc/prometheus/prometheus.yml