Add ALERT_DISCORD_IDS label and env variable

.env.sample.holesky

@@ -259,6 +259,12 @@ LIDODVEXIT_EXIT_EPOCH=256
 # See available tags https://github.com/prometheus/prometheus/releases.
 #PROMETHEUS_VERSION=
 
+# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below.
+# Enable developer mode on discord with User Settings > Advanced. 
+# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. 
+# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`)
+#ALERT_DISCORD_IDS=""
+
 # Uncomment these if you have log exporting with Promtail
 # and want to disable log export on a particular container.
 #EL_NETHERMIND_PROMTAIL_MONITORED=false

.env.sample.hoodi

@@ -266,6 +266,12 @@ LIDODVEXIT_EXIT_EPOCH=256
 # See available tags https://github.com/prometheus/prometheus/releases.
 #PROMETHEUS_VERSION=
 
+# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below.
+# Enable developer mode on discord with User Settings > Advanced. 
+# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. 
+# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`)
+#ALERT_DISCORD_IDS=""
+
 # Uncomment these if you have log exporting with Promtail
 # and want to disable log export on a particular container.
 #EL_NETHERMIND_PROMTAIL_MONITORED=false

.env.sample.mainnet

@@ -267,6 +267,12 @@ LIDODVEXIT_EXIT_EPOCH=194048
 # See available tags https://github.com/prometheus/prometheus/releases.
 #PROMETHEUS_VERSION=
 
+# To get Alerted with Obol Agent monitoring on Discord, specify your Discord ID(s) below.
+# Enable developer mode on discord with User Settings > Advanced. 
+# Then right click on a user's profile picture or name and select Copy ID to get a unique 18-digit number that represents their account. 
+# Specify multiple discord IDs using comma separation. (e.g. `DISCORD_IDS=123456789098765432,098765432123456789`)
+#ALERT_DISCORD_IDS=""
+
 # Uncomment these if you have log exporting with Promtail
 # and want to disable log export on a particular container.
 #EL_NETHERMIND_PROMTAIL_MONITORED=false

.gitignore

@@ -16,3 +16,4 @@ teku/validator/
 # Teku directory for storing logs
 teku/logs/
 commit-boost/config.toml
+prometheus/prometheus.yml

docker-compose.yml

@@ -198,6 +198,7 @@ services:
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
       - ./grafana/dashboards:/etc/dashboards
       - ./data/grafana:/var/lib/grafana
+    entrypoint: /etc/prometheus/run.sh
     restart: unless-stopped
 
   loki:

prometheus/prometheus.yml

@@ -21,6 +21,9 @@ scrape_configs:
   - job_name: "charon"
     static_configs:
       - targets: ["charon:3620"]
+    relabel_configs:
+      - target_label: alert_discord_ids
+        replacement: "$ALERT_DISCORD_IDS"
   - job_name: "lodestar"
     static_configs:
       - targets: [ "lodestar:5064" ]

prometheus/prometheus.yml.example

@@ -0,0 +1,29 @@
+global:
+  scrape_interval: 30s # Set the scrape interval to every 30 seconds.
+  evaluation_interval: 30s # Evaluate rules every 30 seconds.
+
+remote_write:
+  - url: https://vm.monitoring.gcp.obol.tech/write
+    authorization:
+      credentials: $PROM_REMOTE_WRITE_TOKEN
+    write_relabel_configs:
+      - source_labels: [job]
+        regex: "charon|nethermind|lighthouse|lodestar"
+        action: keep # Keeps charon metrics and drop metrics from other containers.
+
+scrape_configs:
+  - job_name: "nethermind"
+    static_configs:
+      - targets: ["nethermind:8008"]
+  - job_name: "lighthouse"
+    static_configs:
+      - targets: ["lighthouse:5054"]
+  - job_name: "charon"
+    static_configs:
+      - targets: ["charon:3620"]
+  - job_name: "lodestar"
+    static_configs:
+      - targets: [ "lodestar:5064" ]
+  - job_name: "validator-ejector"
+    static_configs:
+      - targets: [ "validator-ejector:8989" ]

prometheus/run.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if [ -z "$SERVICE_OWNER" ]
+then
+  echo "\$SERVICE_OWNER variable is empty" >&2
+  exit 1
+fi
+
+if [ -z "$PROM_REMOTE_WRITE_TOKEN" ]
+then
+  echo "\$PROM_REMOTE_WRITE_TOKEN variable is empty" >&2
+  exit 1
+fi
+
+sed -e "s|\$PROM_REMOTE_WRITE_TOKEN|${PROM_REMOTE_WRITE_TOKEN}|g" \
+    -e "s|\$SERVICE_OWNER|${SERVICE_OWNER}|g" \
+    -e "s|\$ALERT_DISCORD_IDS|${ALERT_DISCORD_IDS}|g" \
+    /etc/prometheus/prometheus.yml.example > /etc/prometheus/prometheus.yml
+
+/bin/prometheus \
+  --config.file=/etc/prometheus/prometheus.yml