feat: add metrics server (#27)

This PR introduces a prometheus metrics server. **It's not yet
configurable and will be as part of akash-network/support#306.**
This will help monitor the incoming requests and overall health of the
various nodes on the network.
Metrics endpoint snapshot (2,000 requests made, with latency-based load
balancing):
```
# HELP proxy_node_count Number of healthy nodes for each type
# TYPE proxy_node_count gauge
proxy_node_count{type="grpc"} 2
proxy_node_count{type="rest"} 11
proxy_node_count{type="rpc"} 9
# HELP proxy_request_count Number of requests per node for each type
# TYPE proxy_request_count counter
proxy_request_count{node="akash-api.kleomedes.network",type="rest"} 77
proxy_request_count{node="akash-api.polkachu.com",type="rest"} 97
proxy_request_count{node="akash-api.validatornode.com",type="rest"} 53
proxy_request_count{node="akash-api.w3coins.io",type="rest"} 97
proxy_request_count{node="akash-mainnet-rest.cosmonautstakes.com:443",type="rest"} 67
proxy_request_count{node="akash-mainnet-rpc.cosmonautstakes.com:443",type="rpc"} 86
proxy_request_count{node="akash-rest.publicnode.com",type="rest"} 111
proxy_request_count{node="akash-rpc.kleomedes.network",type="rpc"} 101
proxy_request_count{node="akash-rpc.polkachu.com",type="rpc"} 91
proxy_request_count{node="akash-rpc.publicnode.com:443",type="rpc"} 99
proxy_request_count{node="akash-rpc.w3coins.io",type="rpc"} 129
proxy_request_count{node="akash.api.arcturian.tech",type="rest"} 120
proxy_request_count{node="akash.c29r3.xyz:443",type="rest"} 91
proxy_request_count{node="akash.rpc.arcturian.tech",type="rpc"} 167
proxy_request_count{node="api-akash-01.stakeflow.io",type="rest"} 113
proxy_request_count{node="rest-akash.ecostake.com",type="rest"} 83
proxy_request_count{node="rest.lavenderfive.com:443",type="rest"} 91
proxy_request_count{node="rpc-akash.ecostake.com:443",type="rpc"} 94
proxy_request_count{node="rpc.akash.bronbro.io:443",type="rpc"} 127
proxy_request_count{node="rpc.lavenderfive.com:443",type="rpc"} 106
```

<img width="1074" alt="Screenshot 2025-04-21 at 20 45 16"
src="https://github.com/user-attachments/assets/2fa1c177-a75e-4233-b4ba-99dd5e6df174"
/>

Author: cloud-j-luna

README.md

@@ -65,8 +65,12 @@ go run cmd/main.go
 go run cmd/main.go --config=./testdata/local.yaml
 ```
 
-## Building
+## Monitoring
 
+### Deploying locally
+You can deploy the monitoring stack locally by running docker compose.
 ```bash
-go build -o akash-rpc-proxy
+docker-compose -f deploy/docker-compose.yml up -d
 ```
+This will start a Grafana and Prometheus instance with datasources pre-configured as well as the dashboards provisioned.
+Access it on [http://localhost:3000](http://localhost:3000) with the default user and password `admin`.

cmd/main.go

@@ -24,6 +24,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/akash-network/rpc-proxy/internal/config"
+	"github.com/akash-network/rpc-proxy/internal/metrics"
 	"github.com/akash-network/rpc-proxy/internal/proxy"
 	"github.com/akash-network/rpc-proxy/internal/seed"
 	"github.com/spf13/cobra"
@@ -79,7 +80,7 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 
 	// Server configuration
 	rootCmd.PersistentFlags().String("server.listen", ":25567", "Address to listen on for HTTP REST & RPC requests")
-	rootCmd.PersistentFlags().String("server.listen-grpc", ":9090", "Address to listen on for gRPC requests")
+	rootCmd.PersistentFlags().String("server.listen-grpc", ":25568", "Address to listen on for gRPC requests")
 	rootCmd.PersistentFlags().Duration("server.timeouts.read", 10*time.Second, "Server read timeout")
 	rootCmd.PersistentFlags().Duration("server.timeouts.write", 10*time.Second, "Server write timeout")
 	rootCmd.PersistentFlags().Duration("server.timeouts.idle", 10*time.Second, "Server idle timeout")
@@ -94,6 +95,7 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 	rootCmd.PersistentFlags().String("seed.url", "https://raw.githubusercontent.com/cosmos/chain-registry/master/akash/chain.json", "URL to fetch initial node list")
 	rootCmd.PersistentFlags().Duration("seed.refresh-interval", 5*time.Minute, "How often to refresh node list")
 	rootCmd.PersistentFlags().String("seed.chain-id", "akashnet-2", "Expected chain ID")
+	rootCmd.PersistentFlags().Bool("seed.enable-remote", true, "Enable remote seed fetching")
 	rootCmd.PersistentFlags().StringSlice("seed.additional-nodes.rpc", []string{}, "Comma-separated list of additional RPC nodes")
 	rootCmd.PersistentFlags().StringSlice("seed.additional-nodes.rest", []string{}, "Comma-separated list of additional REST nodes")
 	rootCmd.PersistentFlags().StringSlice("seed.additional-nodes.grpc", []string{}, "Comma-separated list of additional gRPC nodes")
@@ -107,6 +109,11 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 	rootCmd.PersistentFlags().String("cors.allow-methods", "GET, POST, PUT, DELETE, OPTIONS", "CORS allowed methods")
 	rootCmd.PersistentFlags().String("cors.allow-headers", "Content-Type, Authorization", "CORS allowed headers")
 
+	// Metrics configuration
+	rootCmd.PersistentFlags().Bool("metrics.enabled", true, "Enable metrics server")
+	rootCmd.PersistentFlags().String("metrics.listen", ":4000", "Address to listen on for metrics")
+	rootCmd.PersistentFlags().String("metrics.path", "/metrics", "Path to expose metrics on")
+
 	// Configuration file support
 	rootCmd.PersistentFlags().StringP("config", "c", "", "config file (default is $HOME/.akash-proxy/config.yaml)")
 
@@ -116,6 +123,17 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 func runProxy(cfg config.Config) {
 	log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
 
+	var metricsServer *http.Server
+	if cfg.Metrics.Enabled {
+		metricsServer = metrics.PrepareMetricsServer(cfg.Metrics.Listen, cfg.Metrics.Path)
+		go func() {
+			log.Info("metrics server", "addr", metricsServer.Addr, "path", cfg.Metrics.Path)
+			if err := metricsServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				panic(err)
+			}
+		}()
+	}
+
 	rpcListener := make(chan seed.Seed, 1)
 	restListener := make(chan seed.Seed, 1)
 	grpcListener := make(chan seed.Seed, 1)
@@ -124,6 +142,7 @@ func runProxy(cfg config.Config) {
 		SeedURL:             cfg.Seed.URL,
 		SeedRefreshInterval: cfg.Seed.RefreshInterval,
 		ChainID:             cfg.Seed.ChainID,
+		EnableRemote:        cfg.Seed.EnableRemote,
 		AdditionalNodes: struct {
 			RPC  []string
 			REST []string
@@ -168,6 +187,13 @@ func runProxy(cfg config.Config) {
 			log.Error("could not close server", "err", err)
 			os.Exit(1)
 		}
+
+		if cfg.Metrics.Enabled && metricsServer != nil {
+			if err := metricsServer.Shutdown(ctx); err != nil {
+				log.Error("could not close metrics server", "err", err)
+				os.Exit(1)
+			}
+		}
 	}()
 
 	proxyGroup.Go(func() error {

deploy/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3.8'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped 

deploy/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'RPC Proxy Dashboards'
+    orgId: 1
+    folder: 'RPC Proxy'
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /etc/grafana/provisioning/dashboards
+      foldersFromFilesStructure: true