Skip to content

Commit 0d32194

Browse files
noamsp-starkwarenoamsp-starkware
authored
apollo_dashboard: add bytes/sec panels for consensus and mempool p2p (#10564)
1 parent d5125d0 commit 0d32194

File tree

9 files changed

+363
-48
lines changed

9 files changed

+363
-48
lines changed

‎crates/apollo_consensus_manager/src/consensus_manager.rs‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ use apollo_network::gossipsub_impl::Topic;
2626
use apollo_network::metrics::{
2727
BroadcastNetworkMetrics,
2828
EventMetrics,
29+
LabeledMessageMetrics,
2930
LatencyMetrics,
31+
MessageMetrics,
3032
NetworkMetrics,
3133
};
3234
use apollo_network::network_manager::{
@@ -51,13 +53,19 @@ use crate::metrics::{
5153
CONSENSUS_NUM_BLACKLISTED_PEERS,
5254
CONSENSUS_NUM_CONNECTED_PEERS,
5355
CONSENSUS_PING_LATENCY,
56+
CONSENSUS_PROPOSALS_DROPPED_MESSAGE_SIZE,
5457
CONSENSUS_PROPOSALS_NUM_DROPPED_MESSAGES,
5558
CONSENSUS_PROPOSALS_NUM_RECEIVED_MESSAGES,
5659
CONSENSUS_PROPOSALS_NUM_SENT_MESSAGES,
60+
CONSENSUS_PROPOSALS_RECEIVED_MESSAGE_SIZE,
61+
CONSENSUS_PROPOSALS_SENT_MESSAGE_SIZE,
5762
CONSENSUS_REVERTED_BATCHER_UP_TO_AND_INCLUDING,
63+
CONSENSUS_VOTES_DROPPED_MESSAGE_SIZE,
5864
CONSENSUS_VOTES_NUM_DROPPED_MESSAGES,
5965
CONSENSUS_VOTES_NUM_RECEIVED_MESSAGES,
6066
CONSENSUS_VOTES_NUM_SENT_MESSAGES,
67+
CONSENSUS_VOTES_RECEIVED_MESSAGE_SIZE,
68+
CONSENSUS_VOTES_SENT_MESSAGE_SIZE,
6169
};
6270

6371
type ProposalStreamMessage = StreamMessage<ProposalPart, HeightAndRound>;
@@ -168,17 +176,35 @@ impl ConsensusManager {
168176
broadcast_metrics_by_topic.insert(
169177
Topic::new(self.config.votes_topic.clone()).hash(),
170178
BroadcastNetworkMetrics {
171-
num_sent_broadcast_messages: CONSENSUS_VOTES_NUM_SENT_MESSAGES,
172-
num_received_broadcast_messages: CONSENSUS_VOTES_NUM_RECEIVED_MESSAGES,
173-
num_dropped_broadcast_messages: CONSENSUS_VOTES_NUM_DROPPED_MESSAGES,
179+
sent_broadcast_message_metrics: MessageMetrics {
180+
num_messages: CONSENSUS_VOTES_NUM_SENT_MESSAGES,
181+
message_size_mb: Some(CONSENSUS_VOTES_SENT_MESSAGE_SIZE),
182+
},
183+
dropped_broadcast_message_metrics: LabeledMessageMetrics {
184+
num_messages: CONSENSUS_VOTES_NUM_DROPPED_MESSAGES,
185+
message_size_mb: Some(CONSENSUS_VOTES_DROPPED_MESSAGE_SIZE),
186+
},
187+
received_broadcast_message_metrics: MessageMetrics {
188+
num_messages: CONSENSUS_VOTES_NUM_RECEIVED_MESSAGES,
189+
message_size_mb: Some(CONSENSUS_VOTES_RECEIVED_MESSAGE_SIZE),
190+
},
174191
},
175192
);
176193
broadcast_metrics_by_topic.insert(
177194
Topic::new(self.config.proposals_topic.clone()).hash(),
178195
BroadcastNetworkMetrics {
179-
num_sent_broadcast_messages: CONSENSUS_PROPOSALS_NUM_SENT_MESSAGES,
180-
num_received_broadcast_messages: CONSENSUS_PROPOSALS_NUM_RECEIVED_MESSAGES,
181-
num_dropped_broadcast_messages: CONSENSUS_PROPOSALS_NUM_DROPPED_MESSAGES,
196+
sent_broadcast_message_metrics: MessageMetrics {
197+
num_messages: CONSENSUS_PROPOSALS_NUM_SENT_MESSAGES,
198+
message_size_mb: Some(CONSENSUS_PROPOSALS_SENT_MESSAGE_SIZE),
199+
},
200+
dropped_broadcast_message_metrics: LabeledMessageMetrics {
201+
num_messages: CONSENSUS_PROPOSALS_NUM_DROPPED_MESSAGES,
202+
message_size_mb: Some(CONSENSUS_PROPOSALS_DROPPED_MESSAGE_SIZE),
203+
},
204+
received_broadcast_message_metrics: MessageMetrics {
205+
num_messages: CONSENSUS_PROPOSALS_NUM_RECEIVED_MESSAGES,
206+
message_size_mb: Some(CONSENSUS_PROPOSALS_RECEIVED_MESSAGE_SIZE),
207+
},
182208
},
183209
);
184210
let network_manager_metrics = Some(NetworkMetrics {

‎crates/apollo_consensus_manager/src/metrics.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,19 @@ define_metrics!(
1010

1111
// Votes topic metrics
1212
MetricCounter { CONSENSUS_VOTES_NUM_SENT_MESSAGES, "apollo_consensus_votes_num_sent_messages", "The number of messages sent by the consensus p2p component over the Votes topic", init = 0 },
13+
MetricHistogram { CONSENSUS_VOTES_SENT_MESSAGE_SIZE, "apollo_consensus_votes_sent_message_size", "The size in MB of messages sent by the consensus p2p component over the Votes topic" },
1314
MetricCounter { CONSENSUS_VOTES_NUM_RECEIVED_MESSAGES, "apollo_consensus_votes_num_received_messages", "The number of messages received by the consensus p2p component over the Votes topic", init = 0 },
15+
MetricHistogram { CONSENSUS_VOTES_RECEIVED_MESSAGE_SIZE, "apollo_consensus_votes_received_message_size", "The size in MB of messages received by the consensus p2p component over the Votes topic" },
1416
LabeledMetricCounter { CONSENSUS_VOTES_NUM_DROPPED_MESSAGES, "apollo_consensus_votes_num_dropped_messages", "The number of messages dropped by the consensus p2p component over the Votes topic", init = 0, labels = NETWORK_BROADCAST_DROP_LABELS },
17+
MetricHistogram { CONSENSUS_VOTES_DROPPED_MESSAGE_SIZE, "apollo_consensus_votes_dropped_message_size", "The size in MB of messages dropped by the consensus p2p component over the Votes topic" },
1518

1619
// Proposals topic metrics
1720
MetricCounter { CONSENSUS_PROPOSALS_NUM_SENT_MESSAGES, "apollo_consensus_proposals_num_sent_messages", "The number of messages sent by the consensus p2p component over the Proposals topic", init = 0 },
21+
MetricHistogram { CONSENSUS_PROPOSALS_SENT_MESSAGE_SIZE, "apollo_consensus_proposals_sent_message_size", "The size in MB of messages sent by the consensus p2p component over the Proposals topic" },
1822
MetricCounter { CONSENSUS_PROPOSALS_NUM_RECEIVED_MESSAGES, "apollo_consensus_proposals_num_received_messages", "The number of messages received by the consensus p2p component over the Proposals topic", init = 0 },
23+
MetricHistogram { CONSENSUS_PROPOSALS_RECEIVED_MESSAGE_SIZE, "apollo_consensus_proposals_received_message_size", "The size in MB of messages received by the consensus p2p component over the Proposals topic" },
1924
LabeledMetricCounter { CONSENSUS_PROPOSALS_NUM_DROPPED_MESSAGES, "apollo_consensus_proposals_num_dropped_messages", "The number of messages dropped by the consensus p2p component over the Proposals topic", init = 0, labels = NETWORK_BROADCAST_DROP_LABELS },
25+
MetricHistogram { CONSENSUS_PROPOSALS_DROPPED_MESSAGE_SIZE, "apollo_consensus_proposals_dropped_message_size", "The size in MB of messages dropped by the consensus p2p component over the Proposals topic" },
2026

2127
// Network events
2228
LabeledMetricCounter { CONSENSUS_NETWORK_EVENTS, "apollo_consensus_network_events", "Network events counter by event type for consensus", init = 0, labels = EVENT_TYPE_LABELS },
@@ -30,11 +36,17 @@ pub(crate) fn register_metrics() {
3036
CONSENSUS_NUM_BLACKLISTED_PEERS.register();
3137
CONSENSUS_PING_LATENCY.register();
3238
CONSENSUS_VOTES_NUM_SENT_MESSAGES.register();
39+
CONSENSUS_VOTES_SENT_MESSAGE_SIZE.register();
3340
CONSENSUS_VOTES_NUM_RECEIVED_MESSAGES.register();
41+
CONSENSUS_VOTES_RECEIVED_MESSAGE_SIZE.register();
3442
CONSENSUS_VOTES_NUM_DROPPED_MESSAGES.register();
43+
CONSENSUS_VOTES_DROPPED_MESSAGE_SIZE.register();
3544
CONSENSUS_PROPOSALS_NUM_SENT_MESSAGES.register();
3645
CONSENSUS_PROPOSALS_NUM_RECEIVED_MESSAGES.register();
3746
CONSENSUS_PROPOSALS_NUM_DROPPED_MESSAGES.register();
47+
CONSENSUS_PROPOSALS_SENT_MESSAGE_SIZE.register();
48+
CONSENSUS_PROPOSALS_RECEIVED_MESSAGE_SIZE.register();
49+
CONSENSUS_PROPOSALS_DROPPED_MESSAGE_SIZE.register();
3850
CONSENSUS_NETWORK_EVENTS.register();
3951
CONSENSUS_REVERTED_BATCHER_UP_TO_AND_INCLUDING.register();
4052
}

‎crates/apollo_dashboard/resources/dev_grafana.json‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,6 +1157,18 @@
11571157
],
11581158
"extra_params": {}
11591159
},
1160+
{
1161+
"title": "Consensus Votes Sent Message Size MB/sec",
1162+
"description": "The rate of MB per second sent by the consensus p2p component over the Votes topic",
1163+
"type": "timeseries",
1164+
"exprs": [
1165+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_votes_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1166+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_votes_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1167+
],
1168+
"extra_params": {
1169+
"unit": "decmbytes"
1170+
}
1171+
},
11601172
{
11611173
"title": "Consensus Votes Number of Received Messages",
11621174
"description": "The increase in the number of vote messages received by consensus p2p (over the selected time range)",
@@ -1166,6 +1178,18 @@
11661178
],
11671179
"extra_params": {}
11681180
},
1181+
{
1182+
"title": "Consensus Votes Received Message Size MB/sec",
1183+
"description": "The rate of MB per second received by the consensus p2p component over the Votes topic",
1184+
"type": "timeseries",
1185+
"exprs": [
1186+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_votes_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1187+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_votes_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1188+
],
1189+
"extra_params": {
1190+
"unit": "decmbytes"
1191+
}
1192+
},
11691193
{
11701194
"title": "Consensus Votes Dropped Messages By Reason",
11711195
"description": "The number of dropped consensus votes messages, by reason (over the selected time range)",
@@ -1175,6 +1199,18 @@
11751199
],
11761200
"extra_params": {}
11771201
},
1202+
{
1203+
"title": "Consensus Votes Dropped Message Size Bytes/sec",
1204+
"description": "The rate of MB per second dropped by the consensus p2p component over the Votes topic",
1205+
"type": "timeseries",
1206+
"exprs": [
1207+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_votes_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1208+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_votes_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1209+
],
1210+
"extra_params": {
1211+
"unit": "decmbytes"
1212+
}
1213+
},
11781214
{
11791215
"title": "Consensus Conflicting Votes",
11801216
"description": "The increase in the number of conflicting votes (over the selected time range)",
@@ -1193,6 +1229,18 @@
11931229
],
11941230
"extra_params": {}
11951231
},
1232+
{
1233+
"title": "Consensus Proposals Sent Message Size MB/sec",
1234+
"description": "The rate of MB per second sent by the consensus p2p component over the Proposals topic",
1235+
"type": "timeseries",
1236+
"exprs": [
1237+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_proposals_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1238+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_proposals_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1239+
],
1240+
"extra_params": {
1241+
"unit": "decmbytes"
1242+
}
1243+
},
11961244
{
11971245
"title": "Consensus Proposals Number of Received Messages",
11981246
"description": "The increase in the number of proposal messages received by consensus p2p (over the selected time range)",
@@ -1202,6 +1250,18 @@
12021250
],
12031251
"extra_params": {}
12041252
},
1253+
{
1254+
"title": "Consensus Proposals Received Message Size MB/sec",
1255+
"description": "The rate of MB per second received by the consensus p2p component over the Proposals topic",
1256+
"type": "timeseries",
1257+
"exprs": [
1258+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_proposals_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1259+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_proposals_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1260+
],
1261+
"extra_params": {
1262+
"unit": "decmbytes"
1263+
}
1264+
},
12051265
{
12061266
"title": "Consensus Proposals Dropped Messages By Reason",
12071267
"description": "The number of dropped consensus proposals messages, by reason (over the selected time range)",
@@ -1211,6 +1271,18 @@
12111271
],
12121272
"extra_params": {}
12131273
},
1274+
{
1275+
"title": "Consensus Proposals Dropped Message Size MB/sec",
1276+
"description": "The rate of MB per second dropped by the consensus p2p component over the Proposals topic",
1277+
"type": "timeseries",
1278+
"exprs": [
1279+
"histogram_quantile(0.50, sum by (le) (rate(apollo_consensus_proposals_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1280+
"histogram_quantile(0.95, sum by (le) (rate(apollo_consensus_proposals_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1281+
],
1282+
"extra_params": {
1283+
"unit": "decmbytes"
1284+
}
1285+
},
12141286
{
12151287
"title": "Consensus Network Events By Type",
12161288
"description": "Network events received by consensus p2p, by event type (over the selected time range)",
@@ -1255,6 +1327,18 @@
12551327
],
12561328
"extra_params": {}
12571329
},
1330+
{
1331+
"title": "Mempool P2p Sent Message Size MB/sec",
1332+
"description": "The rate of MB per second sent by the mempool p2p component",
1333+
"type": "timeseries",
1334+
"exprs": [
1335+
"histogram_quantile(0.50, sum by (le) (rate(apollo_mempool_p2p_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1336+
"histogram_quantile(0.95, sum by (le) (rate(apollo_mempool_p2p_sent_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1337+
],
1338+
"extra_params": {
1339+
"unit": "decmbytes"
1340+
}
1341+
},
12581342
{
12591343
"title": "Number of received messages",
12601344
"description": "Count of the received p2p messages (10m window)",
@@ -1264,6 +1348,18 @@
12641348
],
12651349
"extra_params": {}
12661350
},
1351+
{
1352+
"title": "Mempool P2p Received Message Size MB/sec",
1353+
"description": "The rate of MB per second received by the mempool p2p component",
1354+
"type": "timeseries",
1355+
"exprs": [
1356+
"histogram_quantile(0.50, sum by (le) (rate(apollo_mempool_p2p_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1357+
"histogram_quantile(0.95, sum by (le) (rate(apollo_mempool_p2p_received_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1358+
],
1359+
"extra_params": {
1360+
"unit": "decmbytes"
1361+
}
1362+
},
12671363
{
12681364
"title": "Mempool P2p Broadcasted Transaction Batch Size",
12691365
"description": "The number of transactions in batches broadcast by the mempool p2p component",
@@ -1292,6 +1388,18 @@
12921388
],
12931389
"extra_params": {}
12941390
},
1391+
{
1392+
"title": "Mempool P2p Dropped Message Size MB/sec",
1393+
"description": "The rate of MB per second dropped by the mempool p2p component",
1394+
"type": "timeseries",
1395+
"exprs": [
1396+
"histogram_quantile(0.50, sum by (le) (rate(apollo_mempool_p2p_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
1397+
"histogram_quantile(0.95, sum by (le) (rate(apollo_mempool_p2p_dropped_message_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
1398+
],
1399+
"extra_params": {
1400+
"unit": "decmbytes"
1401+
}
1402+
},
12951403
{
12961404
"title": "Ping Latency",
12971405
"description": "The ping latency distribution for mempool p2p connections",

0 commit comments

Comments
 (0)