diff --git a/modules/infra/api/iface.c b/modules/infra/api/iface.c index 9d25f4e69..abe552652 100644 --- a/modules/infra/api/iface.c +++ b/modules/infra/api/iface.c @@ -7,6 +7,7 @@ #include "module.h" #include +#include #include static struct gr_iface *iface_to_api(const struct iface *priv) { @@ -235,17 +236,22 @@ METRIC_COUNTER( ); METRIC_COUNTER(m_cp_tx_bytes, "iface_cp_tx_bytes", "Number of bytes transmitted by control plane."); +METRIC_GAUGE(m_speed_bps, "iface_speed_bps", "Interface speed in bits per second."); + static void iface_metrics_collect(struct metrics_writer *w) { struct iface *iface = NULL; struct metrics_ctx ctx; - char vrf[16]; while ((iface = iface_next(GR_IFACE_TYPE_UNDEF, iface)) != NULL) { const struct iface_type *type = iface_type_get(iface->type); + char id_str[8]; + snprintf(id_str, sizeof(id_str), "%u", iface->id); metrics_ctx_init( &ctx, w, + "id", + id_str, "name", iface->name, "type", @@ -258,8 +264,10 @@ static void iface_metrics_collect(struct metrics_writer *w) { ); if (iface->mode == GR_IFACE_MODE_VRF) { - snprintf(vrf, sizeof(vrf), "%u", iface->vrf_id); - metrics_labels_add(&ctx, "vrf", vrf, NULL); + const struct iface *vrf_iface = iface_from_id(iface->vrf_id); + metrics_labels_add( + &ctx, "vrf", vrf_iface ? vrf_iface->name : "[deleted]", NULL + ); } else { const struct iface *domain = iface_from_id(iface->domain_id); metrics_labels_add( @@ -267,6 +275,14 @@ static void iface_metrics_collect(struct metrics_writer *w) { ); } + // Attach the MAC as a label so the address is reported on + // every per-iface metric without requiring a separate API call. + struct rte_ether_addr mac_addr = {0}; + char mac_str[18] = "00:00:00:00:00:00"; + if (iface_get_eth_addr(iface, &mac_addr) == 0) + snprintf(mac_str, sizeof(mac_str), ETH_F, &mac_addr); + metrics_labels_add(&ctx, "mac", mac_str, NULL); + metric_emit(&ctx, &m_up, !!(iface->flags & GR_IFACE_F_UP)); metric_emit(&ctx, &m_running, !!(iface->state & GR_IFACE_S_RUNNING)); metric_emit(&ctx, &m_mtu, iface->mtu); @@ -297,6 +313,10 @@ static void iface_metrics_collect(struct metrics_writer *w) { metric_emit(&ctx, &m_cp_tx_packets, cp_tx_pkts); metric_emit(&ctx, &m_cp_tx_bytes, cp_tx_bytes); + // gr_iface.speed is in Megabit/sec; convert to bit/sec for + // the metric. 0 means unknown / link down. + metric_emit(&ctx, &m_speed_bps, (uint64_t)iface->speed * 1000000ULL); + // Dispatch to type-specific collector if (type->metrics_collect != NULL) type->metrics_collect(&ctx, iface); diff --git a/modules/infra/control/port.c b/modules/infra/control/port.c index a9c1e90b9..4762c89e3 100644 --- a/modules/infra/control/port.c +++ b/modules/infra/control/port.c @@ -746,7 +746,44 @@ METRIC_GAUGE(m_txqs, "iface_port_txqs", "Number of TX queues."); METRIC_GAUGE(m_rxq_size, "iface_port_rxq_size", "Number of descriptors in RX queues."); METRIC_GAUGE(m_txq_size, "iface_port_txq_size", "Number of descriptors in TX queues."); METRIC_COUNTER(m_rx_missed, "iface_port_rx_missed", "Number of packets dropped by HW."); +METRIC_COUNTER(m_rx_errors, "iface_port_rx_errors", "Number of RX packets with errors."); METRIC_COUNTER(m_tx_errors, "iface_port_tx_errors", "Number of TX failures."); +METRIC_COUNTER(m_port_rx_bytes, "iface_port_rx_bytes", "Number of bytes received by HW."); +METRIC_COUNTER(m_port_tx_bytes, "iface_port_tx_bytes", "Number of bytes transmitted by HW."); +METRIC_COUNTER( + m_port_xstat, + "iface_port_xstat", + "Raw PMD extended statistic. The xstat label carries the driver-native counter name." +); + +static void port_xstats_emit(struct metrics_ctx *ctx, uint16_t port_id) { + int n = rte_eth_xstats_get_names(port_id, NULL, 0); + if (n <= 0) + return; + + struct rte_eth_xstat_name *names = calloc(n, sizeof(*names)); + struct rte_eth_xstat *values = calloc(n, sizeof(*values)); + if (names == NULL || values == NULL) + goto out; + + if (rte_eth_xstats_get_names(port_id, names, n) != n) + goto out; + if (rte_eth_xstats_get(port_id, values, n) != n) + goto out; + + // Save the base label set; rewind it between iterations so each emit + // replaces (not appends) the xstat label. + size_t base_len = ctx->labels_len; + for (int i = 0; i < n; i++) { + ctx->labels_len = base_len; + metrics_labels_add(ctx, "xstat", names[i].name, NULL); + metric_emit(ctx, &m_port_xstat, values[i].value); + } + ctx->labels_len = base_len; +out: + free(names); + free(values); +} static void port_metrics_collect(struct metrics_ctx *ctx, const struct iface *iface) { const struct iface_info_port *port = iface_info_port(iface); @@ -765,8 +802,13 @@ static void port_metrics_collect(struct metrics_ctx *ctx, const struct iface *if if (rte_eth_stats_get(port->port_id, &stats) == 0) { metric_emit(ctx, &m_rx_missed, stats.imissed); + metric_emit(ctx, &m_rx_errors, stats.ierrors); metric_emit(ctx, &m_tx_errors, stats.oerrors); + metric_emit(ctx, &m_port_rx_bytes, stats.ibytes); + metric_emit(ctx, &m_port_tx_bytes, stats.obytes); } + + port_xstats_emit(ctx, port->port_id); } static struct event *link_event; diff --git a/modules/infra/control/vlan.c b/modules/infra/control/vlan.c index a44d3d233..d1150a80d 100644 --- a/modules/infra/control/vlan.c +++ b/modules/infra/control/vlan.c @@ -4,6 +4,7 @@ #include "event.h" #include "iface.h" #include "log.h" +#include "metrics.h" #include "module.h" #include "rcu.h" #include "vlan.h" @@ -228,6 +229,13 @@ static void vlan_to_api(void *info, const struct iface *iface) { *api = vlan->base; } +static void vlan_metrics_collect(struct metrics_ctx *ctx, const struct iface *iface) { + const struct iface_info_vlan *vlan = iface_info_vlan(iface); + const struct iface *parent = iface_from_id(vlan->parent_id); + + metrics_labels_add(ctx, "parent", parent ? parent->name : "[deleted]", NULL); +} + static const struct iface_type iface_type_vlan = { .id = GR_IFACE_TYPE_VLAN, .pub_size = sizeof(struct gr_iface_info_vlan), @@ -242,6 +250,7 @@ static const struct iface_type iface_type_vlan = { .del_eth_addr = iface_vlan_del_eth_addr, .set_promisc = iface_vlan_promisc_set, .to_api = vlan_to_api, + .metrics_collect = vlan_metrics_collect, }; static void vlan_init(struct event_base *) { diff --git a/modules/infra/datapath/bond_output.c b/modules/infra/datapath/bond_output.c index 1fea67bdf..cc33530b1 100644 --- a/modules/infra/datapath/bond_output.c +++ b/modules/infra/datapath/bond_output.c @@ -204,7 +204,7 @@ bond_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, const struct iface *member; rte_edge_t edge; - IFACE_STATS_VARS(tx); + IFACE_STATS_VARS(tx, self); for (unsigned i = 0; i < nb_objs; i++) { struct rte_mbuf *mbuf = objs[i]; @@ -224,14 +224,14 @@ bond_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, t->member_iface_id = member->id; } - IFACE_STATS_INC(tx, mbuf, member); + IFACE_STATS_INC(tx, self, mbuf, member); edge = PORT_OUTPUT; next: rte_node_enqueue_x1(graph, node, edge, mbuf); } - IFACE_STATS_FLUSH(tx); + IFACE_STATS_FLUSH(tx, self); return nb_objs; } diff --git a/modules/infra/datapath/iface_input.c b/modules/infra/datapath/iface_input.c index c17d92fc6..e6e41fdb1 100644 --- a/modules/infra/datapath/iface_input.c +++ b/modules/infra/datapath/iface_input.c @@ -52,12 +52,14 @@ static uint16_t iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs) { uint16_t last_iface_id, last_vlan_id; const struct iface *vlan_iface; + const struct iface *parent_iface; struct iface_mbuf_data *d; struct rte_mbuf *m; uint16_t vlan_id; rte_edge_t edge; - IFACE_STATS_VARS(rx); + IFACE_STATS_VARS(rx, self); + IFACE_STATS_VARS(rx, parent); last_iface_id = GR_IFACE_ID_UNDEF; last_vlan_id = UINT16_MAX; @@ -67,6 +69,7 @@ iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, m = objs[i]; d = iface_mbuf_data(m); vlan_id = d->vlan_id; + parent_iface = d->iface; if (d->vlan_id != 0 && d->iface->mode == GR_IFACE_MODE_VRF) { if (last_iface_id != d->iface->id || d->vlan_id != last_vlan_id) { @@ -87,7 +90,9 @@ iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, goto next; } - IFACE_STATS_INC(rx, m, d->iface); + IFACE_STATS_INC(rx, self, m, d->iface); + if (parent_iface != d->iface) + IFACE_STATS_INC(rx, parent, m, parent_iface); edge = edges[d->iface->mode]; next: @@ -100,7 +105,8 @@ iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, rte_node_enqueue_x1(graph, node, edge, m); } - IFACE_STATS_FLUSH(rx); + IFACE_STATS_FLUSH(rx, self); + IFACE_STATS_FLUSH(rx, parent); return nb_objs; } diff --git a/modules/infra/datapath/iface_output.c b/modules/infra/datapath/iface_output.c index 94a01bec4..27a3c6582 100644 --- a/modules/infra/datapath/iface_output.c +++ b/modules/infra/datapath/iface_output.c @@ -57,21 +57,25 @@ static uint16_t iface_output_process( uint16_t nb_objs ) { const struct iface *iface; + const struct iface *parent; struct iface_mbuf_data *d; struct rte_mbuf *m; rte_edge_t edge; - IFACE_STATS_VARS(tx); + IFACE_STATS_VARS(tx, self); + IFACE_STATS_VARS(tx, parent); for (uint16_t i = 0; i < nb_objs; i++) { m = objs[i]; d = iface_mbuf_data(m); iface = d->iface; + parent = NULL; if (iface->type == GR_IFACE_TYPE_VLAN) { const struct iface_info_vlan *vlan = iface_info_vlan(iface); d->vlan_id = vlan->vlan_id; iface = iface_from_id(vlan->parent_id); + parent = iface; } if (gr_mbuf_is_traced(m)) { @@ -89,7 +93,9 @@ static uint16_t iface_output_process( goto next; } - IFACE_STATS_INC(tx, m, d->iface); + IFACE_STATS_INC(tx, self, m, d->iface); + if (parent != NULL) + IFACE_STATS_INC(tx, parent, m, parent); d->iface = iface; edge = iface_type_edges[iface->type]; @@ -97,7 +103,8 @@ static uint16_t iface_output_process( rte_node_enqueue_x1(graph, node, edge, m); } - IFACE_STATS_FLUSH(tx); + IFACE_STATS_FLUSH(tx, self); + IFACE_STATS_FLUSH(tx, parent); return nb_objs; } diff --git a/modules/infra/datapath/rxtx.h b/modules/infra/datapath/rxtx.h index 63729a853..63eb4a027 100644 --- a/modules/infra/datapath/rxtx.h +++ b/modules/infra/datapath/rxtx.h @@ -80,35 +80,37 @@ uint16_t rx_bond_process(struct rte_graph *, struct rte_node *, void **, uint16_ uint16_t tx_process(struct rte_graph *, struct rte_node *, void **, uint16_t); uint16_t tx_shared_process(struct rte_graph *, struct rte_node *, void **, uint16_t); -#define IFACE_STATS_VARS(dir) \ - struct iface_stats *dir##_stats; \ - uint16_t dir##_last_iface_id = GR_IFACE_ID_UNDEF; \ - uint16_t dir##_packets = 0; \ - uint64_t dir##_bytes = 0; +#define IFACE_STATS_VARS(dir, acc) \ + struct iface_stats *dir##_##acc##_stats; \ + uint16_t dir##_##acc##_last_iface_id = GR_IFACE_ID_UNDEF; \ + uint16_t dir##_##acc##_packets = 0; \ + uint64_t dir##_##acc##_bytes = 0; -#define IFACE_STATS_INC(dir, mbuf, iface) \ +#define IFACE_STATS_INC(dir, acc, mbuf, iface) \ do { \ - if (iface->id != dir##_last_iface_id) { \ - if (dir##_packets != 0) { \ - dir##_stats = iface_get_stats( \ - rte_lcore_id(), dir##_last_iface_id \ + if (iface->id != dir##_##acc##_last_iface_id) { \ + if (dir##_##acc##_packets != 0) { \ + dir##_##acc##_stats = iface_get_stats( \ + rte_lcore_id(), dir##_##acc##_last_iface_id \ ); \ - dir##_stats->dir##_packets += dir##_packets; \ - dir##_stats->dir##_bytes += dir##_bytes; \ - dir##_packets = 0; \ - dir##_bytes = 0; \ + dir##_##acc##_stats->dir##_packets += dir##_##acc##_packets; \ + dir##_##acc##_stats->dir##_bytes += dir##_##acc##_bytes; \ + dir##_##acc##_packets = 0; \ + dir##_##acc##_bytes = 0; \ } \ - dir##_last_iface_id = iface->id; \ + dir##_##acc##_last_iface_id = iface->id; \ } \ - dir##_packets += 1; \ - dir##_bytes += rte_pktmbuf_pkt_len(mbuf); \ + dir##_##acc##_packets += 1; \ + dir##_##acc##_bytes += rte_pktmbuf_pkt_len(mbuf); \ } while (0) -#define IFACE_STATS_FLUSH(dir) \ +#define IFACE_STATS_FLUSH(dir, acc) \ do { \ - if (dir##_packets != 0) { \ - dir##_stats = iface_get_stats(rte_lcore_id(), dir##_last_iface_id); \ - dir##_stats->dir##_packets += dir##_packets; \ - dir##_stats->dir##_bytes += dir##_bytes; \ + if (dir##_##acc##_packets != 0) { \ + dir##_##acc##_stats = iface_get_stats( \ + rte_lcore_id(), dir##_##acc##_last_iface_id \ + ); \ + dir##_##acc##_stats->dir##_packets += dir##_##acc##_packets; \ + dir##_##acc##_stats->dir##_bytes += dir##_##acc##_bytes; \ } \ } while (0) diff --git a/modules/infra/datapath/xconnect.c b/modules/infra/datapath/xconnect.c index 3bbd6676e..b2b0b9c11 100644 --- a/modules/infra/datapath/xconnect.c +++ b/modules/infra/datapath/xconnect.c @@ -20,21 +20,21 @@ xconnect_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui struct rte_mbuf *mbuf; rte_edge_t edge; - IFACE_STATS_VARS(rx); - IFACE_STATS_VARS(tx); + IFACE_STATS_VARS(rx, self); + IFACE_STATS_VARS(tx, self); for (uint16_t i = 0; i < nb_objs; i++) { mbuf = objs[i]; iface = mbuf_data(mbuf)->iface; peer = iface_from_id(iface->domain_id); - IFACE_STATS_INC(rx, mbuf, iface); + IFACE_STATS_INC(rx, self, mbuf, iface); if (peer != NULL && peer->type == GR_IFACE_TYPE_PORT) { mbuf_data(mbuf)->iface = peer; edge = OUTPUT; - IFACE_STATS_INC(tx, mbuf, peer); + IFACE_STATS_INC(tx, self, mbuf, peer); } else { edge = NO_PORT; } @@ -45,8 +45,8 @@ xconnect_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui rte_node_enqueue_x1(graph, node, edge, mbuf); } - IFACE_STATS_FLUSH(rx); - IFACE_STATS_FLUSH(tx); + IFACE_STATS_FLUSH(rx, self); + IFACE_STATS_FLUSH(tx, self); return nb_objs; } diff --git a/modules/infra/datapath/xvrf.c b/modules/infra/datapath/xvrf.c index 1879c0f35..1c08c88fa 100644 --- a/modules/infra/datapath/xvrf.c +++ b/modules/infra/datapath/xvrf.c @@ -29,7 +29,7 @@ xvrf_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16 struct rte_mbuf *m; rte_edge_t edge; - IFACE_STATS_VARS(rx); + IFACE_STATS_VARS(rx, self); for (uint16_t i = 0; i < nb_objs; i++) { m = objs[i]; @@ -43,7 +43,7 @@ xvrf_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16 eth_data->domain = ETH_DOMAIN_LOCAL; // XXX: increment tx stats on source VRF - IFACE_STATS_INC(rx, m, eth_data->iface); + IFACE_STATS_INC(rx, self, m, eth_data->iface); if (gr_mbuf_is_traced(m) || (eth_data->iface->flags & GR_IFACE_F_PACKET_TRACE)) { struct trace_vrf_data *t = gr_mbuf_trace_add(m, node, sizeof(*t)); @@ -53,7 +53,7 @@ xvrf_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16 rte_node_enqueue_x1(graph, node, edge, m); } - IFACE_STATS_FLUSH(rx); + IFACE_STATS_FLUSH(rx, self); return nb_objs; } diff --git a/modules/ipip/datapath_in.c b/modules/ipip/datapath_in.c index 688f095f6..c80dab8c9 100644 --- a/modules/ipip/datapath_in.c +++ b/modules/ipip/datapath_in.c @@ -33,7 +33,7 @@ ipip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, struct iface *ipip; rte_edge_t edge; - IFACE_STATS_VARS(rx); + IFACE_STATS_VARS(rx, self); ipip = NULL; last_src = 0; @@ -66,7 +66,7 @@ ipip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, eth_data->iface = ipip; eth_data->domain = ETH_DOMAIN_LOCAL; edge = IP_INPUT; - IFACE_STATS_INC(rx, mbuf, ipip); + IFACE_STATS_INC(rx, self, mbuf, ipip); next: if (gr_mbuf_is_traced(mbuf) || (ipip && ipip->flags & GR_IFACE_F_PACKET_TRACE)) { struct trace_ipip_data *t = gr_mbuf_trace_add(mbuf, node, sizeof(*t)); @@ -75,7 +75,7 @@ ipip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, rte_node_enqueue_x1(graph, node, edge, mbuf); } - IFACE_STATS_FLUSH(rx); + IFACE_STATS_FLUSH(rx, self); return nb_objs; } diff --git a/modules/ipip/datapath_out.c b/modules/ipip/datapath_out.c index 351042e84..0fbe40698 100644 --- a/modules/ipip/datapath_out.c +++ b/modules/ipip/datapath_out.c @@ -34,7 +34,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, struct rte_mbuf *mbuf; rte_edge_t edge; - IFACE_STATS_VARS(tx); + IFACE_STATS_VARS(tx, self); for (uint16_t i = 0; i < nb_objs; i++) { mbuf = objs[i]; @@ -72,7 +72,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, } ip_set_fields(outer, &tunnel); - IFACE_STATS_INC(tx, mbuf, iface); + IFACE_STATS_INC(tx, self, mbuf, iface); // Resolve nexthop for the encapsulated packet. ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote); @@ -82,7 +82,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, rte_node_enqueue_x1(graph, node, edge, mbuf); } - IFACE_STATS_FLUSH(tx); + IFACE_STATS_FLUSH(tx, self); return nb_objs; }