diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4f5d09fb7..60a2fd001 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -73,7 +73,7 @@ jobs: libibverbs-dev libasan8 libcmocka-dev libedit-dev libarchive-dev \ libevent-dev libmnl-dev libnuma-dev python3-pyelftools \ socat tcpdump traceroute graphviz iproute2 iputils-ping ndisc6 jq \ - dnsmasq systemd-coredump \ + dnsmasq systemd-coredump libpcap-dev libbpf-dev \ "linux-modules-extra-$(uname -r)" if echo $MESON_EXTRA_OPTS | grep -q frr=enabled ; then sudo apt-get install -qy --no-install-recommends \ @@ -148,7 +148,8 @@ jobs: apt install -qy --no-install-recommends \ make gcc ccache git meson scdoc python3-pyelftools ca-certificates pkg-config \ crossbuild-essential-arm64 libcmocka-dev:arm64 libedit-dev:arm64 \ - libevent-dev:arm64 libmnl-dev:arm64 libnuma-dev:arm64 + libevent-dev:arm64 libmnl-dev:arm64 libnuma-dev:arm64 \ + libpcap-dev:arm64 libbpf-dev:arm64 - uses: actions/checkout@v4 with: persist-credentials: false diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 937ed3ed0..2ac60e591 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -31,7 +31,7 @@ jobs: libjson-c-dev libmnl-dev libnuma-dev libprotobuf-c-dev libreadline-dev \ librtr-dev libtool libyang-dev meson ninja-build \ patch pkg-config protobuf-c-compiler python3-dev python3-pyelftools \ - systemd-dev texinfo + systemd-dev texinfo libpcap-dev libbpf-dev - uses: actions/checkout@v4 with: fetch-depth: 0 # force fetch all history @@ -99,7 +99,7 @@ jobs: libyang-devel libtool make meson ninja-build \ numactl-devel pkgconf protobuf-c-compiler protobuf-c-devel \ python3-pyelftools python3-devel rdma-core-devel readline-devel \ - rpm-build scl-utils systemd + rpm-build scl-utils systemd libpcap-devel libbpf-devel - uses: actions/checkout@v4 with: fetch-depth: 0 # force fetch all history diff --git a/README.md b/README.md index 9ccc92fe5..7680a5bd8 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ image. dnf install git gcc make meson ninja-build pkgconf \ python3-pyelftools scdoc libmnl-devel \ libcmocka-devel libedit-devel libevent-devel numactl-devel \ - libarchive-devel rdma-core-devel + libarchive-devel rdma-core-devel libpcap-devel libbpf-devel ``` or @@ -270,7 +270,7 @@ or apt install git gcc make meson ninja-build pkgconf \ python3-pyelftools scdoc \ libcmocka-dev libedit-dev libevent-dev libnuma-dev libmnl-dev \ - libarchive-dev libibverbs-dev + libarchive-dev libibverbs-dev libpcap-dev libbpf-dev ``` Important: `grout` requires at least `gcc` 13 or `clang` 15. diff --git a/docs/graph.svg b/docs/graph.svg index 81a0301f8..12485a47d 100644 --- a/docs/graph.svg +++ b/docs/graph.svg @@ -1,942 +1,960 @@ - - - - + + + bond_output - -bond_output + +bond_output - + port_output - -port_output + +port_output bond_output->port_output - - + + iface_input - -iface_input + +iface_input + + + +mirror + +mirror + + + +iface_input->mirror + + - + xconnect - -xconnect + +xconnect - + iface_input->xconnect - - + + - + eth_input - -eth_input + +eth_input - + iface_input->eth_input - - + + - + bridge_input - -bridge_input + +bridge_input - + iface_input->bridge_input - - + + iface_output - -iface_output + +iface_output - + iface_output->bond_output - - + + - + +iface_output->mirror + + + + + iface_output->port_output - - + + - + iface_output->bridge_input - - + + - + vxlan_output - -vxlan_output + +vxlan_output - + iface_output->vxlan_output - - + + - + port_tx - -port_tx + +port_tx - + port_output->port_tx - - + + - + port_rx - -port_rx + +port_rx - + port_rx->iface_input - - + + - + xconnect->port_output - - + + - + lacp_input - -lacp_input + +lacp_input eth_input->lacp_input - - + + - + snap_input - -snap_input + +snap_input eth_input->snap_input - - + + - + arp_input - -arp_input + +arp_input eth_input->arp_input - - + + - + ip_input - -ip_input + +ip_input eth_input->ip_input - - + + - + ip6_input - -ip6_input + +ip6_input eth_input->ip6_input - - + + - + eth_output - -eth_output + +eth_output eth_output->iface_output - - + + - + l2_redirect - -l2_redirect + +l2_redirect - + lacp_output - -lacp_output + +lacp_output - + lacp_output->eth_output - - + + - + snap_input->l2_redirect - - + + - + arp_input_reply - -arp_input_reply + +arp_input_reply - + arp_input->arp_input_reply - - + + - + arp_input_request - -arp_input_request + +arp_input_request - + arp_input->arp_input_request - - + + - + arp_output_reply - -arp_output_reply + +arp_output_reply - + arp_output_reply->eth_output - - + + - + arp_output_request - -arp_output_request + +arp_output_request - + arp_output_request->eth_output - - + + - + bridge_flood - -bridge_flood + +bridge_flood - + bridge_flood->iface_input - - + + - + bridge_flood->iface_output - - + + - + vxlan_flood - -vxlan_flood + +vxlan_flood - + bridge_flood->vxlan_flood - - + + - + bridge_input->iface_input - - + + - + bridge_input->iface_output - - + + - + bridge_input->bridge_flood - - + + - + vxlan_flood->iface_output - - + + - + ospf_redirect - -ospf_redirect + +ospf_redirect - + ospf_redirect->l2_redirect - - + + - + loopback_input - -loopback_input + +loopback_input - + loopback_input->ip_input - - + + - + loopback_input->ip6_input - - + + - + loopback_output - -loopback_output + +loopback_output - + xvrf - -xvrf + +xvrf - + xvrf->ip_input - - + + - + xvrf->ip6_input - - + + - + ip_forward - -ip_forward + +ip_forward - + ip_output - -ip_output + +ip_output - + ip_forward->ip_output - - + + - + ip_fragment - -ip_fragment + +ip_fragment - + ip_fragment->ip_output - - + + - + ip_hold - -ip_hold + +ip_hold - + ip_input->ip_forward - - + + - + ip_input_local - -ip_input_local + +ip_input_local - + ip_input->ip_input_local - - + + - + ip_input->ip_output - - + + - + dnat44_dynamic - -dnat44_dynamic + +dnat44_dynamic - + ip_input->dnat44_dynamic - - + + - + dnat44_static - -dnat44_static + +dnat44_static - + ip_input->dnat44_static - - + + - + ip_loadbalance - -ip_loadbalance + +ip_loadbalance - + ip_loadbalance->ip_output - - + + - + ip_input_local->ospf_redirect - - + + - + ipip_input - -ipip_input + +ipip_input - + ip_input_local->ipip_input - - + + - + icmp_input - -icmp_input + +icmp_input - + ip_input_local->icmp_input - - + + - + l4_input_local - -l4_input_local + +l4_input_local - + ip_input_local->l4_input_local - - + + - + ip_output->eth_output - - + + - + ip_output->xvrf - - + + - + ip_output->ip_fragment - - + + - + ip_output->ip_hold - - + + - + ip_output->ip_loadbalance - - + + - + ipip_output - -ipip_output + +ipip_output - + ip_output->ipip_output - - + + - + sr6_output - -sr6_output + +sr6_output - + ip_output->sr6_output - - + + - + ip6_forward - -ip6_forward + +ip6_forward - + ip6_output - -ip6_output + +ip6_output - + ip6_forward->ip6_output - - + + - + ip6_hold - -ip6_hold + +ip6_hold - + ip6_input->ip6_forward - - + + - + ip6_input_local - -ip6_input_local + +ip6_input_local - + ip6_input->ip6_input_local - - + + - + ip6_input->ip6_output - - + + - + sr6_local - -sr6_local + +sr6_local - + ip6_input->sr6_local - - + + - + ip6_loadbalance - -ip6_loadbalance + +ip6_loadbalance - + ip6_loadbalance->ip6_output - - + + - + ip6_input_local->ospf_redirect - - + + - + icmp6_input - -icmp6_input + +icmp6_input - + ip6_input_local->icmp6_input - - + + - + ip6_input_local->l4_input_local - - + + - + ip6_output->eth_output - - + + - + ip6_output->xvrf - - + + - + ip6_output->ip6_hold - - + + - + ip6_output->ip6_loadbalance - - + + - + ip6_output->sr6_output - - + + - + ipip_input->ip_input - - + + - + ipip_output->ip_output - - + + - + vxlan_input - -vxlan_input + +vxlan_input - + vxlan_input->iface_input - - + + - + vxlan_output->ip_output - - + + - + dnat44_dynamic->ip_forward - - + + - + dnat44_dynamic->ip_input_local - - + + - + dnat44_static->ip_forward - - + + - + dnat44_static->ip_input_local - - + + - + sr6_local->ip_input - - + + - + sr6_local->ip6_input - - + + - + sr6_local->ip6_input_local - - + + - + sr6_output->ip6_output - - + + - + icmp_output - -icmp_output + +icmp_output - + icmp_input->icmp_output - - + + - + icmp_local_send - -icmp_local_send + +icmp_local_send - + icmp_local_send->icmp_output - - + + - + icmp_output->ip_output - - + + - + icmp6_output - -icmp6_output + +icmp6_output - + icmp6_input->icmp6_output - - + + - + ndp_na_input - -ndp_na_input + +ndp_na_input - + icmp6_input->ndp_na_input - - + + - + ndp_ns_input - -ndp_ns_input + +ndp_ns_input - + icmp6_input->ndp_ns_input - - + + - + ndp_rs_input - -ndp_rs_input + +ndp_rs_input - + icmp6_input->ndp_rs_input - - + + - + icmp6_local_send - -icmp6_local_send + +icmp6_local_send - + icmp6_local_send->icmp6_output - - + + - + icmp6_output->ip6_output - - + + - + ndp_na_output - -ndp_na_output + +ndp_na_output - + ndp_na_output->icmp6_output - - + + - + ndp_ns_output - -ndp_ns_output + +ndp_ns_output - + ndp_ns_output->icmp6_output - - + + - + l4_input_local->vxlan_input - - + + - + l4_loopback_output - -l4_loopback_output + +l4_loopback_output - + l4_input_local->l4_loopback_output - - + + - + dhcp_input - -dhcp_input + +dhcp_input - + l4_input_local->dhcp_input - - + + - + l4_loopback_output->loopback_output - - + + diff --git a/meson.build b/meson.build index 6ef4ac068..1bb3e59df 100644 --- a/meson.build +++ b/meson.build @@ -65,7 +65,7 @@ dpdk_dep = dependency( 'werror=false', 'tests=false', 'enable_drivers=net/virtio,net/vhost,net/i40e,net/ice,net/iavf,net/ixgbe,net/null,net/tap,common/mlx5,net/mlx5,bus/auxiliary,net/vmxnet3', - 'enable_libs=graph,hash,fib,rib,pcapng,gso,vhost,cryptodev,dmadev,security', + 'enable_libs=graph,hash,fib,rib,pcapng,bpf,gso,vhost,cryptodev,dmadev,security', 'disable_apps=*', 'enable_docs=false', 'developer_mode=disabled', @@ -103,6 +103,11 @@ elif compiler.has_function( grout_cflags += ['-DHAVE_RTE_FIB_TBL8_GET_STATS'] endif +pcap_dep = dependency('libpcap', required: false, method: 'pkg-config') +if not pcap_dep.found() + pcap_dep = compiler.find_library('pcap', required: true) +endif + src = [] inc = [] @@ -124,7 +129,7 @@ subdir('frr') grout_exe = executable( 'grout', src, include_directories: inc + api_inc, - dependencies: [dpdk_dep, ev_core_dep, ev_extra_dep, ev_thread_dep, mnl_dep, numa_dep], + dependencies: [dpdk_dep, ev_core_dep, ev_extra_dep, ev_thread_dep, mnl_dep, numa_dep, pcap_dep], c_args: ['-D__GROUT_MAIN__'] + grout_cflags, install: true, ) diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index 667517711..920bc09b1 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -34,6 +34,7 @@ typedef enum : uint16_t { GR_IFACE_F_PACKET_TRACE = GR_BIT16(2), GR_IFACE_F_SNAT_STATIC = GR_BIT16(3), GR_IFACE_F_SNAT_DYNAMIC = GR_BIT16(4), + GR_IFACE_F_MIRROR = GR_BIT16(5), } gr_iface_flags_t; // Interface state flags. @@ -70,6 +71,9 @@ typedef enum : uint8_t { #define GR_IFACE_SET_VRF GR_BIT64(3) #define GR_IFACE_SET_DOMAIN GR_BIT64(4) #define GR_IFACE_SET_DESCR GR_BIT64(5) +#define GR_IFACE_SET_MIRROR_FILTER GR_BIT64(6) + +#define GR_IFACE_MIRROR_FILTER_SIZE 64 // Generic struct for all network interfaces. struct __gr_iface_base { @@ -94,6 +98,7 @@ struct gr_iface { char name[IFNAMSIZ]; // NUL terminated. char description[256]; // NUL terminated. + char mirror_filter[GR_IFACE_MIRROR_FILTER_SIZE]; // BPF filter (tcpdump expr), empty = none. uint8_t info[]; // Type specific interface info. }; @@ -436,6 +441,16 @@ struct gr_affinity_cpu_set_req { // struct gr_affinity_cpu_set_resp { }; +// Mirror capture (pcapng) +#define GR_MIRROR_CAPTURE_SET REQUEST_TYPE(GR_INFRA_MODULE, 0x0080) + +#define GR_MIRROR_CAPTURE_PATH_SIZE 256 + +struct gr_mirror_capture_set_req { + bool enabled; + char path[GR_MIRROR_CAPTURE_PATH_SIZE]; +}; + // Helper function to convert iface type enum to string static inline const char *gr_iface_type_name(gr_iface_type_t type) { switch (type) { diff --git a/modules/infra/cli/gr_cli_iface.h b/modules/infra/cli/gr_cli_iface.h index 16f472e2e..7f95ee06b 100644 --- a/modules/infra/cli/gr_cli_iface.h +++ b/modules/infra/cli/gr_cli_iface.h @@ -73,7 +73,8 @@ int arg_iface( CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("set", "Modify an existing interface.")) #define IFACE_ATTRS_CMD \ - "(up|down),(promisc PROMISC),(mtu MTU),((vrf VRF)|(domain DOMAIN)),(description DESCR)" + "(up|down),(promisc PROMISC),(mtu MTU),((vrf VRF)|(domain DOMAIN)),(description DESCR)," \ + "(mirror MIRROR),(filter FILTER)" #define IFACE_ATTRS_ARGS \ with_help("Set the interface UP.", ec_node_str("up", "up")), \ @@ -81,6 +82,14 @@ int arg_iface( "Enable/disable promiscuous mode.", \ EC_NODE_OR("PROMISC", ec_node_str("", "on"), ec_node_str("", "off")) \ ), \ + with_help( \ + "Enable/disable port mirroring.", \ + EC_NODE_OR("MIRROR", ec_node_str("", "on"), ec_node_str("", "off")) \ + ), \ + with_help( \ + "BPF filter expression (tcpdump syntax), empty to clear.", \ + ec_node("any", "FILTER") \ + ), \ with_help("Set the interface DOWN.", ec_node_str("down", "down")), \ with_help( \ "Maximum transmission unit size.", \ diff --git a/modules/infra/cli/iface.c b/modules/infra/cli/iface.c index 592e3f3fb..73a41ae69 100644 --- a/modules/infra/cli/iface.c +++ b/modules/infra/cli/iface.c @@ -210,6 +210,8 @@ static ssize_t iface_flags_format(char *buf, size_t len, const struct gr_iface * SAFE_BUF(snprintf, len, " tracing"); if (iface->flags & (GR_IFACE_F_SNAT_STATIC | GR_IFACE_F_SNAT_DYNAMIC)) SAFE_BUF(snprintf, len, " snat"); + if (iface->flags & GR_IFACE_F_MIRROR) + SAFE_BUF(snprintf, len, " mirror"); return n; err: @@ -223,7 +225,7 @@ uint64_t parse_iface_args( size_t info_size, bool update ) { - const char *name, *promisc; + const char *name, *promisc, *mirror, *filter; uint64_t set_attrs = 0; name = arg_str(p, "NAME"); @@ -256,6 +258,24 @@ uint64_t parse_iface_args( iface->flags &= ~GR_IFACE_F_PROMISC; set_attrs |= GR_IFACE_SET_FLAGS; } + mirror = arg_str(p, "MIRROR"); + if (mirror != NULL && strcmp(mirror, "on") == 0) { + iface->flags |= GR_IFACE_F_MIRROR; + set_attrs |= GR_IFACE_SET_FLAGS; + } else if (mirror != NULL && strcmp(mirror, "off") == 0) { + iface->flags &= ~GR_IFACE_F_MIRROR; + set_attrs |= GR_IFACE_SET_FLAGS; + } + filter = arg_str(p, "FILTER"); + if (filter != NULL) { + if (strlen(filter) >= GR_IFACE_MIRROR_FILTER_SIZE) { + errno = ENAMETOOLONG; + goto err; + } + memccpy(iface->mirror_filter, filter, 0, GR_IFACE_MIRROR_FILTER_SIZE); + iface->mirror_filter[GR_IFACE_MIRROR_FILTER_SIZE - 1] = '\0'; + set_attrs |= GR_IFACE_SET_MIRROR_FILTER; + } if (arg_u16(p, "MTU", &iface->mtu) == 0) set_attrs |= GR_IFACE_SET_MTU; @@ -517,6 +537,13 @@ static cmd_status_t iface_show(struct gr_api_client *c, const struct ec_pnode *p else gr_object_field(o, "speed", GR_DISP_INT, "%u", iface->speed); + if (iface->flags & GR_IFACE_F_MIRROR) { + if (iface->mirror_filter[0] != '\0') + printf("mirror-filter: %s\n", iface->mirror_filter); + else + printf("mirror-filter: (none)\n"); + } + type = type_from_id(iface->type); assert(type != NULL); type->show(c, iface, o); diff --git a/modules/infra/control/gr_iface.h b/modules/infra/control/gr_iface.h index 043cd77ec..c60aa16fd 100644 --- a/modules/infra/control/gr_iface.h +++ b/modules/infra/control/gr_iface.h @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -14,15 +15,19 @@ #include #include +struct rte_bpf; + struct __rte_cache_aligned iface { BASE(__gr_iface_base); gr_vec struct iface **subinterfaces; char *name; char *description; + char *mirror_filter; int cp_id; // Control plane (Linux) port ID int cp_fd; // control plane fd struct event *cp_ev; // libevent to poll cp_fd + struct rte_bpf *mirror_bpf; // Compiled BPF filter, NULL = mirror all. alignas(alignof(void *)) uint8_t info[/* size depends on type */]; }; @@ -82,6 +87,8 @@ int iface_set_up_down(struct iface *, bool up); int iface_set_promisc(struct iface *, bool enabled); uint16_t ifaces_count(gr_iface_type_t type_id); struct iface *iface_next(gr_iface_type_t type_id, const struct iface *prev); +int iface_mirror_filter_compile(const char *expr, struct rte_bpf **out); +void iface_mirror_filter_destroy(struct rte_bpf *bpf); uint16_t vrf_default_get_or_create(void); diff --git a/modules/infra/control/iface.c b/modules/infra/control/iface.c index 9e524ff30..db06de832 100644 --- a/modules/infra/control/iface.c +++ b/modules/infra/control/iface.c @@ -373,11 +373,45 @@ int iface_reconfig( goto err; } + if (set_attrs & GR_IFACE_SET_MIRROR_FILTER) { + size_t len = strnlen(conf->mirror_filter, sizeof(conf->mirror_filter)); + char *new_mirror_filter = NULL; + + if (len >= sizeof(conf->mirror_filter)) + return errno_set(ENAMETOOLONG); + if (len > 0) { + new_mirror_filter = strdup(conf->mirror_filter); + if (new_mirror_filter == NULL) + return errno_set(ENOMEM); + } + free(iface->mirror_filter); + iface->mirror_filter = new_mirror_filter; + if (iface->mirror_filter[0] != '\0') { + struct rte_bpf *old_bpf = iface->mirror_bpf; + struct rte_bpf *new_bpf = NULL; + if((ret = iface_mirror_filter_compile(conf->mirror_filter, &new_bpf)) < 0) + return errno_set(-ret); + iface->mirror_bpf = new_bpf; + rte_rcu_qsbr_synchronize(gr_datapath_rcu(), RTE_QSBR_THRID_INVALID); + iface_mirror_filter_destroy(old_bpf); + } + } + + if (set_attrs & GR_IFACE_SET_FLAGS) { if ((ret = iface_set_promisc(iface, conf->flags & GR_IFACE_F_PROMISC)) < 0) goto err; if ((ret = iface_set_up_down(iface, conf->flags & GR_IFACE_F_UP)) < 0) goto err; + if (conf->flags & GR_IFACE_F_MIRROR) { + iface->flags |= GR_IFACE_F_MIRROR; + } else { + iface->flags &= ~GR_IFACE_F_MIRROR; + struct rte_bpf *old = iface->mirror_bpf; + iface->mirror_bpf = NULL; + rte_rcu_qsbr_synchronize(gr_datapath_rcu(), RTE_QSBR_THRID_INVALID); + iface_mirror_filter_destroy(old); + } } if (set_attrs & GR_IFACE_SET_VRF) { @@ -612,6 +646,8 @@ int iface_destroy(struct iface *iface) { iface->flags &= ~GR_IFACE_F_UP; gr_event_push(GR_EVENT_IFACE_STATUS_DOWN, iface); } + iface_mirror_filter_destroy(iface->mirror_bpf); + iface->mirror_bpf = NULL; detach_domain(iface); ifaces[iface->id] = NULL; diff --git a/modules/infra/control/meson.build b/modules/infra/control/meson.build index c9008981c..5c80b4ad5 100644 --- a/modules/infra/control/meson.build +++ b/modules/infra/control/meson.build @@ -11,6 +11,7 @@ src += files( 'lacp.c', 'loopback.c', 'mempool.c', + 'mirror.c', 'netlink.c', 'nexthop.c', 'port.c', diff --git a/modules/infra/control/mirror.c b/modules/infra/control/mirror.c new file mode 100644 index 000000000..2796d3d44 --- /dev/null +++ b/modules/infra/control/mirror.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 Christophe Fontaine + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define MIRROR_PCAPNG_POOL "mirror_pcapng_pool" +#define MIRROR_PCAPNG_POOL_SIZE (RTE_GRAPH_BURST_SIZE * 4) + +static rte_pcapng_t *mirror_pcapng; +static struct rte_mempool *mirror_pcapng_mp; +static atomic_bool mirror_pcapng_enabled = false; + +void mirror_pcapng_cb(void *obj, uintptr_t, const struct control_queue_drain *drain) { + struct rte_mbuf *m = obj; + struct mbuf_data *d = mbuf_data(m); + + if (drain != NULL && drain->event == GR_EVENT_IFACE_REMOVE && d->iface == drain->obj) + goto free_mbuf; + + if (!atomic_load(&mirror_pcapng_enabled) || mirror_pcapng == NULL + || mirror_pcapng_mp == NULL) + goto free_mbuf; + + ssize_t written = rte_pcapng_write_packets(mirror_pcapng, &m, 1); + if (written < 0) + LOG(ERR, "mirror pcapng write: %s", rte_strerror(rte_errno)); + +free_mbuf: + rte_pktmbuf_free(m); +} + +bool mirror_pcapng_enabled_get(void) { + return atomic_load(&mirror_pcapng_enabled); +} + +static int mirror_pcapng_open(const char *path) { + int fd; + + if (mirror_pcapng != NULL) + return 0; + + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + return errno_set(errno); + + mirror_pcapng = rte_pcapng_fdopen(fd, NULL, NULL, GROUT_VERSION, NULL); + if (mirror_pcapng == NULL) { + close(fd); + return errno_set(rte_errno); + } + + return 0; +} + +static void mirror_pcapng_close(void) { +} + +int mirror_pcapng_enable(const char *path) { + int ret; + + if (path == NULL || path[0] == '\0') + return errno_set(EINVAL); + + ret = mirror_pcapng_open(path); + if (ret < 0) + return ret; + + atomic_store(&mirror_pcapng_enabled, true); + return 0; +} + +void mirror_pcapng_disable(void) { + atomic_store(&mirror_pcapng_enabled, false); + mirror_pcapng_close(); +} + +static void mirror_module_fini(struct event_base *) { + mirror_pcapng_disable(); +} + +static struct gr_module mirror_module = { + .name = "mirror", + .depends_on = "graph", + .fini = mirror_module_fini, +}; + +static struct api_out mirror_capture_set(const void *request, struct api_ctx *) { + const struct gr_mirror_capture_set_req *req = request; + + if (req->enabled) { + if (req->path[0] == '\0') + return api_out(EINVAL, 0, NULL); + if (mirror_pcapng_enable(req->path) < 0) + return api_out(errno, 0, NULL); + } else { + mirror_pcapng_disable(); + } + return api_out(0, 0, NULL); +} + +RTE_INIT(mirror_constructor) { + gr_register_module(&mirror_module); + gr_api_handler(GR_MIRROR_CAPTURE_SET, mirror_capture_set); +} + +int iface_mirror_filter_compile(const char *expr, struct rte_bpf **out) { + struct bpf_program fcode; + struct rte_bpf_prm *prm; + struct rte_bpf *bpf; + pcap_t *pcap; + + *out = NULL; + + if (expr == NULL || expr[0] == '\0') + return 0; + + pcap = pcap_open_dead(DLT_EN10MB, 262144); + if (pcap == NULL) { + LOG(ERR, "pcap_open_dead failed"); + return errno_set(EINVAL); + } + + if (pcap_compile(pcap, &fcode, expr, 1, PCAP_NETMASK_UNKNOWN) != 0) { + LOG(ERR, "mirror filter \"%s\": %s", expr, pcap_geterr(pcap)); + pcap_close(pcap); + return errno_set(EINVAL); + } + + prm = rte_bpf_convert(&fcode); + pcap_freecode(&fcode); + pcap_close(pcap); + + if (prm == NULL) { + LOG(ERR, "rte_bpf_convert \"%s\": %s", expr, rte_strerror(rte_errno)); + return -rte_errno; + } + + bpf = rte_bpf_load(prm); + rte_free(prm); + + if (bpf == NULL) { + LOG(ERR, "rte_bpf_load \"%s\": %s", expr, rte_strerror(rte_errno)); + return -rte_errno; + } + + *out = bpf; + return 0; +} + +void iface_mirror_filter_destroy(struct rte_bpf *bpf) { + if (bpf != NULL) + rte_bpf_destroy(bpf); +} diff --git a/modules/infra/datapath/gr_mbuf.h b/modules/infra/datapath/gr_mbuf.h index 369421f3b..95cba8e52 100644 --- a/modules/infra/datapath/gr_mbuf.h +++ b/modules/infra/datapath/gr_mbuf.h @@ -70,6 +70,9 @@ void gr_mbuf_trace_copy(struct rte_mbuf *dst, struct rte_mbuf *src); // Detach the trace items from an mbuf and store them in the trace buffer. void gr_mbuf_trace_finish(struct rte_mbuf *m); +// dynfield to store a unique packet id, for pcapng +extern int pcapng_packetid_offset; + // Deep copy of an mbuf: duplicates mbuf, copies mbuf priv data and traces static inline struct rte_mbuf *gr_mbuf_copy(struct rte_mbuf *m, size_t data_len) { struct rte_mbuf *copy = rte_pktmbuf_copy(m, m->pool, 0, data_len); diff --git a/modules/infra/datapath/iface_input.c b/modules/infra/datapath/iface_input.c index b589a0797..469c85295 100644 --- a/modules/infra/datapath/iface_input.c +++ b/modules/infra/datapath/iface_input.c @@ -12,10 +12,14 @@ #include #include +#include +#include + enum { IFACE_MODE_UNKNOWN = 0, IFACE_DOWN, UNKNOWN_VLAN, + MIRROR, NB_EDGES, }; @@ -50,11 +54,13 @@ static int iface_input_trace_format(char *buf, size_t len, const void *data, siz return -1; } + static uint16_t iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs) { uint16_t last_iface_id, last_vlan_id; const struct iface *vlan_iface; struct iface_mbuf_data *d; + uint16_t copy_count = 0; struct rte_mbuf *m; uint16_t vlan_id; rte_edge_t edge; @@ -89,11 +95,30 @@ iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, goto next; } + if (d->iface->flags & GR_IFACE_F_MIRROR) { + int copy = 1; + if (d->iface->mirror_bpf) { + struct rte_bpf_jit jit; + rte_bpf_get_jit(d->iface->mirror_bpf, &jit); + if (jit.func) + copy = jit.func(m); + else + copy = rte_bpf_exec(d->iface->mirror_bpf, m); + } + if (copy) { + struct rte_mbuf *c = rte_pcapng_copy(d->iface->id, 0, m, m->pool, rte_pktmbuf_pkt_len(m), RTE_PCAPNG_DIRECTION_IN, ""); + rte_node_enqueue_x1(graph, node, MIRROR, c); + copy_count++; + } + } + IFACE_STATS_INC(rx, m, d->iface); edge = edges[d->iface->mode]; next: if (gr_mbuf_is_traced(m)) { + if (pcapng_packetid_offset >= 0) { } + struct iface_input_trace_data *t = gr_mbuf_trace_add(m, node, sizeof(*t)); t->iface_id = d->iface->id; t->mode = d->iface->mode; @@ -104,7 +129,7 @@ iface_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, IFACE_STATS_FLUSH(rx); - return nb_objs; + return nb_objs + copy_count; } static struct rte_node_register node = { @@ -117,6 +142,7 @@ static struct rte_node_register node = { [IFACE_MODE_UNKNOWN] = "iface_mode_unknown", [IFACE_DOWN] = "iface_input_admin_down", [UNKNOWN_VLAN] = "iface_input_unknown_vlan", + [MIRROR] = "mirror", // other edges are updated dynamically with iface_input_mode_register }, }; diff --git a/modules/infra/datapath/iface_output.c b/modules/infra/datapath/iface_output.c index 146419ebb..84cabbc21 100644 --- a/modules/infra/datapath/iface_output.c +++ b/modules/infra/datapath/iface_output.c @@ -10,12 +10,16 @@ #include #include +#include +#include + #include enum { INVAL = 0, IFACE_DOWN, NO_PARENT, + MIRROR, NB_EDGES, }; @@ -58,6 +62,7 @@ static uint16_t iface_output_process( ) { const struct iface *iface; struct iface_mbuf_data *d; + uint16_t copy_count = 0; struct rte_mbuf *m; rte_edge_t edge; @@ -88,6 +93,22 @@ static uint16_t iface_output_process( edge = IFACE_DOWN; goto next; } + if (iface->flags & GR_IFACE_F_MIRROR) { + int copy = 1; + if (d->iface->mirror_bpf) { + struct rte_bpf_jit jit; + rte_bpf_get_jit(d->iface->mirror_bpf, &jit); + if (jit.func) + copy = jit.func(m); + else + copy = rte_bpf_exec(d->iface->mirror_bpf, m); + } + if (copy) { + struct rte_mbuf *c = rte_pcapng_copy(d->iface->id, 0, m, m->pool, rte_pktmbuf_pkt_len(m), RTE_PCAPNG_DIRECTION_OUT, ""); + rte_node_enqueue_x1(graph, node, MIRROR, c); + copy_count++; + } + } IFACE_STATS_INC(tx, m, d->iface); @@ -99,7 +120,7 @@ static uint16_t iface_output_process( IFACE_STATS_FLUSH(tx); - return nb_objs; + return nb_objs + copy_count; } static struct rte_node_register node = { @@ -112,6 +133,7 @@ static struct rte_node_register node = { [INVAL] = "iface_output_inval_type", [IFACE_DOWN] = "iface_output_admin_down", [NO_PARENT] = "iface_output_vlan_no_parent", + [MIRROR] = "mirror", }, }; diff --git a/modules/infra/datapath/meson.build b/modules/infra/datapath/meson.build index b784fa981..6c915841d 100644 --- a/modules/infra/datapath/meson.build +++ b/modules/infra/datapath/meson.build @@ -15,6 +15,7 @@ src += files( 'lacp_output.c', 'loop_input.c', 'loop_output.c', + 'mirror.c', 'xvrf.c', 'main_loop.c', 'port_output.c', diff --git a/modules/infra/datapath/mirror.c b/modules/infra/datapath/mirror.c new file mode 100644 index 000000000..1815479a4 --- /dev/null +++ b/modules/infra/datapath/mirror.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 Christophe Fontaine + +#include +#include +#include +#include +#include +#include +#include + +enum { + PCAPNG, + NB_EDGES, +}; + +static uint16_t +mirror_process(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs) { + struct mbuf_data *d; + struct rte_mbuf *m; + + for (uint16_t i = 0; i < nb_objs; i++) { + m = objs[i]; + d = mbuf_data(m); + if (gr_mbuf_is_traced(m)) + gr_mbuf_trace_add(m, node, 0); + + trace_log_packet(m, "mirror", d->iface->name); + control_output_set_cb(m, mirror_pcapng_cb, 0); + rte_node_enqueue_x1(graph, node, PCAPNG, m); + } + + return nb_objs; +} + +static struct rte_node_register node = { + .name = "mirror", + .process = mirror_process, + .nb_edges = NB_EDGES, + .next_nodes = { + [PCAPNG] = "control_output", + }, +}; + +static struct gr_node_info info = { + .node = &node, + .type = GR_NODE_T_L1, +}; + +GR_NODE_REGISTER(info); +GR_DROP_REGISTER(mirror_drop); diff --git a/subprojects/dpdk.wrap b/subprojects/dpdk.wrap index e9225123e..3ed8a442d 100644 --- a/subprojects/dpdk.wrap +++ b/subprojects/dpdk.wrap @@ -6,7 +6,9 @@ diff_files = dpdk/iavf-fix-reported-max-TX-and-RX-queues-in-ethdev-inf.patch, dpdk/fib-expose-tbl8-usage-statistics.patch, dpdk/hash-avoid-leaking-entries-on-RCU-defer-queue-failur.patch, - dpdk/hash-free-replaced-data-on-overwrite-when-RCU-is-con.patch + dpdk/hash-free-replaced-data-on-overwrite-when-RCU-is-con.patch, + dpdk/0001-bpf-arm64-fix-offset-type-to-allow-a-negative-jump.patch, + dpdk/0002-bpf-arm64-support-packet-data-load-instructions.patch [provide] dependency_names = libdpdk diff --git a/subprojects/packagefiles/dpdk/0001-bpf-arm64-fix-offset-type-to-allow-a-negative-jump.patch b/subprojects/packagefiles/dpdk/0001-bpf-arm64-fix-offset-type-to-allow-a-negative-jump.patch new file mode 100644 index 000000000..98569d46a --- /dev/null +++ b/subprojects/packagefiles/dpdk/0001-bpf-arm64-fix-offset-type-to-allow-a-negative-jump.patch @@ -0,0 +1,33 @@ +From 9778a186b5c0ad9a3615acc42a57af2a2ad90c07 Mon Sep 17 00:00:00 2001 +From: Christophe Fontaine +Date: Thu, 19 Mar 2026 12:35:32 +0100 +Subject: [PATCH 1/2] bpf/arm64: fix offset type to allow a negative jump + +The epilogue code can be before the call to +'emit_return_zero_if_src_zero', but the code only supported a +jump forward. + +Fix the type of jump_to_epilogue from unsigned to signed, +to be able to jump backward. + +Signed-off-by: Christophe Fontaine +--- + lib/bpf/bpf_jit_arm64.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c +index a04ef33a9c..099822e9f1 100644 +--- a/lib/bpf/bpf_jit_arm64.c ++++ b/lib/bpf/bpf_jit_arm64.c +@@ -957,7 +957,7 @@ static void + emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src) + { + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); +- uint16_t jump_to_epilogue; ++ int32_t jump_to_epilogue; + + emit_cbnz(ctx, is64, src, 3); + emit_mov_imm(ctx, is64, r0, 0); +-- +2.53.0 + diff --git a/subprojects/packagefiles/dpdk/0002-bpf-arm64-support-packet-data-load-instructions.patch b/subprojects/packagefiles/dpdk/0002-bpf-arm64-support-packet-data-load-instructions.patch new file mode 100644 index 000000000..ad3f89586 --- /dev/null +++ b/subprojects/packagefiles/dpdk/0002-bpf-arm64-support-packet-data-load-instructions.patch @@ -0,0 +1,283 @@ +From d4197fdb6a8b0811f3d42b3cec331e99d3a4ce40 Mon Sep 17 00:00:00 2001 +From: Christophe Fontaine +Date: Thu, 19 Mar 2026 12:35:46 +0100 +Subject: [PATCH 2/2] bpf/arm64: support packet data load instructions + +arm64 jit compiler didn't support reading from a packet. +Enable arm64 JIT to generate native code for +(BPF_ABS | | BPF_LD) and (BPF_IND | | BPF_LD) +instructions. + +Both 'fast path' and 'slow path' are implemented, similar to +the x86_64 JIT, where we call '__rte_pktmbuf_read' if the +requested data is not in the first segment. + +Added unit test which focuses only on BPF_LD | BPF_ABS/BPF_IND and +a BPF_LD | BPF_ABS slow path call. + +Signed-off-by: Christophe Fontaine +--- + app/test/test_bpf.c | 121 ++++++++++++++++++++++++++++++++++++++++ + lib/bpf/bpf_jit_arm64.c | 86 ++++++++++++++++++++++++++++ + 2 files changed, 207 insertions(+) + +diff --git a/app/test/test_bpf.c b/app/test/test_bpf.c +index dd24722450..799f9d30ca 100644 +--- a/app/test/test_bpf.c ++++ b/app/test/test_bpf.c +@@ -2750,6 +2750,79 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { + }, + }; + ++/* load mbuf (BPF_ABS/BPF_IND) test-cases */ ++static const struct ebpf_insn test_ld_mbuf0_prog[] = { ++ /* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ ++ { ++ .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), ++ .dst_reg = EBPF_REG_6, ++ .src_reg = EBPF_REG_1, ++ }, ++ /* load IPv4 version and IHL */ ++ { ++ .code = (BPF_LD | BPF_ABS | BPF_B), ++ .imm = offsetof(struct rte_ipv4_hdr, version_ihl), ++ }, ++ { ++ .code = (BPF_JMP | EBPF_EXIT), ++ }, ++}; ++ ++/* load mbuf (BPF_ABS/BPF_IND) test-cases */ ++static const struct ebpf_insn test_ld_slow_mbuf0_prog[] = { ++ /* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ ++ { ++ .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), ++ .dst_reg = EBPF_REG_6, ++ .src_reg = EBPF_REG_1, ++ }, ++ /* load from chained mbuf */ ++ { ++ .code = (BPF_LD | BPF_ABS | BPF_B), ++ /* 201: second mbuf, built by test_ld_mbuf1_prepare */ ++ .imm = 201 + 0x42, ++ }, ++ { ++ .code = (BPF_JMP | EBPF_EXIT), ++ }, ++}; ++ ++static const struct ebpf_insn test_ld_ind_mbuf0_prog[] = { ++ /* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ ++ { ++ .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), ++ .dst_reg = EBPF_REG_6, ++ .src_reg = EBPF_REG_1, ++ }, ++ { ++ /* Set return value to one. */ ++ .code = (EBPF_ALU64 | EBPF_MOV | BPF_K), ++ .dst_reg = EBPF_REG_0, ++ .imm = 0, ++ }, ++ /* load IPv4 version and IHL */ ++ { ++ .code = (BPF_LD | BPF_IND | BPF_B), ++ .src_reg = EBPF_REG_0, ++ .imm = offsetof(struct rte_ipv4_hdr, version_ihl), ++ }, ++ { ++ .code = (BPF_JMP | EBPF_EXIT), ++ }, ++}; ++ ++static int ++test_ld_mbuf0_check(uint64_t rc, const void *arg) ++{ ++ return cmp_res(__func__, 0x45, rc, arg, arg, 0); ++} ++ ++static int ++test_ld_slow_mbuf0_check(uint64_t rc, const void *arg) ++{ ++ return cmp_res(__func__, 0x42, rc, arg, arg, 0); ++} ++ + /* load mbuf (BPF_ABS/BPF_IND) test-cases */ + static const struct ebpf_insn test_ld_mbuf1_prog[] = { + +@@ -3417,6 +3490,54 @@ static const struct bpf_test tests[] = { + /* for now don't support function calls on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, ++ { ++ .name = "test_ld_abs_mbuf0", ++ .arg_sz = sizeof(struct dummy_mbuf), ++ .prm = { ++ .ins = test_ld_mbuf0_prog, ++ .nb_ins = RTE_DIM(test_ld_mbuf0_prog), ++ .prog_arg = { ++ .type = RTE_BPF_ARG_PTR_MBUF, ++ .buf_size = sizeof(struct dummy_mbuf), ++ }, ++ }, ++ .prepare = test_ld_mbuf1_prepare, ++ .check_result = test_ld_mbuf0_check, ++ /* mbuf as input argument is not supported on 32 bit platform */ ++ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), ++ }, ++ { ++ .name = "test_ld_slow_mbuf0", ++ .arg_sz = sizeof(struct dummy_mbuf), ++ .prm = { ++ .ins = test_ld_slow_mbuf0_prog, ++ .nb_ins = RTE_DIM(test_ld_slow_mbuf0_prog), ++ .prog_arg = { ++ .type = RTE_BPF_ARG_PTR_MBUF, ++ .buf_size = sizeof(struct dummy_mbuf), ++ }, ++ }, ++ .prepare = test_ld_mbuf1_prepare, ++ .check_result = test_ld_slow_mbuf0_check, ++ /* mbuf as input argument is not supported on 32 bit platform */ ++ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), ++ }, ++ { ++ .name = "test_ld_ind_mbuf0", ++ .arg_sz = sizeof(struct dummy_mbuf), ++ .prm = { ++ .ins = test_ld_ind_mbuf0_prog, ++ .nb_ins = RTE_DIM(test_ld_ind_mbuf0_prog), ++ .prog_arg = { ++ .type = RTE_BPF_ARG_PTR_MBUF, ++ .buf_size = sizeof(struct dummy_mbuf), ++ }, ++ }, ++ .prepare = test_ld_mbuf1_prepare, ++ .check_result = test_ld_mbuf0_check, ++ /* mbuf as input argument is not supported on 32 bit platform */ ++ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), ++ }, + { + .name = "test_ld_mbuf1", + .arg_sz = sizeof(struct dummy_mbuf), +diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c +index 099822e9f1..5b3b80cb86 100644 +--- a/lib/bpf/bpf_jit_arm64.c ++++ b/lib/bpf/bpf_jit_arm64.c +@@ -8,6 +8,7 @@ + + #include + #include ++#include + + #include "bpf_impl.h" + +@@ -1123,6 +1124,75 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off) + emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off)); + } + ++/* ++ * Emit code for BPF_LD | BPF_ABS/IND: load from packet. ++ * Implements both a fast path, which computes the offset and read directly ++ * and a slow path, which calls __rte_pktmbuf_read(mbuf, off, len, buf) ++ * when the data is not in the first segment. ++ */ ++static void ++emit_ld_mbuf(struct a64_jit_ctx *ctx, uint32_t op, uint8_t tmp1, uint8_t tmp2, ++ uint8_t src, int32_t imm) ++{ ++ uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); ++ uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); ++ uint32_t mode = BPF_MODE(op); ++ uint32_t opsz = BPF_SIZE(op); ++ uint32_t sz = bpf_size(opsz); ++ ++ /* r0 = mbuf (R6) */ ++ emit_mov_64(ctx, A64_R(0), r6); ++ ++ /* r1 = off: for ABS use imm, for IND use src + imm */ ++ if (mode == BPF_ABS) { ++ emit_mov_imm(ctx, 0, A64_R(1), imm); ++ } else { ++ emit_mov_imm(ctx, 0, tmp2, imm); ++ emit_add(ctx, 0, tmp2, src); ++ emit_mov_64(ctx, A64_R(1), tmp2); ++ } ++ ++ /* r2 = len, 1/2/4 bytes */ ++ emit_mov_imm32(ctx, 0, A64_R(2), sz); ++ /* r3 = buf (SP) */ ++ emit_mov_64(ctx, A64_R(3), A64_SP); ++ ++ /* tmp1 = mbuf->data_len */ ++ emit_mov_imm(ctx, 1, tmp1, offsetof(struct rte_mbuf, data_len)); ++ emit_ldr(ctx, BPF_W, tmp1, r6, tmp1); ++ ++ /* tmp2 = off + sz */ ++ emit_add_imm_64(ctx, tmp2, A64_R(1), sz); ++ /* if off+sz > data_len, jump to slow path */ ++ emit_cmp(ctx, 1, tmp2, tmp1); ++ emit_b_cond(ctx, A64_HI, 8); ++ ++ /* Fast path, read directly, pointer to the data will be in A64_R(0) */ ++ /* A64_R(0) = mbuf->buf_addr */ ++ emit_mov_imm(ctx, 1, tmp1, offsetof(struct rte_mbuf, buf_addr)); ++ emit_ldr(ctx, EBPF_DW, A64_R(0), r6, tmp1); ++ /* tmp2 = * mbuf->data_off */ ++ emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_off)); ++ emit_ldr(ctx, BPF_H, tmp2, r6, tmp2); ++ ++ /* A64_R(0) += data_off + off */ ++ emit_add(ctx, 1, A64_R(0), tmp2); ++ emit_add(ctx, 1, A64_R(0), A64_R(1)); ++ ++ /* End of Fast Path, skip slow path */ ++ emit_b(ctx, 4); ++ ++ /* slow path, call __rte_pktmbuf_read */ ++ emit_call(ctx, tmp1, __rte_pktmbuf_read); ++ /* check return value of __rte_pktmbuf_read */ ++ emit_return_zero_if_src_zero(ctx, 1, A64_R(0)); ++ ++ /* A64_R(0) points to the data, load 1/2/4 bytes into r0*/ ++ emit_ldr(ctx, opsz, r0, A64_R(0), A64_ZR); ++ if (sz != sizeof(uint8_t)) ++ emit_be(ctx, r0, sz * CHAR_BIT); ++} ++ + static void + check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) + { +@@ -1137,6 +1207,13 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) + switch (op) { + /* Call imm */ + case (BPF_JMP | EBPF_CALL): ++ /* BPF_LD | BPF_ABS/IND use __rte_pktmbuf_read */ ++ case (BPF_LD | BPF_ABS | BPF_B): ++ case (BPF_LD | BPF_ABS | BPF_H): ++ case (BPF_LD | BPF_ABS | BPF_W): ++ case (BPF_LD | BPF_IND | BPF_B): ++ case (BPF_LD | BPF_IND | BPF_H): ++ case (BPF_LD | BPF_IND | BPF_W): + ctx->foundcall = 1; + return; + } +@@ -1338,6 +1415,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) + emit_mov_imm(ctx, 1, dst, u64); + i++; + break; ++ /* load absolute/indirect from packet */ ++ case (BPF_LD | BPF_ABS | BPF_B): ++ case (BPF_LD | BPF_ABS | BPF_H): ++ case (BPF_LD | BPF_ABS | BPF_W): ++ case (BPF_LD | BPF_IND | BPF_B): ++ case (BPF_LD | BPF_IND | BPF_H): ++ case (BPF_LD | BPF_IND | BPF_W): ++ emit_ld_mbuf(ctx, op, tmp1, tmp2, src, imm); ++ break; + /* *(size *)(dst + off) = src */ + case (BPF_STX | BPF_MEM | BPF_B): + case (BPF_STX | BPF_MEM | BPF_H): +-- +2.53.0 +