tcpdump / libpcap - packet traces - pcap-grout.so - WIP

.clang-format

@@ -94,7 +94,7 @@ IncludeCategories:
     Priority:        -1
   - Regex:           '^<gr_'
     Priority:        100
-  - Regex:           '^<(rte_|event|ecoli|numa.h|grout.h)'
+  - Regex:           '^<(rte_|event|ecoli|pcap|numa.h|grout.h)'
     Priority:        500
   - Regex:           '^<cmocka'
     Priority:        100000

.github/workflows/check.yml

@@ -77,7 +77,7 @@ jobs:
           # grout dependencies
           sudo apt-get update -qy
           sudo apt-get install -qy --no-install-recommends \
-            make gcc gdb ccache ninja-build meson git scdoc \
+            make gcc gdb ccache ninja-build meson git scdoc bison flex \
             libibverbs-dev libasan8 libcmocka-dev libedit-dev libarchive-dev \
             libevent-dev libmnl-dev libnuma-dev python3-pyelftools \
             socat tcpdump traceroute graphviz iproute2 iputils-ping ndisc6 jq \

.gitignore

@@ -21,5 +21,6 @@
 /subprojects/dpdk-*
 /subprojects/libecoli-*
 /subprojects/frr-*
+/subprojects/libpcap
 /subprojects/packagecache
 /subprojects/.wraplock

api/gr_api.h

@@ -58,6 +58,14 @@ gr_api_client_send(struct gr_api_client *, uint32_t req_type, size_t tx_len, con
 // Returns -EMSGSIZE if payload is non-empty but smaller than min_resp_size.
 int gr_api_client_recv(struct gr_api_client *, uint32_t req_type, uint32_t for_id, void **rx_data);
 
+int gr_api_client_recv_fd(
+	struct gr_api_client *,
+	uint32_t req_type,
+	uint32_t for_id,
+	void **rx_data,
+	int *fd
+);
+
 // Send a request and receive the response.
 // Validates response payload size against GR_REQ-declared type.
 // Caller must free(*rx_data) after use.
@@ -78,6 +86,22 @@ static inline int gr_api_client_send_recv(
 // internal, called when interrupting gr_api_client_stream_foreach()
 int __gr_api_client_stream_drain(struct gr_api_client *, uint32_t req_type, uint32_t for_id);
 
+// Send a request and receive the response with an optional file descriptor.
+// If fd is non-NULL and the server sends an fd via SCM_RIGHTS, it is stored in *fd.
+static inline int gr_api_client_send_recv_fd(
+	struct gr_api_client *client,
+	uint32_t req_type,
+	size_t tx_len,
+	const void *tx_data,
+	void **rx_data,
+	int *fd
+) {
+	long int ret = gr_api_client_send(client, req_type, tx_len, tx_data);
+	if (ret < 0)
+		return ret;
+	return gr_api_client_recv_fd(client, req_type, ret, rx_data, fd);
+}
+
 // Send a request and iterate over the received stream of responses.
 //
 // @param obj Iterator variable (const pointer to response object type).

api/gr_api_client_impl.h

@@ -101,6 +101,7 @@ const char *gr_api_message_name(uint32_t type) {
 struct response {
 	struct gr_api_response header;
 	void *payload;
+	int fd; // received via SCM_RIGHTS, -1 if none
 	STAILQ_ENTRY(response) next;
 };
 
@@ -153,6 +154,8 @@ int gr_api_client_disconnect(struct gr_api_client *client) {
 	while (!STAILQ_EMPTY(&client->responses)) {
 		struct response *resp = STAILQ_FIRST(&client->responses);
 		STAILQ_REMOVE_HEAD(&client->responses, next);
+		if (resp->fd >= 0)
+			close(resp->fd);
 		free(resp->payload);
 		free(resp);
 	}
@@ -224,16 +227,61 @@ long int gr_api_client_send(
 	return req.id;
 }
 
-int gr_api_client_recv(
+// Receive a response header, potentially with an SCM_RIGHTS fd.
+// Uses recvmsg() so ancillary data is captured.
+static int
+recv_response_header(const struct gr_api_client *c, struct gr_api_response *resp, int *recv_fd) {
+	*recv_fd = -1;
+
+	union {
+		char buf[CMSG_SPACE(sizeof(int))];
+		struct cmsghdr align;
+	} cmsg_buf;
+	memset(&cmsg_buf, 0, sizeof(cmsg_buf));
+
+	struct iovec iov = {.iov_base = resp, .iov_len = sizeof(*resp)};
+	struct msghdr msg = {
+		.msg_iov = &iov,
+		.msg_iovlen = 1,
+		.msg_control = cmsg_buf.buf,
+		.msg_controllen = sizeof(cmsg_buf.buf),
+	};
+
+	ssize_t n = recvmsg(c->sock_fd, &msg, MSG_CMSG_CLOEXEC);
+
+	if (n == 0) {
+		errno = ECONNRESET;
+		return -1;
+	}
+	if (n < 0)
+		return -1;
+	if ((size_t)n < sizeof(*resp)) {
+		errno = EPROTO;
+		return -1;
+	}
+
+	struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+	if (cmsg != NULL && cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
+		memcpy(recv_fd, CMSG_DATA(cmsg), sizeof(int));
+
+	return 0;
+}
+
+int gr_api_client_recv_fd(
 	struct gr_api_client *client,
 	uint32_t req_type,
 	uint32_t for_id,
-	void **rx_data
+	void **rx_data,
+	int *fd
 ) {
 	struct response *cached = NULL;
 	const struct api_message *m;
 	struct gr_api_response resp;
 	void *payload = NULL;
+	int recv_fd = -1;
+
+	if (fd != NULL)
+		*fd = -1;
 
 	if (client == NULL)
 		return errno_set(EINVAL);
@@ -249,12 +297,13 @@ int gr_api_client_recv(
 		STAILQ_REMOVE(&client->responses, cached, response, next);
 		resp = cached->header;
 		payload = cached->payload;
+		recv_fd = cached->fd;
 		free(cached);
 		goto out;
 	}
 recv:
 	// No matching cached message, try to receive one from the socket.
-	if (recv_all(client, &resp, sizeof(resp)) != sizeof(resp))
+	if (recv_response_header(client, &resp, &recv_fd) < 0)
 		goto err;
 
 	if (resp.payload_len > GR_API_MAX_MSG_LEN) {
@@ -275,8 +324,10 @@ int gr_api_client_recv(
 			goto err;
 		cached->header = resp;
 		cached->payload = payload;
+		cached->fd = recv_fd;
 		STAILQ_INSERT_TAIL(&client->responses, cached, next);
 		payload = NULL;
+		recv_fd = -1;
 		// And try to receive the next message until we get the correct ID.
 		goto recv;
 	}
@@ -299,13 +350,28 @@ int gr_api_client_recv(
 		assert(rx_data != NULL);
 		*rx_data = payload;
 	}
+	if (fd != NULL)
+		*fd = recv_fd;
+	else if (recv_fd >= 0)
+		close(recv_fd);
 
 	return 0;
 err:
+	if (recv_fd >= 0)
+		close(recv_fd);
 	free(payload);
 	return -errno;
 }
 
+int gr_api_client_recv(
+	struct gr_api_client *client,
+	uint32_t req_type,
+	uint32_t for_id,
+	void **rx_data
+) {
+	return gr_api_client_recv_fd(client, req_type, for_id, rx_data, NULL);
+}
+
 int gr_api_client_event_recv(const struct gr_api_client *c, struct gr_api_event **event) {
 	const struct api_message *m;
 	struct gr_api_event header;

docs/meson.build

@@ -74,7 +74,7 @@ custom_target(
 # Individual command man pages
 # The list is hardcoded since we can't run grcli during meson configuration.
 grcli_commands = [
-  'address', 'affinity', 'conntrack', 'dnat44', 'events', 'fdb', 'flood',
+  'address', 'affinity', 'capture', 'conntrack', 'dnat44', 'events', 'fdb', 'flood',
   'graph', 'interface', 'log', 'nexthop', 'ping', 'route', 'router-advert',
   'snat44', 'stats', 'trace', 'traceroute', 'tunsrc',
 ]

main/api.c

@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <sys/uio.h>
 #include <unistd.h>
 
 LOG_TYPE("api");
@@ -249,6 +250,50 @@ void api_send(struct api_ctx *ctx, uint32_t len, const void *payload) {
 		LOG(ERR, "pid=%d cannot write payload", ctx->pid);
 }
 
+// Send a response header + optional payload together with a file
+// descriptor via sendmsg(SCM_RIGHTS). Closes the fd after sending.
+static void
+send_response_with_fd(struct bufferevent *bev, struct gr_api_response *resp, struct api_out *out) {
+	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
+
+	struct iovec iov[2];
+	int iovlen = 1;
+	iov[0].iov_base = resp;
+	iov[0].iov_len = sizeof(*resp);
+	if (out->len > 0 && out->payload != NULL) {
+		iov[1].iov_base = out->payload;
+		iov[1].iov_len = out->len;
+		iovlen = 2;
+	}
+
+	union {
+		char buf[CMSG_SPACE(sizeof(int))];
+		struct cmsghdr align;
+	} cmsg_buf;
+	memset(&cmsg_buf, 0, sizeof(cmsg_buf));
+
+	struct msghdr msg = {
+		.msg_iov = iov,
+		.msg_iovlen = iovlen,
+		.msg_control = cmsg_buf.buf,
+		.msg_controllen = sizeof(cmsg_buf.buf),
+	};
+	struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type = SCM_RIGHTS;
+	cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+	memcpy(CMSG_DATA(cmsg), &out->fd, sizeof(int));
+
+	ssize_t ret;
+	do {
+		ret = sendmsg(bufferevent_getfd(bev), &msg, MSG_NOSIGNAL);
+	} while (ret < 0 && errno == EINTR);
+	if (ret < 0)
+		LOG(ERR, "sendmsg with fd: %s", strerror(errno));
+	close(out->fd);
+	out->fd = -1;
+}
+
 static void read_cb(struct bufferevent *bev, void *priv) {
 	struct evbuffer *input = bufferevent_get_input(bev);
 	struct api_ctx *ctx = priv;
@@ -294,7 +339,7 @@ static void read_cb(struct bufferevent *bev, void *priv) {
 	// Reset state for next request
 	ctx->header_complete = false;
 
-	struct api_out out;
+	struct api_out out = {.fd = -1};
 
 	// We have a complete request, process it
 	const struct api_handler *handler = lookup_api_handler(ctx->header.type);
@@ -333,16 +378,19 @@ static void read_cb(struct bufferevent *bev, void *priv) {
 		.payload_len = out.len,
 	};
 
-	if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
-		LOG(ERR, "failed to write header");
-	if (out.len > 0) {
-		assert(out.payload != NULL);
-		if (bufferevent_write(bev, out.payload, out.len) < 0)
-			LOG(ERR, "failed to write payload");
+	if (out.fd >= 0) {
+		send_response_with_fd(bev, &resp, &out);
+	} else {
+		if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
+			LOG(ERR, "failed to write header");
+		if (out.len > 0) {
+			assert(out.payload != NULL);
+			if (bufferevent_write(bev, out.payload, out.len) < 0)
+				LOG(ERR, "failed to write payload");
+		}
+		bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
 	}
 
-	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
-
 	free(req_payload);
 	free(out.payload);
 

main/module.h

@@ -15,10 +15,16 @@ struct api_out {
 	uint32_t status;
 	uint32_t len;
 	void *payload;
+	int fd; // file descriptor to pass via SCM_RIGHTS, -1 = none
 };
 
 static inline struct api_out api_out(uint32_t status, uint32_t len, void *payload) {
-	struct api_out out = {status, len, payload};
+	struct api_out out = {status, len, payload, -1};
+	return out;
+}
+
+static inline struct api_out api_out_fd(uint32_t status, uint32_t len, void *payload, int fd) {
+	struct api_out out = {status, len, payload, fd};
 	return out;
 }
 

meson.build

@@ -64,6 +64,11 @@ if not compiler.compiles('''
   error(compiler.get_id(), compiler.version(), 'does not support C23 __VA_OPT__ macro')
 endif
 
+pcap_dep = dependency(
+  'libpcap',
+  version: '>= 1.11.0',
+  fallback: ['libpcap', 'pcap_dep'],
+)
 dpdk_dep = dependency(
   'libdpdk',
   version : '>= 25.11',
@@ -73,15 +78,14 @@ dpdk_dep = dependency(
     'werror=false',
     'tests=false',
     'enable_drivers=net/virtio,net/vhost,net/i40e,net/ice,net/iavf,net/ixgbe,net/null,net/tap,common/mlx5,net/mlx5,bus/auxiliary,net/vmxnet3',
-    'enable_libs=graph,hash,fib,rib,pcapng,gso,vhost,cryptodev,dmadev,security',
+    'enable_libs=graph,hash,fib,rib,pcapng,bpf,gso,vhost,cryptodev,dmadev,security',
     'disable_apps=*',
     'enable_docs=false',
     'developer_mode=disabled',
     'cpu_instruction_set=' + get_option('dpdk_cpu'),
   ],
   static: true,
 )
-
 ev_core_dep = dependency('libevent_core')
 ev_extra_dep = dependency('libevent_extra')
 ev_thread_dep = dependency('libevent_pthreads')
@@ -129,6 +133,7 @@ subdir('modules')
 subdir('cli')
 subdir('api')
 subdir('frr')
+subdir('pcap')
 
 fs = import('fs')
 abidiff = find_program('abidiff', native: true, required: false)
@@ -153,15 +158,15 @@ endif
 grout_exe = executable(
   'grout', src,
   include_directories: inc + api_inc,
-  dependencies: [dpdk_dep, ev_core_dep, ev_extra_dep, ev_thread_dep, mnl_dep, numa_dep],
+  dependencies: [dpdk_dep, ev_core_dep, ev_extra_dep, ev_thread_dep, mnl_dep, numa_dep, pcap_dep],
   c_args: ['-D__GROUT_MAIN__'] + grout_cflags,
   install: true,
 )
 
 grcli_exe = executable(
   'grcli', cli_src + grout_header,
   include_directories: cli_inc + api_inc,
-  dependencies: [ecoli_dep],
+  dependencies: [ecoli_dep, pcap_dep],
   c_args: cli_cflags + grout_cflags,
   install: true,
 )

meson_options.txt

@@ -30,3 +30,8 @@ option(
   'dpdk_cpu', type: 'string', value: 'auto',
   description: 'Value of DPDK cpu_instruction_set when compiled as a subproject.',
 )
+
+option(
+  'pcap', type: 'feature', value: 'auto',
+  description: 'Build pcap-grout.so libpcap plugin. If set to "auto", only build if libpcap-dev is found.',
+)