netstacklat: Make HOL-blocking filtering configurable

To make netstacklat only report latency based on the local network and
application stack, it by default excludes TCP reads that may be
head-of-line (HOL) blocked by a missing segment.

However, in some scenarios, it may be desirable to still include these
HOL-blocked measurements. For example, in a data center where the full
network is under your control, high latency caused by drops in the
data center network could still be an issue you wish netstacklat to
detect. Therefore, add the -y/--include-tcp-hol-delay option, which
disables the mechanism used to filter out potentially HOL-blocked
reads.

An alternative design to simply disabling the filtering mechanism,
would be to separate the potentially HOL-blocked measurements into
their own histogram (e.g. "tcp-socket-read-hol-blocked"). However,
that would further increase the data cardinality (which may be a
concern when using it together ebpf-exporter). Furthermore, by simply
disabling the filtering instead we can reduce the overhead a bit and
also allow it as an option on systems were the tcp_data_queue_ofo()
function cannot be probed.

Signed-off-by: Simon Sundberg <[email protected]>

Author: simosund

netstacklat/netstacklat.bpf.c

@@ -27,6 +27,7 @@ volatile const struct netstacklat_bpf_config user_config = {
 	.filter_cgroup = false,
 	.groupby_ifindex = false,
 	.groupby_cgroup = false,
+	.include_hol_blocked = false,
 };
 
 /*
@@ -484,7 +485,7 @@ int BPF_PROG(netstacklat_tcp_recv_timestamp, void *msg, struct sock *sk,
 	struct timespec64 *ts = &tss->ts[0];
 
 	/* skip if preceeding sock read ended in ooo-range */
-	if (tcp_read_in_ooo_range(sk))
+	if (!user_config.include_hol_blocked && tcp_read_in_ooo_range(sk))
 		return 0;
 
 	record_socket_latency(sk, NULL,
@@ -507,6 +508,15 @@ SEC("fentry/tcp_data_queue_ofo")
 int BPF_PROG(netstacklat_tcp_data_queue_ofo, struct sock *sk,
 	     struct sk_buff *skb)
 {
+	if (!user_config.include_hol_blocked)
+		/*
+		 * It's better to not load this program at all if the ooo-range
+		 * tracking isn't needed (like done by netstacklat.c).
+		 * But if an external loader (like ebpf-exporter) is used,
+		 * this should at least minimze the unncecessary overhead.
+		 */
+		tcp_update_ooo_range(sk, skb);
+
 	if (!filter_network(skb, sk))
 		return 0;
 

netstacklat/netstacklat.c

@@ -83,18 +83,19 @@ struct netstacklat_config {
 };
 
 static const struct option long_options[] = {
-	{ "help",              no_argument,       NULL, 'h' },
-	{ "report-interval",   required_argument, NULL, 'r' },
-	{ "list-probes",       no_argument,       NULL, 'l' },
-	{ "enable-probes",     required_argument, NULL, 'e' },
-	{ "disable-probes",    required_argument, NULL, 'd' },
-	{ "pids",              required_argument, NULL, 'p' },
-	{ "interfaces",        required_argument, NULL, 'i' },
-	{ "network-namespace", required_argument, NULL, 'n' },
-	{ "cgroups",           required_argument, NULL, 'c' },
-	{ "min-queuelength",   required_argument, NULL, 'q' },
-	{ "groupby-interface", no_argument,       NULL, 'I' },
-	{ "groupby-cgroup",    no_argument,       NULL, 'C' },
+	{ "help",                  no_argument,       NULL, 'h' },
+	{ "report-interval",       required_argument, NULL, 'r' },
+	{ "list-probes",           no_argument,       NULL, 'l' },
+	{ "enable-probes",         required_argument, NULL, 'e' },
+	{ "disable-probes",        required_argument, NULL, 'd' },
+	{ "pids",                  required_argument, NULL, 'p' },
+	{ "interfaces",            required_argument, NULL, 'i' },
+	{ "network-namespace",     required_argument, NULL, 'n' },
+	{ "cgroups",               required_argument, NULL, 'c' },
+	{ "min-queuelength",       required_argument, NULL, 'q' },
+	{ "groupby-interface",     no_argument,       NULL, 'I' },
+	{ "groupby-cgroup",        no_argument,       NULL, 'C' },
+	{ "include-tcp-hol-delay", no_argument,       NULL, 'y' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -565,6 +566,7 @@ static int parse_arguments(int argc, char *argv[],
 	conf->bpf_conf.filter_cgroup = false;
 	conf->bpf_conf.groupby_ifindex = false;
 	conf->bpf_conf.groupby_cgroup = false;
+	conf->bpf_conf.include_hol_blocked = false;
 
 	for (i = 0; i < NETSTACKLAT_N_HOOKS; i++)
 		// All probes enabled by default
@@ -659,6 +661,9 @@ static int parse_arguments(int argc, char *argv[],
 		case 'C': // groupby-cgroup
 			conf->bpf_conf.groupby_cgroup = true;
 			break;
+		case 'y': // include-tcp-hol-delay
+			conf->bpf_conf.include_hol_blocked = true;
+			break;
 		case 'h': // help
 			print_usage(stdout, argv[0]);
 			exit(EXIT_SUCCESS);
@@ -1113,6 +1118,10 @@ static void set_programs_to_load(const struct netstacklat_config *conf,
 			bpf_program__set_autoload(progs.progs[i],
 						  conf->enabled_hooks[hook]);
 	}
+
+	if (conf->bpf_conf.include_hol_blocked)
+		bpf_program__set_autoload(
+			obj->progs.netstacklat_tcp_data_queue_ofo, false);
 }
 
 static int set_map_sizes(const struct netstacklat_config *conf,

netstacklat/netstacklat.h

@@ -77,6 +77,7 @@ struct netstacklat_bpf_config {
 	bool filter_cgroup;
 	bool groupby_ifindex;
 	bool groupby_cgroup;
+	bool include_hol_blocked;
 };
 
 #endif