Feat kind support map pinning v3 (#67)

* bpf: map pinning support

* cleanup: comments in bpfWrapper.c

* dpcnisyncer: basic signalling from CNI to DP

* dpcnisyncer: delete BPFFS signalled from CNI

* pool-manager: dp-cni-syncer cleanup

* SyncServer: one server for all BPF map managers

* pool-manager: remove map cleanup

* dpcnisyncer: add clean up

* Makefile: Add missing copyright tag

* fixup: unit tests

* fixup: static analysis

* linter: add directive to ignore generated code

* fixup: fuzzing test

* Add missing protobuf dependency

* syncer: explicit configuration

* plugin: updates after rebase

* bpfwrapper: use libxdp for redirect program

* cleanup: dead/commented code

* linter error: run clang format
---------

Signed-off-by: Maryam Tahhan <[email protected]>

Author: maryamtahhan

Makefile

@@ -27,6 +27,7 @@ clangformat:
 	@echo
 	-clang-format -i -style=file internal/bpf/*.c internal/bpf/*.h
 	-clang-format -i -style=file internal/bpf/xdp-pass/*.c
+	-clang-format -i -style=file internal/bpf/xdp-afxdp-redirect/*.c
 	@echo
 	@echo
 
@@ -37,18 +38,22 @@ format: clangformat
 	@echo
 	@echo
 
+buildxdp:
+	@echo "******     Build xdp_pass     ******"
+	make -C ./internal/bpf/xdp-pass/
+	@echo "******     Build xdp_afxdp_redirect     ******"
+	make -C ./internal/bpf/xdp-afxdp-redirect/
+	@echo
+
 buildc:
 	@echo "******     Build BPF     ******"
 	@echo
 	gcc ./internal/bpf/bpfWrapper.c -lxdp -c -o ./internal/bpf/bpfWrapper.o
 	ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o  &> /dev/null
-	@echo "******     Build xdp_pass     ******"
-	make -C ./internal/bpf/xdp-pass/
-	@echo
 	@echo
 	@echo
 
-builddp: buildc
+builddp: buildc buildxdp
 	@echo "******     Build DP      ******"
 	@echo
 	go build -o ./bin/afxdp-dp ./cmd/deviceplugin

README.md

@@ -407,6 +407,10 @@ Note: User 0 does not imply that the pod needs to be privileged.
 
 UdsServerDisable is a Boolean configuration. If set to true, devices in this pool will not have the BPF app loaded onto the netdev. This means no UDS server is spun up when a device is allocated to a pod. By default, this is set to false.
 
+#### BpfMapPinningEnable
+
+BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration.
+
 #### UdsTimeout
 
 UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds.

cmd/deviceplugin/main.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/intel/afxdp-plugins-for-kubernetes/constants"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/deviceplugin"
+	"github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/host"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/logformats"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/networking"
@@ -97,9 +98,16 @@ func main() {
 	}
 	logging.Infof("Host meets requirements")
 
+	//START THE SYNCER SERVER TODO CHECK BPF MAP
+	dpCniSyncerServer, err := dpcnisyncerserver.NewSyncerServer()
+	if err != nil {
+		logging.Errorf("Error creating the DpCniSyncerServer")
+	}
+	logging.Debugf("DP<=>CNI grpc Syncer started")
+
 	// pool configs
 	logging.Infof("Getting device pools")
-	poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler)
+	poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler, dpCniSyncerServer)
 	if err != nil {
 		logging.Warningf("Error getting device pools: %v", err)
 		exit(constants.Plugins.DevicePlugin.ExitPoolError)

constants/constants.go

@@ -82,6 +82,10 @@ var (
 	udsSockDir    = "/tmp/afxdp_dp/"  // host location where we place our uds sockets. If changing location remember to update daemonset mount point
 	udsPodPath    = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
 
+	/* BPF*/
+	bpfMapPodPath = "/tmp/xsks_map"
+	xsk_map       = "/xsks_map"
+
 	udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
 
 	/* Handshake*/
@@ -129,6 +133,8 @@ var (
 	DeviceFile deviceFile
 	/* DeviceFile contains constants related to the devicefile */
 	EthtoolFilter ethtoolFilter
+	/* Bpf contains constants related to the BPF Map pinning */
+	Bpf bpf
 )
 
 type cni struct {
@@ -213,6 +219,11 @@ type uds struct {
 	Handshake   handshake
 }
 
+type bpf struct {
+	BpfMapPodPath string
+	Xsk_map       string
+}
+
 type handshake struct {
 	Version             string
 	RequestVersion      string
@@ -334,6 +345,11 @@ func init() {
 		},
 	}
 
+	Bpf = bpf{
+		BpfMapPodPath: bpfMapPodPath,
+		Xsk_map:       xsk_map,
+	}
+
 	EthtoolFilter = ethtoolFilter{
 		EthtoolFilterRegex: ethtoolFilterRegex,
 	}

deployments/daemonset-kind.yaml

@@ -13,7 +13,8 @@ data:
           {
             "name":"myPool",
             "mode":"primary",
-            "UdsServerDisable": false,
+            "UdsServerDisable": true,
+            "BpfMapPinningEnable": true,
             "drivers":[
               {
                 "name":"veth"
@@ -72,6 +73,8 @@ spec:
           volumeMounts:
             - name: unixsock
               mountPath: /tmp/afxdp_dp/
+            - name: bpfmappinning
+              mountPath: /var/run/afxdp_dp/
               mountPropagation: Bidirectional
             - name: devicesock
               mountPath: /var/lib/kubelet/device-plugins/
@@ -87,6 +90,9 @@ spec:
         - name: unixsock
           hostPath:
             path: /tmp/afxdp_dp/
+        - name: bpfmappinning
+          hostPath:
+            path: /var/run/afxdp_dp/
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/device-plugins/

examples/cndp-0-0.yaml

@@ -0,0 +1,26 @@
+# A working example of BPF MAP PINNING
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cndp-0-0
+  annotations:
+    k8s.v1.cni.cncf.io/networks: afxdp-network
+spec:
+  containers:
+    - name: cndp-0
+      command: ["/bin/bash"]
+      args: ["-c", "./jsonc_gen.sh -kp ; cndpfwd -c config.jsonc lb;"]
+      image:  quay.io/mtahhan/cndp-map-pinning:latest
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+          #capabilities:
+          #add:
+          #  - NET_RAW
+          #  - IPC_LOCK
+          #  - BPF
+      resources:
+        requests:
+          afxdp/myPool: '1'
+        limits:
+          afxdp/myPool: '1'

examples/nad_with_syncer.yaml

@@ -0,0 +1,27 @@
+# WARNING: This is an example definition only. Remove all comments before use.
+
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: afxdp-network                                        # Name of this network, pods will request this network by name
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: afxdp/myPool            # Needs to match the device plugin pool name / resource type
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "afxdp",                                       # CNI binary, leave as afxdp
+      "mode": "primary",                                     # CNI mode setting (required)
+      "logFile": "afxdp-cni.log",                            # CNI log file location (optional)
+      "logLevel": "debug",                                   # CNI logging level (optional)
+      "dpSyncer": true,                                      # Sync with Device Plugin over gRPC MUST BE SET for bpf map pinning
+      "ipam": {                                              # CNI IPAM plugin and associated config (optional)
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.220",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'

go.mod

@@ -6,17 +6,20 @@ require (
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.1.1
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
+	github.com/golang/protobuf v1.5.3
 	github.com/google/gofuzz v1.1.0
 	github.com/google/uuid v1.3.0
 	github.com/intel/afxdp-plugins-for-kubernetes/pkg/goclient v0.0.0
 	github.com/intel/afxdp-plugins-for-kubernetes/pkg/subfunctions v0.0.0
+	github.com/moby/sys/mount v0.3.3
 	github.com/pkg/errors v0.9.1
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
 	golang.org/x/net v0.17.0
 	google.golang.org/grpc v1.53.0
+	google.golang.org/protobuf v1.30.0 // indirect
 	gotest.tools v2.2.0+incompatible
 	k8s.io/apimachinery v0.25.2
 	k8s.io/kubelet v0.25.2

go.sum

@@ -735,8 +735,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -894,8 +895,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
+github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
@@ -1774,8 +1779,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

images/amd64.dockerfile

@@ -17,8 +17,8 @@ COPY . /usr/src/afxdp_k8s_plugins
 WORKDIR /usr/src/afxdp_k8s_plugins
 RUN apt-get update \
 && apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 \
-&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6  \
-&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.7~deb12u1 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.7~deb12u1 \
 && apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 \
 && make buildcni
 
@@ -39,4 +39,5 @@ COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-pass/xdp_pass.o /afxdp/xdp_pass.o
+COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.o /afxdp/xdp_afxdp_redirect.o
 ENTRYPOINT ["/afxdp/entrypoint.sh"]