Support for card disabling, allowlist, denylist, descheduling.

Signed-off-by: Ukri Niemimuukko <[email protected]>

Author: uniemimu

extender/scheduler.go

@@ -1,4 +1,4 @@
-//Package extender contains types and logic to respond to requests from a Kubernetes http scheduler extender.
+// Package extender contains types and logic to respond to requests from a Kubernetes http scheduler extender.
 package extender
 
 import (
@@ -11,7 +11,7 @@ import (
 	"k8s.io/klog/v2"
 )
 
-//postOnly check if the method type is POST
+// postOnly check if the method type is POST.
 func postOnly(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPost {
@@ -24,7 +24,7 @@ func postOnly(next http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-//contentLength check the if the request size is adequate
+// contentLength check the if the request size is adequate.
 func contentLength(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.ContentLength > 1*1000*1000*1000 {
@@ -37,7 +37,7 @@ func contentLength(next http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-//requestContentType verify the content type of the request
+// requestContentType verify the content type of the request.
 func requestContentType(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		requestContentType := r.Header.Get("Content-Type")
@@ -65,7 +65,7 @@ if the content type is not correct - i.e. NOT application/json - the response wi
 will not run.
 */
 
-//handlerWithMiddleware is handler wrapped with middleware to serve the prechecks at endpoint
+// handlerWithMiddleware is handler wrapped with middleware to serve the prechecks at endpoint.
 func handlerWithMiddleware(handle http.HandlerFunc) http.HandlerFunc {
 	return requestContentType(
 		contentLength(
@@ -74,7 +74,7 @@ func handlerWithMiddleware(handle http.HandlerFunc) http.HandlerFunc {
 	)
 }
 
-//error handler deals with requests sent to an invalid endpoint and returns a 404.
+// error handler deals with requests sent to an invalid endpoint and returns a 404.
 func errorHandler(w http.ResponseWriter, r *http.Request) {
 	klog.V(2).InfoS("Requested resource: '"+r.URL.Path+"' not found", "component", "extender")
 	w.Header().Add("Content-Type", "application/json")
@@ -106,7 +106,7 @@ func (m Server) StartServer(port string, certFile string, keyFile string, caFile
 	klog.V(2).InfoS("Scheduler extender server failed to start "+err.Error(), "component", "extender")
 }
 
-//Configuration values including algorithms etc for the TAS scheduling endpoint.
+// Configuration values including algorithms etc for the TAS scheduling endpoint.
 func configureSecureServer(port string, caFile string) *http.Server {
 	caCert, err := ioutil.ReadFile(caFile)
 	if err != nil {

extender/types.go

@@ -7,19 +7,19 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
-//Scheduler has the capabilities needed to prioritize and filter nodes based on http requests.
+// Scheduler has the capabilities needed to prioritize and filter nodes based on http requests.
 type Scheduler interface {
 	Bind(w http.ResponseWriter, r *http.Request)
 	Prioritize(w http.ResponseWriter, r *http.Request)
 	Filter(w http.ResponseWriter, r *http.Request)
 }
 
-//Server type wraps the implementation of the extender.
+// Server type wraps the implementation of the extender.
 type Server struct {
 	Scheduler
 }
 
-//TODO: These types are in the k8s.io/kubernetes/extender/api package
+// TODO: These types are in the k8s.io/kubernetes/extender/api package
 // Some import issue is making them tough to access, so they are reimplemented here pending a solution.
 
 // HostPriority represents the priority of scheduling to a particular host, higher priority is better.

gpu-aware-scheduling/Makefile

@@ -5,7 +5,7 @@ endif
 .PHONY: test all build gpu-extender images format clean
 
 test:
-		go test ./...  -v *_test.go
+	go test ./...  -v *_test.go
 
 all: format build
 
@@ -16,9 +16,14 @@ image: gpu-extender
 	docker build -f deploy/images/Dockerfile_gpu-extender ../ -t $(IMAGE_PATH)gpu-extender$(IMAGE_TAG)
 
 format:
-		gofmt -w -s .
+	gofmt -w -s .
 
 clean:
-		rm -f ./bin/*
+	rm -f ./bin/*
+
+mock:
+	mockery --name=CacheAPI --dir=pkg/gpuscheduler --inpkg --note="+build !validation\nre-generate with 'make mock'"
+	mockery --name=InternalCacheAPI --dir=pkg/gpuscheduler --inpkg --note="+build !validation\nre-generate with 'make mock'"
+
 e2e:
 

gpu-aware-scheduling/README.md

@@ -128,6 +128,8 @@ name |type | description| usage | default|
 |key| string | location of the key file for the TLS endpoint| --key=/root/key.txt | /etc/kubernetes/pki/ca.key
 |cacert| string | location of the ca certificate for the TLS endpoint| --key=/root/cacert.txt | /etc/kubernetes/pki/ca.crt
 |unsafe| bool | whether or not to listen on a TLS endpoint with the scheduler extender | --unsafe=true| false
+|enableAllowlist| bool | enable POD-annotation based GPU allowlist feature | --enableAllowlist| false
+|enableDenylist| bool | enable POD-annotation based GPU denylist feature | --enableDenylist| false
 
 ## Adding the resource to make a deployment use GAS Scheduler Extender
 

gpu-aware-scheduling/cmd/gas-scheduler-extender/main.go

@@ -11,7 +11,7 @@ import (
 func main() {
 	var (
 		kubeConfig, port, certFile, keyFile, caFile string
-		unsafe                                      bool
+		unsafe, enableAllowlist, enableDenylist     bool
 	)
 
 	flag.StringVar(&kubeConfig, "kubeConfig", "/root/.kube/config", "location of kubernetes config file")
@@ -20,6 +20,8 @@ func main() {
 	flag.StringVar(&keyFile, "key", "/etc/kubernetes/pki/ca.key", "key file extender will use for authentication")
 	flag.StringVar(&caFile, "cacert", "/etc/kubernetes/pki/ca.crt", "ca file extender will use for authentication")
 	flag.BoolVar(&unsafe, "unsafe", false, "unsafe instances of GPU aware scheduler will be served over simple http.")
+	flag.BoolVar(&enableAllowlist, "enableAllowlist", false, "enable allowed GPUs annotation (csv list of names)")
+	flag.BoolVar(&enableDenylist, "enableDenylist", false, "enable denied GPUs annotation (csv list of names)")
 	klog.InitFlags(nil)
 	flag.Parse()
 
@@ -28,7 +30,7 @@ func main() {
 		panic(err)
 	}
 
-	gasscheduler := gpuscheduler.NewGASExtender(kubeClient)
+	gasscheduler := gpuscheduler.NewGASExtender(kubeClient, enableAllowlist, enableDenylist)
 	sch := extender.Server{Scheduler: gasscheduler}
 	sch.StartServer(port, certFile, keyFile, caFile, unsafe)
 	klog.Flush()

gpu-aware-scheduling/docs/usage.md

@@ -65,6 +65,15 @@ Your PODs then, needs to ask for some GPU-resources. Like this:
 
 A complete example pod yaml is located in [docs/example](./example)
 
+## Allowlist and Denylist
+
+You can use POD-annotations in your POD-templates to list the GPU names which you allow, or deny for your deployment. The values for the annotations are comma separated value lists of the form "card0,card1,card2", and the names of the annotations are:
+
+- `gas-allow`
+- `gas-deny`
+
+Note that the feature is disabled by default. You need to enable allowlist and/or denylist via command line flags.
+
 ## Summary in a chronological order
 
 - GPU-plugin initcontainer installs an NFD hook which prints labels for you, based on the Intel GPUs it finds