[release-4.20] OCPBUGS-65680: enforce client side auth requirement for metrics endpoint

cmd/manager/main.go

@@ -2,16 +2,18 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
 	"flag"
 	"fmt"
 	"net/http"
 	"os"
 	"runtime"
 	"time"
 
-	"github.com/operator-framework/operator-marketplace/pkg/certificateauthority"
+	ca "github.com/operator-framework/operator-marketplace/pkg/certificateauthority"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -24,6 +26,7 @@ import (
 	configv1 "github.com/operator-framework/operator-marketplace/pkg/apis/config/v1"
 	apiutils "github.com/operator-framework/operator-marketplace/pkg/apis/operators/shared"
 	"github.com/operator-framework/operator-marketplace/pkg/controller"
+	"github.com/operator-framework/operator-marketplace/pkg/controller/configmap"
 	"github.com/operator-framework/operator-marketplace/pkg/controller/options"
 	"github.com/operator-framework/operator-marketplace/pkg/defaults"
 	"github.com/operator-framework/operator-marketplace/pkg/metrics"
@@ -112,12 +115,6 @@ func main() {
 		os.Exit(0)
 	}
 
-	// set TLS to serve metrics over a secure channel if cert is provided
-	// cert is provided by default by the marketplace-trusted-ca volume mounted as part of the marketplace-operator deployment
-	if err := metrics.ServePrometheus(tlsCertPath, tlsKeyPath); err != nil {
-		logger.Fatalf("failed to serve prometheus metrics: %s", err)
-	}
-
 	namespace, err := apiutils.GetWatchNamespace()
 	if err != nil {
 		logger.Fatalf("failed to get watch namespace: %v", err)
@@ -153,10 +150,18 @@ func main() {
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.ConfigMap{}: {
-					Field: fields.SelectorFromSet(fields.Set{
-						"metadata.namespace": namespace,
-						"metadata.name":      certificateauthority.TrustedCaConfigMapName,
-					}),
+					Namespaces: map[string]cache.Config{
+						namespace: {
+							FieldSelector: fields.SelectorFromSet(fields.Set{
+								"metadata.name": ca.TrustedCaConfigMapName,
+							}),
+						},
+						configmap.ClientCANamespace: {
+							FieldSelector: fields.SelectorFromSet(fields.Set{
+								"metadata.name": configmap.ClientCAConfigMapName,
+							}),
+						},
+					},
 				},
 			},
 		},
@@ -165,6 +170,26 @@ func main() {
 		logger.Fatal(err)
 	}
 
+	clientCAStore := ca.NewClientCAStore(x509.NewCertPool())
+	// Best effort attempt to fetch client rootCA
+	// Should not fail if this does not immediately succeed, the configmap controller will continue to
+	// watch for the right configmap for updating this certpool as soon as it is created.
+	caData, err := configmap.GetClientCAFromConfigMap(context.TODO(), mgr.GetClient(), types.NamespacedName{Name: configmap.ClientCAConfigMapName, Namespace: configmap.ClientCANamespace})
+	if err == nil && len(caData) > 0 {
+		clientCAStore.Update(caData)
+	} else if err != nil {
+		logger.Warn("failed to initialize client CA certPool for the metrics endpoint: %w", err)
+	} else if len(caData) == 0 {
+		logger.Warn("could not find client CA to initialize client rootCA certpool, the clientCA configMap may not be initialized properly yet")
+	}
+
+	// set TLS to serve metrics over a secure channel if cert is provided
+	// cert is provided by default by the marketplace-trusted-ca volume mounted as part of the marketplace-operator deployment
+	if err := metrics.ServePrometheus(tlsCertPath, tlsKeyPath, clientCAStore); err != nil {
+		logger.Fatalf("failed to serve prometheus metrics: %s", err)
+	}
+
+
 	logger.Info("setting up health checks")
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -192,7 +217,7 @@ func main() {
 		}
 
 		logger.Info("setting up controllers")
-		if err := controller.AddToManager(mgr, options.ControllerOptions{}); err != nil {
+		if err := controller.AddToManager(mgr, options.ControllerOptions{ClientCAStore: clientCAStore}); err != nil {
 			logger.Fatal(err)
 		}
 

manifests/06_role_binding.yaml

@@ -36,3 +36,23 @@ roleRef:
   kind: Role
   name: marketplace-operator
   apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: marketplace-operator-auth-reader
+  namespace: kube-system
+  annotations:
+    include.release.openshift.io/hypershift: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+    capability.openshift.io/name: "marketplace"
+subjects:
+- kind: ServiceAccount
+  name: marketplace-operator
+  namespace: openshift-marketplace
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader

pkg/certificateauthority/clientcaconfigmap.go

@@ -0,0 +1,33 @@
+package certificateauthority
+
+import (
+	"crypto/x509"
+	"sync"
+)
+
+type ClientCAStore struct {
+	clientCA *x509.CertPool
+	mutex sync.RWMutex
+}
+
+func NewClientCAStore(certpool *x509.CertPool) *ClientCAStore {
+	if certpool == nil {
+		certpool = x509.NewCertPool()
+	}
+	return &ClientCAStore{clientCA: certpool}
+}
+
+func (c *ClientCAStore) Update(newCAPEM []byte) {
+	if newCAPEM == nil {
+		return
+	}
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.clientCA.AppendCertsFromPEM(newCAPEM)
+}
+
+func (c *ClientCAStore) GetCA() *x509.CertPool {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	return c.clientCA
+}

pkg/controller/configmap/configmap_controller.go

@@ -3,6 +3,7 @@ package configmap
 import (
 	"context"
 	"os"
+
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 
 	mktconfig "github.com/operator-framework/operator-marketplace/pkg/apis/config/v1"
@@ -11,25 +12,36 @@ import (
 	"github.com/operator-framework/operator-marketplace/pkg/controller/options"
 	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const (
+	// the rootCA for authenticating client certs is made available at the
+	// kube-system/extension-apiserver-authentication configMap, under the key
+	// client-ca-file.
+	ClientCANamespace = "kube-system"
+	ClientCAConfigMapName = "extension-apiserver-authentication"
+	ClientCAKey = "client-ca-file"
+)
+
 // Add creates a new ConfigMap Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, _ options.ControllerOptions) error {
-	return add(mgr, newReconciler(mgr))
+func Add(mgr manager.Manager, o options.ControllerOptions) error {
+	return add(mgr, NewReconciler(mgr, o.ClientCAStore))
 }
 
 // newReconciler returns a new ReconcileConfigMap.
-func newReconciler(mgr manager.Manager) *ReconcileConfigMap {
+func NewReconciler(mgr manager.Manager, clientCAStore *ca.ClientCAStore) *ReconcileConfigMap {
 	client := mgr.GetClient()
 	return &ReconcileConfigMap{
 		client:  client,
 		handler: ca.NewHandler(client),
+		clientCAStore: clientCAStore,
 	}
 }
 
@@ -54,13 +66,21 @@ func getPredicateFunctions() predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {
 			// If the ConfigMap is created we should kick off an event.
+			if e.Object.GetName() == ClientCAConfigMapName && 
+				e.Object.GetNamespace() == ClientCANamespace {
+					return true
+			}
 			if e.Object.GetName() == ca.TrustedCaConfigMapName {
 				return true
 			}
 			return false
 		},
 		UpdateFunc: func(e event.UpdateEvent) bool {
 			// If the ConfigMap is updated we should kick off an event.
+			if e.ObjectOld.GetName() == ClientCAConfigMapName && 
+				e.ObjectOld.GetNamespace() == ClientCANamespace {
+					return true
+			}
 			if e.ObjectOld.GetName() == ca.TrustedCaConfigMapName {
 				return true
 			}
@@ -81,13 +101,17 @@ var _ reconcile.Reconciler = &ReconcileConfigMap{}
 type ReconcileConfigMap struct {
 	client  client.Client
 	handler ca.Handler
+	clientCAStore *ca.ClientCAStore
 }
 
 // Reconcile will restart the marketplace operator if the Certificate Authority ConfigMap is
 // not in sync with the Certificate Authority bundle on disk..
 func (r *ReconcileConfigMap) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
 	log.Printf("Reconciling ConfigMap %s/%s", request.Namespace, request.Name)
 
+	if request.Name == ClientCAConfigMapName && request.Namespace == ClientCANamespace {
+		return r.updateClientCA(ctx, request)
+	}
 	// Check if the CA ConfigMap is in the same namespace that Marketplace is deployed in.
 	isConfigMapInOtherNamespace, err := shared.IsObjectInOtherNamespace(request.Namespace)
 	if err != nil {
@@ -114,3 +138,32 @@ func isRunningOnPod() bool {
 	_, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	return !os.IsNotExist(err)
 }
+
+func (r *ReconcileConfigMap) updateClientCA(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	caData, err := GetClientCAFromConfigMap(ctx, r.client, request.NamespacedName)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	if len(caData) == 0 {
+		return reconcile.Result{}, nil
+	}
+	r.clientCAStore.Update(caData)
+	return reconcile.Result{}, nil
+}
+
+// GetClientCAFromConfigMap is used for fetching the clientCA from the provided configMap reference.
+// This is used both for initializing and updating the cached clientCA certPool.
+func GetClientCAFromConfigMap(ctx context.Context, c client.Client, configMapKey types.NamespacedName) ([]byte, error) {
+	// Get configMap object
+	clientCAConfigMap := &corev1.ConfigMap{}
+	if err := c.Get(ctx, configMapKey, clientCAConfigMap); err != nil {
+		// Requested object was not found, could have been deleted after reconcile request.
+		// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
+		// Return and don't requeue
+		return nil, client.IgnoreNotFound(err)
+	}
+	if newCA, ok := clientCAConfigMap.Data[ClientCAKey]; ok {
+		return []byte(newCA), nil
+	}
+	return nil, nil
+}