[issue-464] Create a Prometheus ServiceMonitor object that can capture/collect metrics from deployed SonataFlow instances

Author: jianrongzhang89

.github/workflows/e2e.yml

@@ -17,6 +17,7 @@ env:
   PYTHON_VERSION: "3.10"
   KIND_VERSION: v0.20.0
   KNATIVE_VERSION: v1.12.5
+  PROMETHEUS_VERSION: v0.70.0
   OPERATOR_IMAGE_NAME: "127.0.0.1:5001/kogito-serverless-operator:0.0.1"
 
 jobs:
@@ -68,6 +69,9 @@ jobs:
       - name: Deploy Knative Eventing and Serving
         run: make KNATIVE_VERSION=${{ env.KNATIVE_VERSION }} deploy-knative
 
+      - name: Deploy Prometheus
+        run: make PROMETHEUS_VERSION=${{ env.PROMETHEUS_VERSION }} deploy-prometheus
+
       - name: Set OPERATOR_IMAGE_NAME to Point to Kind's Local Registry
         run: echo "OPERATOR_IMAGE_NAME=${{ env.OPERATOR_IMAGE_NAME }}" >> $GITHUB_ENV
 
@@ -92,6 +96,9 @@ jobs:
       - name: Run E2E Tests for Persistent Flows
         run: make test-e2e label=flows-persistence
 
+      - name: Run E2E Tests for Workflow Monitoring
+        run: make test-e2e label=flows-monitoring
+
       - name: Run E2E Tests for Platform
         run: make test-e2e label=platform
 

Makefile

@@ -123,8 +123,7 @@ test: manifests generate envtest test-api ## Run tests.
 	@$(MAKE) vet
 	@$(MAKE) fmt
 	@echo "🔍 Running controller tests..."
-	@KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
-	go test $(shell go list ./... | grep -v /test/) -coverprofile cover.out > /dev/null 2>&1
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(shell go list ./... | grep -v /test/) -coverprofile cover.out
 	@echo "✅  Tests completed successfully. Coverage report generated: cover.out."
 
 .PHONY: test-api
@@ -264,6 +263,8 @@ GOLANGCI_LINT_VERSION ?= v1.57.2
 KIND_VERSION ?= v0.20.0
 KNATIVE_VERSION ?= v1.13.2
 TIMEOUT_SECS ?= 180s
+PROMETHEUS_VERSION ?= v0.70.0
+GRAFANA_VERSION ?= v5.13.0
 
 KNATIVE_SERVING_PREFIX ?= "https://github.com/knative/serving/releases/download/knative-$(KNATIVE_VERSION)"
 KNATIVE_EVENTING_PREFIX ?= "https://github.com/knative/eventing/releases/download/knative-$(KNATIVE_VERSION)"
@@ -402,7 +403,7 @@ generate-all: generate generate-deploy bundle
 	@$(MAKE) fmt
 
 .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up and running to run this target, and run container-builder before the test
-label = "flows-ephemeral" # possible values are flows-ephemeral, flows-persistence, platform, cluster
+label = "flows-ephemeral" # possible values are flows-ephemeral, flows-persistence, flows-monitoring, platform, cluster
 test-e2e:
 ifeq ($(label), cluster)
 	@echo "🌐 Running e2e tests for cluster..."
@@ -424,8 +425,13 @@ else ifeq ($(label), flows-persistence)
 	go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go \
 	-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
 	-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
+else ifeq ($(label), flows-monitoring)
+	@echo "🔁 Running e2e tests for flows-monitoring..."
+	go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go \
+	-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
+	-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
 else
-	@echo "❌  Invalid label. Please use one of: cluster, platform, flows-ephemeral, flows-persistence"
+	@echo "❌  Invalid label. Please use one of: cluster, platform, flows-ephemeral, flows-persistence, flows-monitoring"
 endif
 
 
@@ -450,6 +456,18 @@ deploy-knative:
 	kubectl wait  --for=condition=Ready=True KnativeServing/knative-serving -n knative-serving --timeout=$(TIMEOUT_SECS)
 	kubectl wait  --for=condition=Ready=True KnativeEventing/knative-eventing -n knative-eventing --timeout=$(TIMEOUT_SECS)
 
+.PHONY: deploy-prometheus
+deploy-prometheus: create-cluster
+	kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/$(PROMETHEUS_VERSION)/bundle.yaml
+	kubectl wait  --for=condition=Available=True deploy/prometheus-operator -n default --timeout=$(TIMEOUT_SECS)
+	kubectl apply -f ./test/testdata/prometheus.yaml -n default
+	kubectl wait  --for=condition=Available=True prometheus/prometheus -n default --timeout=$(TIMEOUT_SECS)
+
+.PHONY: deploy-grafana
+deploy-grafana: create-cluster
+	kubectl create -f https://github.com/grafana/grafana-operator/releases/download/$(GRAFANA_VERSION)/kustomize-cluster_scoped.yaml
+	kubectl wait  --for=condition=Available=True deploy/grafana-operator-controller-manager -n grafana --timeout=$(TIMEOUT_SECS)
+
 .PHONY: delete-cluster
 delete-cluster: install-kind
 	kind delete cluster && $(BUILDER) rm -f kind-registry

api/v1alpha08/sonataflowplatform_types.go

@@ -63,6 +63,9 @@ type SonataFlowPlatformSpec struct {
 	// These properties MAY NOT be propagated to a SonataFlowClusterPlatform since PropertyVarSource can only refer local context sources.
 	// +optional
 	Properties *PropertyPlatformSpec `json:"properties,omitempty"`
+	// Settings for Prometheus monitoring
+	// +optional
+	Monitoring *PlatformMonitoringOptionsSpec `json:"monitoring,omitempty"`
 }
 
 // PlatformEventingSpec specifies the Knative Eventing integration details in the platform.
@@ -74,6 +77,15 @@ type PlatformEventingSpec struct {
 	Broker *duckv1.Destination `json:"broker,omitempty"`
 }
 
+// PlatformMonitoringOptionsSpec specifies the settings for monitoring
+// +k8s:openapi-gen=true
+type PlatformMonitoringOptionsSpec struct {
+	// Enabled indicates whether monitoring with Prometheus metrics is enabled
+	// +optional
+	// +default: false
+	Enabled bool `json:"enabled,omitempty"`
+}
+
 // PlatformCluster is the kind of orchestration cluster the platform is installed into
 // +kubebuilder:validation:Enum=kubernetes;openshift
 type PlatformCluster string

api/v1alpha08/zz_generated.deepcopy.go

@@ -524,6 +524,14 @@ spec:
                         type: string
                     type: object
                 type: object
+              monitoring:
+                description: Settings for Prometheus monitoring
+                properties:
+                  enabled:
+                    description: Enabled indicates whether monitoring with Prometheus
+                      metrics is enabled
+                    type: boolean
+                type: object
               persistence:
                 description: |-
                   Persistence defines the platform persistence configuration. When this field is set,

bundle/manifests/sonataflow.org_sonataflowplatforms.yaml

@@ -27,14 +27,14 @@ import (
 	"github.com/apache/incubator-kie-kogito-serverless-operator/internal/controller"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/internal/controller/cfg"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/version"
+	prometheus "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"k8s.io/klog/v2/klogr"
 	eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1"
 	sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
-	"k8s.io/klog/v2/klogr"
-
 	"k8s.io/klog/v2"
 
 	"github.com/apache/incubator-kie-kogito-serverless-operator/utils"
@@ -66,6 +66,7 @@ func init() {
 	utilruntime.Must(sourcesv1.AddToScheme(scheme))
 	utilruntime.Must(eventingv1.AddToScheme(scheme))
 	utilruntime.Must(servingv1.AddToScheme(scheme))
+	utilruntime.Must(prometheus.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 

cmd/main.go

@@ -524,6 +524,14 @@ spec:
                         type: string
                     type: object
                 type: object
+              monitoring:
+                description: Settings for Prometheus monitoring
+                properties:
+                  enabled:
+                    description: Enabled indicates whether monitoring with Prometheus
+                      metrics is enabled
+                    type: boolean
+                type: object
               persistence:
                 description: |-
                   Persistence defines the platform persistence configuration. When this field is set,

config/crd/bases/sonataflow.org_sonataflowplatforms.yaml

@@ -21,6 +21,17 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - sonataflow.org
   resources:

config/rbac/role.yaml

@@ -2247,6 +2247,7 @@ github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-2023020916533
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v27 v27.0.6 h1:oiOZuBmGHvrGM1X9uNUAUlLgp5r1UUO/M/KnbHnLRlQ=
 github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
+github.com/google/go-jsonnet v0.18.0/go.mod h1:C3fTzyVJDslXdiTqw/bTFk7vSGyCtH3MGRbDfvEwGd0=
 github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk=
 github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -2588,6 +2589,7 @@ github.com/openzipkin/zipkin-go v0.3.0/go.mod h1:4c3sLeE8xjNqehmF5RpAFLPLJxXscc0
 github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA=
 github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY=
 github.com/operator-framework/api v0.1.1 h1:DbfxRJUPMQlQW6nbfoNzWLxv1rIv13Gt8GbsF2aglFk=
+github.com/operator-framework/operator-lib v0.11.0/go.mod h1:RpyKhFAoG6DmKTDIwMuO6pI3LRc8IE9rxEYWy476o6g=
 github.com/operator-framework/operator-registry v1.6.1 h1:Ow0Ko9DRIZ4xvH55vFAslcTy6A9FhlIeXvm+FhyRd84=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=

go.work.sum

@@ -30,7 +30,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1"
 	sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1"
@@ -44,7 +43,6 @@ import (
 
 var servingClient clientservingv1.ServingV1Interface
 var eventingClient clienteventingv1.EventingV1Interface
-var discoveryClient discovery.DiscoveryInterface
 
 type Availability struct {
 	Eventing bool
@@ -92,23 +90,8 @@ func NewKnativeEventingClient(cfg *rest.Config) (*clienteventingv1.EventingV1Cli
 	return clienteventingv1.NewForConfig(cfg)
 }
 
-func getDiscoveryClient(cfg *rest.Config) (discovery.DiscoveryInterface, error) {
-	if discoveryClient == nil {
-		if cli, err := discovery.NewDiscoveryClientForConfig(cfg); err != nil {
-			return nil, err
-		} else {
-			discoveryClient = cli
-		}
-	}
-	return discoveryClient, nil
-}
-
-func SetDiscoveryClient(cli discovery.DiscoveryInterface) {
-	discoveryClient = cli
-}
-
 func GetKnativeAvailability(cfg *rest.Config) (*Availability, error) {
-	if cli, err := getDiscoveryClient(cfg); err != nil {
+	if cli, err := utils.GetDiscoveryClient(cfg); err != nil {
 		return nil, err
 	} else {
 		apiList, err := cli.ServerGroups()