Skip to content

Commit 921a9a9

Browse files
JoibelCopilot
andauthored
chore(test): run tests with gotestsum and retries (#14623)
Signed-off-by: Alan Clucas <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 81309bb commit 921a9a9

File tree

6 files changed

+269
-67
lines changed

6 files changed

+269
-67
lines changed

.github/workflows/ci-build.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,12 +243,20 @@ jobs:
243243
- test: test-api
244244
profile: postgres
245245
use-api: true
246+
- test: test-metrics
247+
profile: mysql
248+
use-api: true
249+
- test: test-metrics
250+
profile: postgres
251+
use-api: true
246252
- test: test-cli
247253
profile: mysql
248254
use-api: true
255+
# Cron tests are not retryable withou a controller restart
249256
- test: test-cron
250257
profile: minimal
251258
use-api: false
259+
retries: 0
252260
- test: test-examples
253261
profile: minimal
254262
use-api: false
@@ -317,6 +325,7 @@ jobs:
317325
- name: Install and start K3S
318326
env:
319327
K8S_VERSION: ${{ matrix.k8s_version || 'max' }}
328+
TEST_RETRIES: ${{ matrix.retries || '2' }}
320329
run: |
321330
. hack/k8s-versions.sh
322331
export INSTALL_K3S_VERSION="${K8S_VERSIONS[$K8S_VERSION]}+k3s1"

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ git-ask-pass.sh
2222
/workflow-controller
2323
/.scannerwork/
2424
/test-results/
25+
/test/reports/
2526
/package-lock.json
2627
/pkg/apiclient/_.secondary.swagger.json
2728
/pkg/apiclient/clusterworkflowtemplate/cluster-workflow-template.swagger.json

.golangci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ run:
88
- examples
99
- corefunctional
1010
- functional
11+
- metrics
1112
- plugins
1213
linters:
1314
enable:

Makefile

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,19 @@ endif
5050
# -- test options
5151
E2E_WAIT_TIMEOUT ?= 90s # timeout for wait conditions
5252
E2E_PARALLEL ?= 20
53-
E2E_SUITE_TIMEOUT ?= 15m
54-
GOTEST ?= go test -v -p 20
53+
E2E_SUITE_TIMEOUT ?= 25m
54+
TEST_RETRIES ?= 2
55+
JSON_TEST_OUTPUT := test/reports/json
56+
# gotest function: gotest(packages, name, parameters)
57+
# packages: passed to gotestsum via --packages parameter
58+
# name: not used currently
59+
# parameters: passed to go test after the --
60+
$(JSON_TEST_OUTPUT):
61+
mkdir -p $(JSON_TEST_OUTPUT)
62+
63+
define gotest
64+
$(TOOL_GOTESTSUM) --rerun-fails=$(TEST_RETRIES) --jsonfile=$(JSON_TEST_OUTPUT)/$(2).json --format=testname --packages $(1) -- $(3)
65+
endef
5566
ALL_BUILD_TAGS ?= api,cli,cron,executor,examples,corefunctional,functional,plugins
5667
BENCHMARK_COUNT ?= 6
5768

@@ -123,6 +134,7 @@ TOOL_OPENAPI_GEN := $(GOPATH)/bin/openapi-gen
123134
TOOL_SWAGGER := $(GOPATH)/bin/swagger
124135
TOOL_GOIMPORTS := $(GOPATH)/bin/goimports
125136
TOOL_GOLANGCI_LINT := $(GOPATH)/bin/golangci-lint
137+
TOOL_GOTESTSUM := $(GOPATH)/bin/gotestsum
126138

127139
# npm bin -g will do this on later npms than we have
128140
NVM_BIN ?= $(shell npm config get prefix)/bin
@@ -400,6 +412,11 @@ $(TOOL_GOIMPORTS): Makefile
400412
ifneq ($(USE_NIX), true)
401413
go install golang.org/x/tools/cmd/[email protected]
402414
endif
415+
$(TOOL_GOTESTSUM): Makefile
416+
# update this in Nix when upgrading it here
417+
ifneq ($(USE_NIX), true)
418+
go install gotest.tools/[email protected]
419+
endif
403420

404421
$(TOOL_CLANG_FORMAT):
405422
ifeq (, $(shell which clang-format))
@@ -527,9 +544,9 @@ lint-ui: ui/dist/app/index.html
527544

528545
# for local we have a faster target that prints to stdout, does not use json, and can cache because it has no coverage
529546
.PHONY: test
530-
test: ui/dist/app/index.html util/telemetry/metrics_list.go util/telemetry/attributes.go
547+
test: ui/dist/app/index.html util/telemetry/metrics_list.go util/telemetry/attributes.go $(TOOL_GOTESTSUM) $(JSON_TEST_OUTPUT)
531548
go build ./...
532-
env KUBECONFIG=/dev/null $(GOTEST) ./...
549+
env KUBECONFIG=/dev/null $(call gotest,./...,unit,-p 20)
533550
# marker file, based on it's modification time, we know how long ago this target was run
534551
@mkdir -p dist
535552
touch dist/test
@@ -651,25 +668,25 @@ mysql-dump:
651668

652669
test-cli: ./dist/argo
653670

654-
test-%:
655-
E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $* -parallel $(E2E_PARALLEL) ./test/e2e
671+
test-%: $(TOOL_GOTESTSUM) $(JSON_TEST_OUTPUT)
672+
E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) $(call gotest,./test/e2e,$@,-timeout $(E2E_SUITE_TIMEOUT) --tags $*)
656673

657674
.PHONY: test-%-sdk
658675
test-%-sdk:
659676
make --directory sdks/$* install test -B
660677

661-
Test%:
662-
E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $(ALL_BUILD_TAGS) -parallel $(E2E_PARALLEL) ./test/e2e -run='.*/$*'
678+
Test%: $(TOOL_GOTESTSUM) $(JSON_TEST_OUTPUT)
679+
E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) $(call gotest,./test/e2e,$@,-timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $(ALL_BUILD_TAGS) -parallel $(E2E_PARALLEL) -run='.*/$*')
663680

664-
Benchmark%:
665-
go test --tags $(ALL_BUILD_TAGS) ./test/e2e -run='$@' -benchmem -count=$(BENCHMARK_COUNT) -bench .
681+
Benchmark%: $(TOOL_GOTESTSUM) $(JSON_TEST_OUTPUT)
682+
$(call gotest,./test/e2e,$@,--tags $(ALL_BUILD_TAGS) -run='$@' -benchmem -count=$(BENCHMARK_COUNT) -bench .)
666683

667684
# clean
668685

669686
.PHONY: clean
670687
clean:
671688
go clean
672-
rm -Rf test-results node_modules vendor v2 v3 argoexec-linux-amd64 dist/* ui/dist
689+
rm -Rf test/reports test-results node_modules vendor v2 v3 argoexec-linux-amd64 dist/* ui/dist
673690

674691
# Build telemetry files
675692
TELEMETRY_BUILDER := $(shell find util/telemetry/builder -type f -name '*.go')

test/e2e/fixtures/metrics.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package fixtures
2+
3+
import (
4+
"regexp"
5+
"strconv"
6+
"strings"
7+
"testing"
8+
9+
"github.com/gavv/httpexpect/v2"
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
// MetricBaseline stores baseline metric values for comparison
14+
type MetricBaseline struct {
15+
t *testing.T
16+
httpClient func() *httpexpect.Expect
17+
baselines map[string]float64
18+
expectedIncreases map[string]float64
19+
}
20+
21+
// NewMetricBaseline creates a new metric baseline tracker
22+
func NewMetricBaseline(t *testing.T, httpClient func() *httpexpect.Expect) *MetricBaseline {
23+
return &MetricBaseline{
24+
t: t,
25+
httpClient: httpClient,
26+
baselines: make(map[string]float64),
27+
expectedIncreases: make(map[string]float64),
28+
}
29+
}
30+
31+
// getCurrentMetricValues fetches current values for the specified metrics
32+
func (mb *MetricBaseline) getCurrentMetricValues(metricsMap map[string]float64) map[string]float64 {
33+
mb.t.Helper()
34+
35+
body := mb.httpClient().GET("").
36+
Expect().
37+
Status(200).
38+
Body().
39+
Raw()
40+
41+
values := make(map[string]float64)
42+
for metric := range metricsMap {
43+
values[metric] = parseMetricValue(body, metric)
44+
}
45+
46+
return values
47+
}
48+
49+
// CaptureBaseline captures baselines for all metrics in the expected increases map
50+
// and stores the expected increases for later use with ExpectIncrease()
51+
func (mb *MetricBaseline) CaptureBaseline(expectedIncreases map[string]float64) *MetricBaseline {
52+
mb.t.Helper()
53+
54+
// Capture current metric values and store them as baselines
55+
currentValues := mb.getCurrentMetricValues(expectedIncreases)
56+
57+
for metric := range expectedIncreases {
58+
value := currentValues[metric]
59+
mb.baselines[metric] = value
60+
mb.t.Logf("Baseline for %s: %f", metric, value)
61+
}
62+
63+
// Store the expected increases for later use
64+
mb.expectedIncreases = expectedIncreases
65+
66+
return mb
67+
}
68+
69+
// ExpectIncrease checks that the metrics have increased by the amounts
70+
// specified in the map passed to CaptureBaseline()
71+
func (mb *MetricBaseline) ExpectIncrease() {
72+
mb.t.Helper()
73+
74+
if len(mb.expectedIncreases) == 0 {
75+
mb.t.Fatal("No expected increases stored. Call CaptureBaseline() first.")
76+
}
77+
78+
currentValues := mb.getCurrentMetricValues(mb.expectedIncreases)
79+
80+
for metric, expectedIncrease := range mb.expectedIncreases {
81+
baseline := mb.baselines[metric] // defaults to 0 if not found
82+
currentValue := currentValues[metric]
83+
actualIncrease := currentValue - baseline
84+
85+
mb.t.Logf("Metric %s: baseline=%f, current=%f, expected_increase=%f, actual_increase=%f",
86+
metric, baseline, currentValue, expectedIncrease, actualIncrease)
87+
88+
assert.InDelta(mb.t, expectedIncrease, actualIncrease, 0.001,
89+
"Expected %s to increase by %f, but it increased by %f (from %f to %f)", metric, expectedIncrease, actualIncrease, baseline, currentValue)
90+
}
91+
}
92+
93+
// parseMetricValue extracts the numeric value from a prometheus metric line
94+
// Returns 0 if the metric is not found
95+
func parseMetricValue(body, metricPattern string) float64 {
96+
// Escape special regex characters in the metric pattern, but keep the spaces
97+
// We'll look for lines that match the pattern and extract the value
98+
lines := strings.Split(body, "\n")
99+
100+
for _, line := range lines {
101+
line = strings.TrimSpace(line)
102+
if line == "" || strings.HasPrefix(line, "#") {
103+
continue
104+
}
105+
106+
// Check if this line matches our metric pattern
107+
if strings.Contains(line, metricPattern) {
108+
// Extract the value using regex
109+
// Prometheus format: metric_name{labels} value
110+
re := regexp.MustCompile(`\s+([0-9.]+)$`)
111+
matches := re.FindStringSubmatch(line)
112+
if len(matches) > 1 {
113+
value, err := strconv.ParseFloat(matches[1], 64)
114+
if err == nil {
115+
return value
116+
}
117+
}
118+
}
119+
}
120+
121+
return 0 // Default to 0 if metric not found
122+
}

0 commit comments

Comments
 (0)