From 8d1aaa6c8ce26ab6be3dd581311e157f10183e22 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 12 Sep 2025 11:18:29 +0200 Subject: [PATCH 1/5] Add dummy test --- server/raft_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/raft_test.go b/server/raft_test.go index ab704357e5c..77347eda745 100644 --- a/server/raft_test.go +++ b/server/raft_test.go @@ -3781,3 +3781,7 @@ func TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) { } } } + +func TestFlakyDetectorCI(t *testing.T) { + // TEST ONLY +} From ddb3155904ee22007bd37e2822ab204871610bdc Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 12 Sep 2025 13:26:42 +0200 Subject: [PATCH 2/5] Add a dummy test in server/pse --- server/pse/pse_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/pse/pse_test.go b/server/pse/pse_test.go index 5924a2be121..1038fc8bbe2 100644 --- a/server/pse/pse_test.go +++ b/server/pse/pse_test.go @@ -22,6 +22,10 @@ import ( "testing" ) +func TestInServerPSE(t *testing.T) { + +} + func TestPSEmulationCPU(t *testing.T) { if runtime.GOOS == "windows" { t.Skipf("Skipping this test on Windows") From dd934adc3b2b5e2a8ca9142aa03c70b10d77a2f1 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 12 Sep 2025 13:29:15 +0200 Subject: [PATCH 3/5] Add a long running dummy test --- server/raft_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/raft_test.go b/server/raft_test.go index 77347eda745..21c3c77a084 100644 --- a/server/raft_test.go +++ b/server/raft_test.go @@ -3785,3 +3785,8 @@ func TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) { func TestFlakyDetectorCI(t *testing.T) { // TEST ONLY } + +func TestFlakyDetectorCILong(t *testing.T) { + // TEST ONLY + time.Sleep(10 * time.Second) +} From 01f818c233b253c9a39750823e32df8095dab61b Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 12 Sep 2025 14:15:22 +0200 Subject: [PATCH 4/5] Add dummy flaky test --- server/raft_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/raft_test.go b/server/raft_test.go index 21c3c77a084..019b10f1bcc 100644 --- a/server/raft_test.go +++ b/server/raft_test.go @@ -3782,6 +3782,12 @@ func TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) { } } +// func TestFlakyDetectorFail(t *testing.T) { +// if rand.Intn(10) == 0 { +// t.FailNow() +// } +// } + func TestFlakyDetectorCI(t *testing.T) { // TEST ONLY } From 40195f94d5c5491dc81eae78dbccbce7a88c7ecd Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 12 Sep 2025 11:16:05 +0200 Subject: [PATCH 5/5] Add workflow to detect flaky tests --- .github/workflows/flaky.yml | 62 +++++++++++++++++++ scripts/gitTestDetector.go | 120 ++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 .github/workflows/flaky.yml create mode 100644 scripts/gitTestDetector.go diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml new file mode 100644 index 00000000000..9394491baca --- /dev/null +++ b/.github/workflows/flaky.yml @@ -0,0 +1,62 @@ +# This GitHub Actions workflow finds new tests in pull requests +# and runs them repeatedly to check for flakiness. +name: 'Flaky Test Detector' + +on: + pull_request: + branches: + - main + +jobs: + stress-new-test-additions: + name: Stress new tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + + - name: Find New Tests + id: find_new_tests + run: go run scripts/gitTestDetector.go + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + + - name: Run New Tests + if: steps.find_new_tests.outputs.new_tests != '' + env: + RUN_COUNT: 100 + TOTAL_TIMEOUT: 100s + run: | + echo "Found new tests to run:" + echo "${{ steps.find_new_tests.outputs.new_tests }}" | tr ' ' '\n' + fail=0 + for test_info in ${{ steps.find_new_tests.outputs.new_tests }}; do + test_package=$(echo $test_info | cut -d':' -f1) + test_name=$(echo $test_info | cut -d':' -f2) + + echo "---" + echo "Running test '$test_package/$test_name' for up to ${TOTAL_TIMEOUT}" + + exit_code=0 + timeout ${TOTAL_TIMEOUT} go test -v -failfast -count=${RUN_COUNT} ./$test_package -run "^${test_name}$" || exit_code=$? + + if [ $exit_code -ne 0 ]; then + if [ $exit_code -eq 124 ]; then + echo "::warning title=Test Timeout::Test '$test_name' did not complete ${RUN_COUNT} runs within the time limit." + else + echo "::error title=Flaky test detected:: Test '$test_name' failed with exit code $exit_code during one of its ${RUN_COUNT} runs." + fail=1 + fi + else + echo "Test '$test_name' passed all ${RUN_COUNT} runs within the time limit." + fi + done + exit $fail diff --git a/scripts/gitTestDetector.go b/scripts/gitTestDetector.go new file mode 100644 index 00000000000..9303a1d6493 --- /dev/null +++ b/scripts/gitTestDetector.go @@ -0,0 +1,120 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func main() { + baseCommit := os.Getenv("BASE_SHA") + headCommit := os.Getenv("HEAD_SHA") + + if baseCommit == "" || headCommit == "" { + fmt.Println("Warning: BASE_SHA or HEAD_SHA not set. Falling back to 'HEAD~1...HEAD'.") + baseCommit = "HEAD~1" + headCommit = "HEAD" + } + + fmt.Printf("Checking for new test cases between %s and %s\n", baseCommit, headCommit) + + diffCmd := exec.Command("git", "diff", "--name-only", + baseCommit, headCommit, "--", "*_test.go") + diffOutput, err := diffCmd.Output() + if err != nil { + log.Fatalf("Failed to run git diff: %v", err) + } + + changedTestFiles := strings.Split(strings.TrimSpace(string(diffOutput)), "\n") + if len(changedTestFiles) == 0 || changedTestFiles[0] == "" { + fmt.Println("No `_test.go` files were changed in this pull request.") + setOutput("new_tests", "") + os.Exit(0) + } + fmt.Printf("Found changed test files: %v\n", changedTestFiles) + + var newTests []string + + for _, file := range changedTestFiles { + oldContent, _ := getFileContentAtCommit(baseCommit, file) + newContent, err := getFileContentAtCommit(headCommit, file) + if err != nil { + fmt.Printf("Could not get new content for %s: %v\n", file, err) + continue + } + + oldTests, _ := getTestFunctions(oldContent) + newTestsMap, err := getTestFunctions(newContent) + if err != nil { + fmt.Printf("Could not parse new content for %s: %v\n", file, err) + continue + } + + testPackage := filepath.Dir(file) + + for testName := range newTestsMap { + if !oldTests[testName] { + fmt.Printf("Found new test: '%s' in file '%s'\n", testName, file) + newTests = append(newTests, fmt.Sprintf("%s:%s", testPackage, testName)) + } + } + } + + if len(newTests) > 0 { + setOutput("new_tests", strings.Join(newTests, " ")) + } else { + fmt.Println("No new test functions were found in the changed files.") + setOutput("new_tests", "") + } +} + +// setOutput appends a key-value pair to the GITHUB_OUTPUT file. +func setOutput(name, value string) { + outputFile := os.Getenv("GITHUB_OUTPUT") + if outputFile == "" { + fmt.Println("GITHUB_OUTPUT not set. Skipping setting output.") + return + } + f, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open GITHUB_OUTPUT file: %v", err) + } + defer f.Close() + if _, err := f.WriteString(fmt.Sprintf("%s=%s\n", name, value)); err != nil { + log.Fatalf("Failed to write to GITHUB_OUTPUT file: %v", err) + } +} + +// getFileContentAtCommit retrieves the content of a file at a specific git commit. +func getFileContentAtCommit(commitHash, filePath string) (string, error) { + cmd := exec.Command("git", "show", fmt.Sprintf("%s:%s", commitHash, filePath)) + output, err := cmd.Output() + if err != nil { + return "", nil + } + return string(output), nil +} + +// getTestFunctions parses Go source code and returns a map of test function names. +func getTestFunctions(source string) (map[string]bool, error) { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, "src.go", source, 0) + if err != nil { + return nil, err + } + tests := make(map[string]bool) + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && strings.HasPrefix(fn.Name.Name, "Test") { + tests[fn.Name.Name] = true + } + return true + }) + return tests, nil +}