diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml new file mode 100644 index 00000000000..9394491baca --- /dev/null +++ b/.github/workflows/flaky.yml @@ -0,0 +1,62 @@ +# This GitHub Actions workflow finds new tests in pull requests +# and runs them repeatedly to check for flakiness. +name: 'Flaky Test Detector' + +on: + pull_request: + branches: + - main + +jobs: + stress-new-test-additions: + name: Stress new tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + + - name: Find New Tests + id: find_new_tests + run: go run scripts/gitTestDetector.go + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + + - name: Run New Tests + if: steps.find_new_tests.outputs.new_tests != '' + env: + RUN_COUNT: 100 + TOTAL_TIMEOUT: 100s + run: | + echo "Found new tests to run:" + echo "${{ steps.find_new_tests.outputs.new_tests }}" | tr ' ' '\n' + fail=0 + for test_info in ${{ steps.find_new_tests.outputs.new_tests }}; do + test_package=$(echo $test_info | cut -d':' -f1) + test_name=$(echo $test_info | cut -d':' -f2) + + echo "---" + echo "Running test '$test_package/$test_name' for up to ${TOTAL_TIMEOUT}" + + exit_code=0 + timeout ${TOTAL_TIMEOUT} go test -v -failfast -count=${RUN_COUNT} ./$test_package -run "^${test_name}$" || exit_code=$? + + if [ $exit_code -ne 0 ]; then + if [ $exit_code -eq 124 ]; then + echo "::warning title=Test Timeout::Test '$test_name' did not complete ${RUN_COUNT} runs within the time limit." + else + echo "::error title=Flaky test detected:: Test '$test_name' failed with exit code $exit_code during one of its ${RUN_COUNT} runs." + fail=1 + fi + else + echo "Test '$test_name' passed all ${RUN_COUNT} runs within the time limit." + fi + done + exit $fail diff --git a/scripts/gitTestDetector.go b/scripts/gitTestDetector.go new file mode 100644 index 00000000000..9303a1d6493 --- /dev/null +++ b/scripts/gitTestDetector.go @@ -0,0 +1,120 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func main() { + baseCommit := os.Getenv("BASE_SHA") + headCommit := os.Getenv("HEAD_SHA") + + if baseCommit == "" || headCommit == "" { + fmt.Println("Warning: BASE_SHA or HEAD_SHA not set. Falling back to 'HEAD~1...HEAD'.") + baseCommit = "HEAD~1" + headCommit = "HEAD" + } + + fmt.Printf("Checking for new test cases between %s and %s\n", baseCommit, headCommit) + + diffCmd := exec.Command("git", "diff", "--name-only", + baseCommit, headCommit, "--", "*_test.go") + diffOutput, err := diffCmd.Output() + if err != nil { + log.Fatalf("Failed to run git diff: %v", err) + } + + changedTestFiles := strings.Split(strings.TrimSpace(string(diffOutput)), "\n") + if len(changedTestFiles) == 0 || changedTestFiles[0] == "" { + fmt.Println("No `_test.go` files were changed in this pull request.") + setOutput("new_tests", "") + os.Exit(0) + } + fmt.Printf("Found changed test files: %v\n", changedTestFiles) + + var newTests []string + + for _, file := range changedTestFiles { + oldContent, _ := getFileContentAtCommit(baseCommit, file) + newContent, err := getFileContentAtCommit(headCommit, file) + if err != nil { + fmt.Printf("Could not get new content for %s: %v\n", file, err) + continue + } + + oldTests, _ := getTestFunctions(oldContent) + newTestsMap, err := getTestFunctions(newContent) + if err != nil { + fmt.Printf("Could not parse new content for %s: %v\n", file, err) + continue + } + + testPackage := filepath.Dir(file) + + for testName := range newTestsMap { + if !oldTests[testName] { + fmt.Printf("Found new test: '%s' in file '%s'\n", testName, file) + newTests = append(newTests, fmt.Sprintf("%s:%s", testPackage, testName)) + } + } + } + + if len(newTests) > 0 { + setOutput("new_tests", strings.Join(newTests, " ")) + } else { + fmt.Println("No new test functions were found in the changed files.") + setOutput("new_tests", "") + } +} + +// setOutput appends a key-value pair to the GITHUB_OUTPUT file. +func setOutput(name, value string) { + outputFile := os.Getenv("GITHUB_OUTPUT") + if outputFile == "" { + fmt.Println("GITHUB_OUTPUT not set. Skipping setting output.") + return + } + f, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open GITHUB_OUTPUT file: %v", err) + } + defer f.Close() + if _, err := f.WriteString(fmt.Sprintf("%s=%s\n", name, value)); err != nil { + log.Fatalf("Failed to write to GITHUB_OUTPUT file: %v", err) + } +} + +// getFileContentAtCommit retrieves the content of a file at a specific git commit. +func getFileContentAtCommit(commitHash, filePath string) (string, error) { + cmd := exec.Command("git", "show", fmt.Sprintf("%s:%s", commitHash, filePath)) + output, err := cmd.Output() + if err != nil { + return "", nil + } + return string(output), nil +} + +// getTestFunctions parses Go source code and returns a map of test function names. +func getTestFunctions(source string) (map[string]bool, error) { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, "src.go", source, 0) + if err != nil { + return nil, err + } + tests := make(map[string]bool) + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && strings.HasPrefix(fn.Name.Name, "Test") { + tests[fn.Name.Name] = true + } + return true + }) + return tests, nil +} diff --git a/server/pse/pse_test.go b/server/pse/pse_test.go index 5924a2be121..1038fc8bbe2 100644 --- a/server/pse/pse_test.go +++ b/server/pse/pse_test.go @@ -22,6 +22,10 @@ import ( "testing" ) +func TestInServerPSE(t *testing.T) { + +} + func TestPSEmulationCPU(t *testing.T) { if runtime.GOOS == "windows" { t.Skipf("Skipping this test on Windows") diff --git a/server/raft_test.go b/server/raft_test.go index ab704357e5c..019b10f1bcc 100644 --- a/server/raft_test.go +++ b/server/raft_test.go @@ -3781,3 +3781,18 @@ func TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) { } } } + +// func TestFlakyDetectorFail(t *testing.T) { +// if rand.Intn(10) == 0 { +// t.FailNow() +// } +// } + +func TestFlakyDetectorCI(t *testing.T) { + // TEST ONLY +} + +func TestFlakyDetectorCILong(t *testing.T) { + // TEST ONLY + time.Sleep(10 * time.Second) +}