diff --git a/tools/flakeguard/cmd/find.go b/tools/flakeguard/cmd/find.go index 170b0e777..ef06b608e 100644 --- a/tools/flakeguard/cmd/find.go +++ b/tools/flakeguard/cmd/find.go @@ -30,7 +30,7 @@ var FindTestsCmd = &cobra.Command{ // Find all changes in test files and get their package names var changedTestPkgs []string if findByTestFilesDiff { - changedTestFiles, err := git.FindChangedFiles(baseRef, "grep '_test\\.go$'", excludes) + changedTestFiles, err := git.FindChangedFiles(projectPath, baseRef, "grep '_test\\.go$'") if err != nil { log.Fatalf("Error finding changed test files: %v", err) } diff --git a/tools/flakeguard/cmd/run.go b/tools/flakeguard/cmd/run.go index 08cbfe35b..7a1254e91 100644 --- a/tools/flakeguard/cmd/run.go +++ b/tools/flakeguard/cmd/run.go @@ -15,6 +15,7 @@ var RunTestsCmd = &cobra.Command{ Use: "run", Short: "Run tests to check if they are flaky", Run: func(cmd *cobra.Command, args []string) { + projectPath, _ := cmd.Flags().GetString("project-path") testPackagesJson, _ := cmd.Flags().GetString("test-packages-json") testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages") runCount, _ := cmd.Flags().GetInt("run-count") @@ -34,10 +35,11 @@ var RunTestsCmd = &cobra.Command{ } runner := runner.Runner{ - Verbose: true, - RunCount: runCount, - UseRace: useRace, - FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0 + ProjectPath: projectPath, + Verbose: true, + RunCount: runCount, + UseRace: useRace, + FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0 } testResults, err := runner.RunTests(testPackages) @@ -82,6 +84,7 @@ var RunTestsCmd = &cobra.Command{ } func init() { + RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects.") RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages") RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run") RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests") diff --git a/tools/flakeguard/git/git.go b/tools/flakeguard/git/git.go index 7cb9794b3..dc87ad31e 100644 --- a/tools/flakeguard/git/git.go +++ b/tools/flakeguard/git/git.go @@ -3,6 +3,7 @@ package git import ( "bytes" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -14,15 +15,15 @@ import ( // FindChangedFiles executes a git diff against a specified base reference and pipes the output through a user-defined grep command or sequence. // The baseRef parameter specifies the base git reference for comparison (e.g., "main", "develop"). // The filterCmd parameter should include the full command to be executed after git diff, such as "grep '_test.go$'" or "grep -v '_test.go$' | sort". -func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]string, error) { - // Constructing the exclusion part of the git command - excludeStr := "" - for _, path := range excludePaths { - excludeStr += fmt.Sprintf("':(exclude)%s' ", path) +func FindChangedFiles(rootGoModPath, baseRef, filterCmd string) ([]string, error) { + // Find directories containing a go.mod file and build an exclusion string + excludeStr, err := buildExcludeStringForGoModDirs(rootGoModPath) + if err != nil { + return nil, fmt.Errorf("error finding go.mod directories: %w", err) } // First command to list files changed between the baseRef and HEAD, excluding specified paths - diffCmdStr := fmt.Sprintf("git diff --name-only --diff-filter=AM %s...HEAD %s", baseRef, excludeStr) + diffCmdStr := fmt.Sprintf("git diff --name-only --diff-filter=AM %s...HEAD -- %s %s", baseRef, rootGoModPath, excludeStr) diffCmd := exec.Command("bash", "-c", diffCmdStr) // Using a buffer to capture stdout and a separate buffer for stderr @@ -36,7 +37,7 @@ func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]strin return nil, fmt.Errorf("error executing git diff command: %s; error: %w; stderr: %s", diffCmdStr, err, errBuf.String()) } - // Check if there are any files listed, if not, return an empty slice + // Check if there are any files listed; if not, return an empty slice diffOutput := strings.TrimSpace(out.String()) if diffOutput == "" { return []string{}, nil @@ -74,6 +75,32 @@ func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]strin return files, nil } +// buildExcludeStringForGoModDirs searches the given root directory for subdirectories +// containing a go.mod file and returns a formatted string to exclude those directories +// (except the root directory if it contains a go.mod file) from git diff. +func buildExcludeStringForGoModDirs(rootGoModPath string) (string, error) { + var excludeStr string + + err := filepath.Walk(rootGoModPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Name() == "go.mod" { + dir := filepath.Dir(path) + // Skip excluding the root directory if go.mod is found there + if dir != rootGoModPath { + excludeStr += fmt.Sprintf("':(exclude)%s/**' ", dir) + } + } + return nil + }) + if err != nil { + return "", err + } + + return excludeStr, nil +} + func Diff(baseBranch string) (*utils.CmdOutput, error) { return utils.ExecuteCmd("git", "diff", "--name-only", baseBranch) } diff --git a/tools/flakeguard/reports/reports.go b/tools/flakeguard/reports/reports.go index 7c430c82e..bdfd63688 100644 --- a/tools/flakeguard/reports/reports.go +++ b/tools/flakeguard/reports/reports.go @@ -4,9 +4,10 @@ type TestResult struct { TestName string TestPackage string PassRatio float64 + Skipped bool // Indicates if the test was skipped Runs int - Outputs []string // Stores outputs for a test - Skipped bool // Indicates if the test was skipped + Outputs []string // Stores outputs for a test + Durations []float64 // Stores elapsed time in seconds for each run of the test } // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold. diff --git a/tools/flakeguard/runner/runner.go b/tools/flakeguard/runner/runner.go index 8a5699e04..3e9812d5f 100644 --- a/tools/flakeguard/runner/runner.go +++ b/tools/flakeguard/runner/runner.go @@ -14,10 +14,11 @@ import ( ) type Runner struct { - Verbose bool // If true, provides detailed logging. - RunCount int // Number of times to run the tests. - UseRace bool // Enable race detector. - FailFast bool // Stop on first test failure. + ProjectPath string // Path to the Go project directory. + Verbose bool // If true, provides detailed logging. + RunCount int // Number of times to run the tests. + UseRace bool // Enable race detector. + FailFast bool // Stop on first test failure. } // RunTests executes the tests for each provided package and aggregates all results. @@ -58,6 +59,7 @@ func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) { log.Printf("Running command: go %s\n", strings.Join(args, " ")) } cmd := exec.Command("go", args...) + cmd.Dir = r.ProjectPath // cmd.Env = append(cmd.Env, "GOFLAGS=-extldflags=-Wl,-ld_classic") // Ensure modules are enabled var out bytes.Buffer @@ -89,10 +91,11 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { scanner := bufio.NewScanner(bytes.NewReader(data)) for scanner.Scan() { var entry struct { - Action string `json:"Action"` - Test string `json:"Test"` - Package string `json:"Package"` - Output string `json:"Output"` + Action string `json:"Action"` + Test string `json:"Test"` + Package string `json:"Package"` + Output string `json:"Output"` + Elapsed float64 `json:"Elapsed"` } if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { return nil, fmt.Errorf("failed to parse json test output: %s, err: %w", scanner.Text(), err) @@ -120,13 +123,16 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { result.Runs++ case "pass": result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs) + result.Durations = append(result.Durations, entry.Elapsed) case "output": result.Outputs = append(result.Outputs, entry.Output) case "fail": result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs) + result.Durations = append(result.Durations, entry.Elapsed) case "skip": result.Skipped = true result.Runs++ + result.Durations = append(result.Durations, entry.Elapsed) } }