chore: flakey test finder

2026-06-29 20:40:05 +01:00 · 2026-06-29 20:40:05 +01:00 · ba7d9f51a1
parent 1cbd1cfb6b
commit ba7d9f51a1
4 changed files with 219 additions and 106 deletions
--- a/.gitignore
+++ b/.gitignore
@ -27,4 +27,5 @@ webui.dev
 sessions.yaml
 docs/build/
 build/
+Binary/
 .cursor
--- a/service/scripts/find-flakey-tests-inf/Makefile
+++ b/service/scripts/find-flakey-tests-inf/Makefile
@ -0,0 +1,5 @@
+codestyle:
+	go fmt ./...
+	go vet ./...
+	gocyclo -over 4 .
+	gocritic check ./...
--- a/service/scripts/find-flakey-tests-inf/main.go
+++ b/service/scripts/find-flakey-tests-inf/main.go
@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@ -50,6 +51,12 @@ type jsonlRecord struct {
 	FailureDetails []testFailure `json:"failureDetails"`
 }

+type testRunState struct {
+	summary       runSummary
+	failures      []testFailure
+	failureOutput map[string]*strings.Builder
+}
+
 func initLog() {
 	logFormat := os.Getenv("OLIVETIN_LOG_FORMAT")

@ -65,27 +72,46 @@ func initLog() {
 	log.SetLevel(log.InfoLevel)
 }

-func serviceRoot() string {
+func hasGoMod(dir string) bool {
+	stat, err := os.Stat(filepath.Join(dir, "go.mod"))
+	return err == nil && !stat.IsDir()
+}
+
+func rootFromExecutable() (string, bool) {
 	exe, err := os.Executable()
-	if err == nil {
-		candidate := filepath.Join(filepath.Dir(exe), "..", "..")
-		if stat, statErr := os.Stat(filepath.Join(candidate, "go.mod")); statErr == nil && !stat.IsDir() {
-			return candidate
-		}
+	if err != nil {
+		return "", false
 	}

+	candidate := filepath.Join(filepath.Dir(exe), "..", "..")
+	if hasGoMod(candidate) {
+		return candidate, true
+	}
+
+	return "", false
+}
+
+func rootFromWorkingDir() string {
 	wd, err := os.Getwd()
 	if err != nil {
 		return "."
 	}

-	if stat, statErr := os.Stat(filepath.Join(wd, "go.mod")); statErr == nil && !stat.IsDir() {
+	if hasGoMod(wd) {
 		return wd
 	}

 	return filepath.Join(wd, "..")
 }

+func serviceRoot() string {
+	if root, ok := rootFromExecutable(); ok {
+		return root
+	}
+
+	return rootFromWorkingDir()
+}
+
 func envOrDefault(name, fallback string) string {
 	if value := strings.TrimSpace(os.Getenv(name)); value != "" {
 		return value
@ -112,20 +138,20 @@ func formatRunCounts(summary runSummary) string {
 	return fmt.Sprintf("%d pass %d fail %d skip", summary.Passes, summary.Failures, summary.Skipped)
 }

-func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
-	timestamp := time.Now().UTC().Format(time.RFC3339)
-	passed := exitCode == 0
-	result := "PASS"
-	if !passed {
-		result = "FAIL"
+func runResultLabel(exitCode int) string {
+	if exitCode == 0 {
+		return "PASS"
 	}
+	return "FAIL"
+}

+func buildRunLogBlock(run int, timestamp string, exitCode int, summary runSummary, failures []testFailure, durationMs int64) string {
 	block := []string{
 		fmt.Sprintf(
 			"=== RUN %d | %s | %s | %d pass %d fail %d skip | %.1fs ===",
 			run,
 			timestamp,
-			result,
+			runResultLabel(exitCode),
 			summary.Passes,
 			summary.Failures,
 			summary.Skipped,
@ -138,10 +164,10 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
 	}

 	block = append(block, "")
-	if err := appendFile(logFile, strings.Join(block, "\n")+"\n"); err != nil {
-		return err
-	}
+	return strings.Join(block, "\n") + "\n"
+}

+func appendJSONLRecord(jsonlFile string, run int, timestamp string, exitCode int, durationMs int64, summary runSummary, failures []testFailure) error {
 	record := jsonlRecord{
 		Run:            run,
 		Timestamp:      timestamp,
@ -161,6 +187,16 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
 	return appendFile(jsonlFile, string(encoded)+"\n")
 }

+func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
+	timestamp := time.Now().UTC().Format(time.RFC3339)
+
+	if err := appendFile(logFile, buildRunLogBlock(run, timestamp, exitCode, summary, failures, durationMs)); err != nil {
+		return err
+	}
+
+	return appendJSONLRecord(jsonlFile, run, timestamp, exitCode, durationMs, summary, failures)
+}
+
 func appendFile(path, content string) error {
 	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
 	if err != nil {
@ -172,6 +208,116 @@ func appendFile(path, content string) error {
 	return err
 }

+func newTestRunState() *testRunState {
+	return &testRunState{
+		failures:      make([]testFailure, 0),
+		failureOutput: make(map[string]*strings.Builder),
+	}
+}
+
+func failureKey(pkg, test string) string {
+	return pkg + "\x00" + test
+}
+
+func (state *testRunState) handlePass(event testEvent) {
+	if event.Test == "" {
+		return
+	}
+	state.summary.Passes++
+}
+
+func (state *testRunState) handleSkip(event testEvent) {
+	if event.Test == "" {
+		return
+	}
+	state.summary.Skipped++
+}
+
+func (state *testRunState) handleFail(event testEvent) {
+	if event.Test == "" {
+		return
+	}
+
+	state.summary.Failures++
+	key := failureKey(event.Package, event.Test)
+	output := ""
+	if builder, ok := state.failureOutput[key]; ok {
+		output = strings.TrimSpace(builder.String())
+	} else {
+		state.failureOutput[key] = &strings.Builder{}
+	}
+
+	state.failures = append(state.failures, testFailure{
+		Package: event.Package,
+		Test:    event.Test,
+		Output:  output,
+	})
+}
+
+func (state *testRunState) handleOutput(event testEvent) {
+	if event.Test == "" {
+		return
+	}
+
+	key := failureKey(event.Package, event.Test)
+	builder, ok := state.failureOutput[key]
+	if !ok {
+		builder = &strings.Builder{}
+		state.failureOutput[key] = builder
+	}
+	builder.WriteString(event.Output)
+}
+
+type testEventHandler func(*testRunState, testEvent)
+
+var testEventHandlers = map[string]testEventHandler{
+	"pass":   (*testRunState).handlePass,
+	"fail":   (*testRunState).handleFail,
+	"skip":   (*testRunState).handleSkip,
+	"output": (*testRunState).handleOutput,
+}
+
+func (state *testRunState) processEvent(event testEvent) {
+	handler, ok := testEventHandlers[event.Action]
+	if !ok {
+		return
+	}
+	handler(state, event)
+}
+
+func (state *testRunState) finalizeFailureOutputs() {
+	for index := range state.failures {
+		key := failureKey(state.failures[index].Package, state.failures[index].Test)
+		if builder, ok := state.failureOutput[key]; ok {
+			state.failures[index].Output = strings.TrimSpace(builder.String())
+		}
+	}
+}
+
+func scanTestEvents(stdout io.Reader, state *testRunState) error {
+	scanner := bufio.NewScanner(stdout)
+	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+	for scanner.Scan() {
+		var event testEvent
+		if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
+			continue
+		}
+		state.processEvent(event)
+	}
+	return scanner.Err()
+}
+
+func finishTestCommand(cmd *exec.Cmd, state *testRunState) (int, runSummary, []testFailure, error) {
+	if err := cmd.Wait(); err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			state.finalizeFailureOutputs()
+			return exitErr.ExitCode(), state.summary, state.failures, nil
+		}
+		return 1, state.summary, state.failures, err
+	}
+	return 0, state.summary, state.failures, nil
+}
+
 func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
 	cmd := exec.Command("go", "test", "./...", "-count=1", "-json")
 	cmd.Dir = rootDir
@ -185,67 +331,57 @@ func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
 		return 1, runSummary{}, nil, err
 	}

-	summary := runSummary{}
-	failures := make([]testFailure, 0)
-	failureOutput := make(map[string]*strings.Builder)
-
-	scanner := bufio.NewScanner(stdout)
-	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
-	for scanner.Scan() {
-		var event testEvent
-		if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
-			continue
-		}
-
-		switch event.Action {
-		case "pass":
-			if event.Test != "" {
-				summary.Passes++
-			}
-		case "fail":
-			if event.Test != "" {
-				summary.Failures++
-				key := event.Package + "\x00" + event.Test
-				failures = append(failures, testFailure{
-					Package: event.Package,
-					Test:    event.Test,
-					Output:  "",
-				})
-				failureOutput[key] = &strings.Builder{}
-			}
-		case "skip":
-			if event.Test != "" {
-				summary.Skipped++
-			}
-		case "output":
-			if event.Test == "" {
-				continue
-			}
-			key := event.Package + "\x00" + event.Test
-			if builder, ok := failureOutput[key]; ok {
-				builder.WriteString(event.Output)
-			}
-		}
+	state := newTestRunState()
+	if err := scanTestEvents(stdout, state); err != nil {
+		return 1, state.summary, state.failures, err
 	}

-	if err := scanner.Err(); err != nil {
-		return 1, summary, failures, err
+	return finishTestCommand(cmd, state)
+}
+
+func buildLogHeader(logFile, jsonlFile string) string {
+	return strings.Join([]string{
+		fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
+		fmt.Sprintf("# Log file: %s", logFile),
+		fmt.Sprintf("# JSONL file: %s", jsonlFile),
+		"",
+	}, "\n") + "\n"
+}
+
+func initRunFiles(logFile, jsonlFile string) error {
+	if err := os.WriteFile(logFile, []byte(buildLogHeader(logFile, jsonlFile)), 0o644); err != nil {
+		return err
+	}
+	return os.WriteFile(jsonlFile, nil, 0o644)
+}
+
+func logRunResult(run, exitCode int, summary runSummary, durationMs int64) {
+	log.Infof(
+		"Run %d: %s | %s (%.1fs) — logged",
+		run,
+		runResultLabel(exitCode),
+		formatRunCounts(summary),
+		float64(durationMs)/1000,
+	)
+}
+
+func executeRun(run int, rootDir, logFile, jsonlFile string) (exitCode int, stop bool) {
+	start := time.Now()
+	exitCode, summary, failures, err := runTestsOnce(rootDir)
+	durationMs := time.Since(start).Milliseconds()
+
+	if err != nil {
+		log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
+		_ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
+		os.Exit(1)
 	}

-	if err := cmd.Wait(); err != nil {
-		if exitErr, ok := err.(*exec.ExitError); ok {
-			for index := range failures {
-				key := failures[index].Package + "\x00" + failures[index].Test
-				if builder, ok := failureOutput[key]; ok {
-					failures[index].Output = strings.TrimSpace(builder.String())
-				}
-			}
-			return exitErr.ExitCode(), summary, failures, nil
-		}
-		return 1, summary, failures, err
+	if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
+		log.WithError(logErr).Fatal("failed to append run log")
 	}

-	return 0, summary, failures, nil
+	logRunResult(run, exitCode, summary, durationMs)
+	return exitCode, exitCode != 0
 }

 func main() {
@ -255,18 +391,8 @@ func main() {
 	logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
 	jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))

-	header := strings.Join([]string{
-		fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
-		fmt.Sprintf("# Log file: %s", logFile),
-		fmt.Sprintf("# JSONL file: %s", jsonlFile),
-		"",
-	}, "\n") + "\n"
-
-	if err := os.WriteFile(logFile, []byte(header), 0o644); err != nil {
-		log.WithError(err).Fatal("failed to initialize log file")
-	}
-	if err := os.WriteFile(jsonlFile, nil, 0o644); err != nil {
-		log.WithError(err).Fatal("failed to initialize jsonl file")
+	if err := initRunFiles(logFile, jsonlFile); err != nil {
+		log.WithError(err).Fatal("failed to initialize output files")
 	}

 	log.Infof("Logging flaky test runs to %s", logFile)
@ -275,28 +401,8 @@ func main() {
 	run := 0
 	for {
 		run++
-
-		start := time.Now()
-		exitCode, summary, failures, err := runTestsOnce(rootDir)
-		durationMs := time.Since(start).Milliseconds()
-
-		if err != nil {
-			log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
-			_ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
-			os.Exit(1)
-		}
-
-		if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
-			log.WithError(logErr).Fatal("failed to append run log")
-		}
-
-		result := "PASS"
-		if exitCode != 0 {
-			result = "FAIL"
-		}
-		log.Infof("Run %d: %s | %s (%.1fs) — logged", run, result, formatRunCounts(summary), float64(durationMs)/1000)
-
-		if exitCode != 0 {
+		exitCode, stop := executeRun(run, rootDir, logFile, jsonlFile)
+		if stop {
 			log.Errorf("Failure on run %d, stopping. See %s", run, logFile)
 			os.Exit(exitCode)
 		}
--- a/var/windows/OliveTin.wxs
+++ b/var/windows/OliveTin.wxs
@ -22,6 +22,7 @@
    <Icon Id="OliveTinIcon" SourceFile="$(var.SourceDir)/OliveTin.exe" />

    <UIRef Id="WixUI_Minimal" />
+    <Property Id="WixUILicenseRtf" Value="License.rtf" />

    <Feature Id="ProductFeature" Title="OliveTin" Level="1">
      <ComponentGroupRef Id="CG.AppFiles" />