diff --git a/.gitignore b/.gitignore
index 6556b58..ad372af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,4 +27,5 @@ webui.dev
sessions.yaml
docs/build/
build/
+Binary/
.cursor
diff --git a/service/scripts/find-flakey-tests-inf/Makefile b/service/scripts/find-flakey-tests-inf/Makefile
new file mode 100644
index 0000000..33ff0cc
--- /dev/null
+++ b/service/scripts/find-flakey-tests-inf/Makefile
@@ -0,0 +1,5 @@
+codestyle:
+ go fmt ./...
+ go vet ./...
+ gocyclo -over 4 .
+ gocritic check ./...
diff --git a/service/scripts/find-flakey-tests-inf/main.go b/service/scripts/find-flakey-tests-inf/main.go
index 5c7e67a..b343c30 100644
--- a/service/scripts/find-flakey-tests-inf/main.go
+++ b/service/scripts/find-flakey-tests-inf/main.go
@@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"fmt"
+ "io"
"os"
"os/exec"
"path/filepath"
@@ -50,6 +51,12 @@ type jsonlRecord struct {
FailureDetails []testFailure `json:"failureDetails"`
}
+type testRunState struct {
+ summary runSummary
+ failures []testFailure
+ failureOutput map[string]*strings.Builder
+}
+
func initLog() {
logFormat := os.Getenv("OLIVETIN_LOG_FORMAT")
@@ -65,27 +72,46 @@ func initLog() {
log.SetLevel(log.InfoLevel)
}
-func serviceRoot() string {
+func hasGoMod(dir string) bool {
+ stat, err := os.Stat(filepath.Join(dir, "go.mod"))
+ return err == nil && !stat.IsDir()
+}
+
+func rootFromExecutable() (string, bool) {
exe, err := os.Executable()
- if err == nil {
- candidate := filepath.Join(filepath.Dir(exe), "..", "..")
- if stat, statErr := os.Stat(filepath.Join(candidate, "go.mod")); statErr == nil && !stat.IsDir() {
- return candidate
- }
+ if err != nil {
+ return "", false
}
+ candidate := filepath.Join(filepath.Dir(exe), "..", "..")
+ if hasGoMod(candidate) {
+ return candidate, true
+ }
+
+ return "", false
+}
+
+func rootFromWorkingDir() string {
wd, err := os.Getwd()
if err != nil {
return "."
}
- if stat, statErr := os.Stat(filepath.Join(wd, "go.mod")); statErr == nil && !stat.IsDir() {
+ if hasGoMod(wd) {
return wd
}
return filepath.Join(wd, "..")
}
+func serviceRoot() string {
+ if root, ok := rootFromExecutable(); ok {
+ return root
+ }
+
+ return rootFromWorkingDir()
+}
+
func envOrDefault(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
@@ -112,20 +138,20 @@ func formatRunCounts(summary runSummary) string {
return fmt.Sprintf("%d pass %d fail %d skip", summary.Passes, summary.Failures, summary.Skipped)
}
-func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
- timestamp := time.Now().UTC().Format(time.RFC3339)
- passed := exitCode == 0
- result := "PASS"
- if !passed {
- result = "FAIL"
+func runResultLabel(exitCode int) string {
+ if exitCode == 0 {
+ return "PASS"
}
+ return "FAIL"
+}
+func buildRunLogBlock(run int, timestamp string, exitCode int, summary runSummary, failures []testFailure, durationMs int64) string {
block := []string{
fmt.Sprintf(
"=== RUN %d | %s | %s | %d pass %d fail %d skip | %.1fs ===",
run,
timestamp,
- result,
+ runResultLabel(exitCode),
summary.Passes,
summary.Failures,
summary.Skipped,
@@ -138,10 +164,10 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
}
block = append(block, "")
- if err := appendFile(logFile, strings.Join(block, "\n")+"\n"); err != nil {
- return err
- }
+ return strings.Join(block, "\n") + "\n"
+}
+func appendJSONLRecord(jsonlFile string, run int, timestamp string, exitCode int, durationMs int64, summary runSummary, failures []testFailure) error {
record := jsonlRecord{
Run: run,
Timestamp: timestamp,
@@ -161,6 +187,16 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
return appendFile(jsonlFile, string(encoded)+"\n")
}
+func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
+ timestamp := time.Now().UTC().Format(time.RFC3339)
+
+ if err := appendFile(logFile, buildRunLogBlock(run, timestamp, exitCode, summary, failures, durationMs)); err != nil {
+ return err
+ }
+
+ return appendJSONLRecord(jsonlFile, run, timestamp, exitCode, durationMs, summary, failures)
+}
+
func appendFile(path, content string) error {
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
@@ -172,6 +208,116 @@ func appendFile(path, content string) error {
return err
}
+func newTestRunState() *testRunState {
+ return &testRunState{
+ failures: make([]testFailure, 0),
+ failureOutput: make(map[string]*strings.Builder),
+ }
+}
+
+func failureKey(pkg, test string) string {
+ return pkg + "\x00" + test
+}
+
+func (state *testRunState) handlePass(event testEvent) {
+ if event.Test == "" {
+ return
+ }
+ state.summary.Passes++
+}
+
+func (state *testRunState) handleSkip(event testEvent) {
+ if event.Test == "" {
+ return
+ }
+ state.summary.Skipped++
+}
+
+func (state *testRunState) handleFail(event testEvent) {
+ if event.Test == "" {
+ return
+ }
+
+ state.summary.Failures++
+ key := failureKey(event.Package, event.Test)
+ output := ""
+ if builder, ok := state.failureOutput[key]; ok {
+ output = strings.TrimSpace(builder.String())
+ } else {
+ state.failureOutput[key] = &strings.Builder{}
+ }
+
+ state.failures = append(state.failures, testFailure{
+ Package: event.Package,
+ Test: event.Test,
+ Output: output,
+ })
+}
+
+func (state *testRunState) handleOutput(event testEvent) {
+ if event.Test == "" {
+ return
+ }
+
+ key := failureKey(event.Package, event.Test)
+ builder, ok := state.failureOutput[key]
+ if !ok {
+ builder = &strings.Builder{}
+ state.failureOutput[key] = builder
+ }
+ builder.WriteString(event.Output)
+}
+
+type testEventHandler func(*testRunState, testEvent)
+
+var testEventHandlers = map[string]testEventHandler{
+ "pass": (*testRunState).handlePass,
+ "fail": (*testRunState).handleFail,
+ "skip": (*testRunState).handleSkip,
+ "output": (*testRunState).handleOutput,
+}
+
+func (state *testRunState) processEvent(event testEvent) {
+ handler, ok := testEventHandlers[event.Action]
+ if !ok {
+ return
+ }
+ handler(state, event)
+}
+
+func (state *testRunState) finalizeFailureOutputs() {
+ for index := range state.failures {
+ key := failureKey(state.failures[index].Package, state.failures[index].Test)
+ if builder, ok := state.failureOutput[key]; ok {
+ state.failures[index].Output = strings.TrimSpace(builder.String())
+ }
+ }
+}
+
+func scanTestEvents(stdout io.Reader, state *testRunState) error {
+ scanner := bufio.NewScanner(stdout)
+ scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+ for scanner.Scan() {
+ var event testEvent
+ if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
+ continue
+ }
+ state.processEvent(event)
+ }
+ return scanner.Err()
+}
+
+func finishTestCommand(cmd *exec.Cmd, state *testRunState) (int, runSummary, []testFailure, error) {
+ if err := cmd.Wait(); err != nil {
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ state.finalizeFailureOutputs()
+ return exitErr.ExitCode(), state.summary, state.failures, nil
+ }
+ return 1, state.summary, state.failures, err
+ }
+ return 0, state.summary, state.failures, nil
+}
+
func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
cmd := exec.Command("go", "test", "./...", "-count=1", "-json")
cmd.Dir = rootDir
@@ -185,67 +331,57 @@ func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
return 1, runSummary{}, nil, err
}
- summary := runSummary{}
- failures := make([]testFailure, 0)
- failureOutput := make(map[string]*strings.Builder)
-
- scanner := bufio.NewScanner(stdout)
- scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
- for scanner.Scan() {
- var event testEvent
- if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
- continue
- }
-
- switch event.Action {
- case "pass":
- if event.Test != "" {
- summary.Passes++
- }
- case "fail":
- if event.Test != "" {
- summary.Failures++
- key := event.Package + "\x00" + event.Test
- failures = append(failures, testFailure{
- Package: event.Package,
- Test: event.Test,
- Output: "",
- })
- failureOutput[key] = &strings.Builder{}
- }
- case "skip":
- if event.Test != "" {
- summary.Skipped++
- }
- case "output":
- if event.Test == "" {
- continue
- }
- key := event.Package + "\x00" + event.Test
- if builder, ok := failureOutput[key]; ok {
- builder.WriteString(event.Output)
- }
- }
+ state := newTestRunState()
+ if err := scanTestEvents(stdout, state); err != nil {
+ return 1, state.summary, state.failures, err
}
- if err := scanner.Err(); err != nil {
- return 1, summary, failures, err
+ return finishTestCommand(cmd, state)
+}
+
+func buildLogHeader(logFile, jsonlFile string) string {
+ return strings.Join([]string{
+ fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
+ fmt.Sprintf("# Log file: %s", logFile),
+ fmt.Sprintf("# JSONL file: %s", jsonlFile),
+ "",
+ }, "\n") + "\n"
+}
+
+func initRunFiles(logFile, jsonlFile string) error {
+ if err := os.WriteFile(logFile, []byte(buildLogHeader(logFile, jsonlFile)), 0o644); err != nil {
+ return err
+ }
+ return os.WriteFile(jsonlFile, nil, 0o644)
+}
+
+func logRunResult(run, exitCode int, summary runSummary, durationMs int64) {
+ log.Infof(
+ "Run %d: %s | %s (%.1fs) — logged",
+ run,
+ runResultLabel(exitCode),
+ formatRunCounts(summary),
+ float64(durationMs)/1000,
+ )
+}
+
+func executeRun(run int, rootDir, logFile, jsonlFile string) (exitCode int, stop bool) {
+ start := time.Now()
+ exitCode, summary, failures, err := runTestsOnce(rootDir)
+ durationMs := time.Since(start).Milliseconds()
+
+ if err != nil {
+ log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
+ _ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
+ os.Exit(1)
}
- if err := cmd.Wait(); err != nil {
- if exitErr, ok := err.(*exec.ExitError); ok {
- for index := range failures {
- key := failures[index].Package + "\x00" + failures[index].Test
- if builder, ok := failureOutput[key]; ok {
- failures[index].Output = strings.TrimSpace(builder.String())
- }
- }
- return exitErr.ExitCode(), summary, failures, nil
- }
- return 1, summary, failures, err
+ if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
+ log.WithError(logErr).Fatal("failed to append run log")
}
- return 0, summary, failures, nil
+ logRunResult(run, exitCode, summary, durationMs)
+ return exitCode, exitCode != 0
}
func main() {
@@ -255,18 +391,8 @@ func main() {
logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))
- header := strings.Join([]string{
- fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
- fmt.Sprintf("# Log file: %s", logFile),
- fmt.Sprintf("# JSONL file: %s", jsonlFile),
- "",
- }, "\n") + "\n"
-
- if err := os.WriteFile(logFile, []byte(header), 0o644); err != nil {
- log.WithError(err).Fatal("failed to initialize log file")
- }
- if err := os.WriteFile(jsonlFile, nil, 0o644); err != nil {
- log.WithError(err).Fatal("failed to initialize jsonl file")
+ if err := initRunFiles(logFile, jsonlFile); err != nil {
+ log.WithError(err).Fatal("failed to initialize output files")
}
log.Infof("Logging flaky test runs to %s", logFile)
@@ -275,28 +401,8 @@ func main() {
run := 0
for {
run++
-
- start := time.Now()
- exitCode, summary, failures, err := runTestsOnce(rootDir)
- durationMs := time.Since(start).Milliseconds()
-
- if err != nil {
- log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
- _ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
- os.Exit(1)
- }
-
- if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
- log.WithError(logErr).Fatal("failed to append run log")
- }
-
- result := "PASS"
- if exitCode != 0 {
- result = "FAIL"
- }
- log.Infof("Run %d: %s | %s (%.1fs) — logged", run, result, formatRunCounts(summary), float64(durationMs)/1000)
-
- if exitCode != 0 {
+ exitCode, stop := executeRun(run, rootDir, logFile, jsonlFile)
+ if stop {
log.Errorf("Failure on run %d, stopping. See %s", run, logFile)
os.Exit(exitCode)
}
diff --git a/var/windows/OliveTin.wxs b/var/windows/OliveTin.wxs
index 15e6184..7c264b7 100644
--- a/var/windows/OliveTin.wxs
+++ b/var/windows/OliveTin.wxs
@@ -22,6 +22,7 @@
+