medlogic
/
olivetin
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: flakey test finder

This commit is contained in:
jamesread 2026-06-29 20:40:05 +01:00
parent 1cbd1cfb6b
commit ba7d9f51a1
4 changed files with 219 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -27,4 +27,5 @@ webui.dev
sessions.yaml
docs/build/
build/
Binary/
.cursor

5
service/scripts/find-flakey-tests-inf/Makefile Normal file
View File

@ -0,0 +1,5 @@
codestyle:
go fmt ./...
go vet ./...
gocyclo -over 4 .
gocritic check ./...

308
service/scripts/find-flakey-tests-inf/main.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
@ -50,6 +51,12 @@ type jsonlRecord struct {
FailureDetails []testFailure `json:"failureDetails"`
}
type testRunState struct {
summary runSummary
failures []testFailure
failureOutput map[string]*strings.Builder
}
func initLog() {
logFormat := os.Getenv("OLIVETIN_LOG_FORMAT")
@ -65,27 +72,46 @@ func initLog() {
log.SetLevel(log.InfoLevel)
}
func serviceRoot() string {
func hasGoMod(dir string) bool {
stat, err := os.Stat(filepath.Join(dir, "go.mod"))
return err == nil && !stat.IsDir()
}
func rootFromExecutable() (string, bool) {
exe, err := os.Executable()
if err == nil {
candidate := filepath.Join(filepath.Dir(exe), "..", "..")
if stat, statErr := os.Stat(filepath.Join(candidate, "go.mod")); statErr == nil && !stat.IsDir() {
return candidate
}
if err != nil {
return "", false
}
candidate := filepath.Join(filepath.Dir(exe), "..", "..")
if hasGoMod(candidate) {
return candidate, true
}
return "", false
}
func rootFromWorkingDir() string {
wd, err := os.Getwd()
if err != nil {
return "."
}
if stat, statErr := os.Stat(filepath.Join(wd, "go.mod")); statErr == nil && !stat.IsDir() {
if hasGoMod(wd) {
return wd
}
return filepath.Join(wd, "..")
}
func serviceRoot() string {
if root, ok := rootFromExecutable(); ok {
return root
}
return rootFromWorkingDir()
}
func envOrDefault(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
@ -112,20 +138,20 @@ func formatRunCounts(summary runSummary) string {
return fmt.Sprintf("%d pass %d fail %d skip", summary.Passes, summary.Failures, summary.Skipped)
}
func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
timestamp := time.Now().UTC().Format(time.RFC3339)
passed := exitCode == 0
result := "PASS"
if !passed {
result = "FAIL"
func runResultLabel(exitCode int) string {
if exitCode == 0 {
return "PASS"
}
return "FAIL"
}
func buildRunLogBlock(run int, timestamp string, exitCode int, summary runSummary, failures []testFailure, durationMs int64) string {
block := []string{
fmt.Sprintf(
"=== RUN %d | %s | %s | %d pass %d fail %d skip | %.1fs ===",
run,
timestamp,
result,
runResultLabel(exitCode),
summary.Passes,
summary.Failures,
summary.Skipped,
@ -138,10 +164,10 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
}
block = append(block, "")
if err := appendFile(logFile, strings.Join(block, "\n")+"\n"); err != nil {
return err
}
return strings.Join(block, "\n") + "\n"
}
func appendJSONLRecord(jsonlFile string, run int, timestamp string, exitCode int, durationMs int64, summary runSummary, failures []testFailure) error {
record := jsonlRecord{
Run: run,
Timestamp: timestamp,
@ -161,6 +187,16 @@ func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSumma
return appendFile(jsonlFile, string(encoded)+"\n")
}
func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
timestamp := time.Now().UTC().Format(time.RFC3339)
if err := appendFile(logFile, buildRunLogBlock(run, timestamp, exitCode, summary, failures, durationMs)); err != nil {
return err
}
return appendJSONLRecord(jsonlFile, run, timestamp, exitCode, durationMs, summary, failures)
}
func appendFile(path, content string) error {
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
@ -172,6 +208,116 @@ func appendFile(path, content string) error {
return err
}
func newTestRunState() *testRunState {
return &testRunState{
failures: make([]testFailure, 0),
failureOutput: make(map[string]*strings.Builder),
}
}
func failureKey(pkg, test string) string {
return pkg + "\x00" + test
}
func (state *testRunState) handlePass(event testEvent) {
if event.Test == "" {
return
}
state.summary.Passes++
}
func (state *testRunState) handleSkip(event testEvent) {
if event.Test == "" {
return
}
state.summary.Skipped++
}
func (state *testRunState) handleFail(event testEvent) {
if event.Test == "" {
return
}
state.summary.Failures++
key := failureKey(event.Package, event.Test)
output := ""
if builder, ok := state.failureOutput[key]; ok {
output = strings.TrimSpace(builder.String())
} else {
state.failureOutput[key] = &strings.Builder{}
}
state.failures = append(state.failures, testFailure{
Package: event.Package,
Test: event.Test,
Output: output,
})
}
func (state *testRunState) handleOutput(event testEvent) {
if event.Test == "" {
return
}
key := failureKey(event.Package, event.Test)
builder, ok := state.failureOutput[key]
if !ok {
builder = &strings.Builder{}
state.failureOutput[key] = builder
}
builder.WriteString(event.Output)
}
type testEventHandler func(*testRunState, testEvent)
var testEventHandlers = map[string]testEventHandler{
"pass": (*testRunState).handlePass,
"fail": (*testRunState).handleFail,
"skip": (*testRunState).handleSkip,
"output": (*testRunState).handleOutput,
}
func (state *testRunState) processEvent(event testEvent) {
handler, ok := testEventHandlers[event.Action]
if !ok {
return
}
handler(state, event)
}
func (state *testRunState) finalizeFailureOutputs() {
for index := range state.failures {
key := failureKey(state.failures[index].Package, state.failures[index].Test)
if builder, ok := state.failureOutput[key]; ok {
state.failures[index].Output = strings.TrimSpace(builder.String())
}
}
}
func scanTestEvents(stdout io.Reader, state *testRunState) error {
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for scanner.Scan() {
var event testEvent
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
continue
}
state.processEvent(event)
}
return scanner.Err()
}
func finishTestCommand(cmd *exec.Cmd, state *testRunState) (int, runSummary, []testFailure, error) {
if err := cmd.Wait(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
state.finalizeFailureOutputs()
return exitErr.ExitCode(), state.summary, state.failures, nil
}
return 1, state.summary, state.failures, err
}
return 0, state.summary, state.failures, nil
}
func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
cmd := exec.Command("go", "test", "./...", "-count=1", "-json")
cmd.Dir = rootDir
@ -185,97 +331,41 @@ func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
return 1, runSummary{}, nil, err
}
summary := runSummary{}
failures := make([]testFailure, 0)
failureOutput := make(map[string]*strings.Builder)
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for scanner.Scan() {
var event testEvent
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
continue
state := newTestRunState()
if err := scanTestEvents(stdout, state); err != nil {
return 1, state.summary, state.failures, err
}
switch event.Action {
case "pass":
if event.Test != "" {
summary.Passes++
}
case "fail":
if event.Test != "" {
summary.Failures++
key := event.Package + "\x00" + event.Test
failures = append(failures, testFailure{
Package: event.Package,
Test: event.Test,
Output: "",
})
failureOutput[key] = &strings.Builder{}
}
case "skip":
if event.Test != "" {
summary.Skipped++
}
case "output":
if event.Test == "" {
continue
}
key := event.Package + "\x00" + event.Test
if builder, ok := failureOutput[key]; ok {
builder.WriteString(event.Output)
}
}
}
if err := scanner.Err(); err != nil {
return 1, summary, failures, err
}
if err := cmd.Wait(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
for index := range failures {
key := failures[index].Package + "\x00" + failures[index].Test
if builder, ok := failureOutput[key]; ok {
failures[index].Output = strings.TrimSpace(builder.String())
}
}
return exitErr.ExitCode(), summary, failures, nil
}
return 1, summary, failures, err
}
return 0, summary, failures, nil
return finishTestCommand(cmd, state)
}
func main() {
initLog()
rootDir := serviceRoot()
logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))
header := strings.Join([]string{
func buildLogHeader(logFile, jsonlFile string) string {
return strings.Join([]string{
fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
fmt.Sprintf("# Log file: %s", logFile),
fmt.Sprintf("# JSONL file: %s", jsonlFile),
"",
}, "\n") + "\n"
}
if err := os.WriteFile(logFile, []byte(header), 0o644); err != nil {
log.WithError(err).Fatal("failed to initialize log file")
}
if err := os.WriteFile(jsonlFile, nil, 0o644); err != nil {
log.WithError(err).Fatal("failed to initialize jsonl file")
func initRunFiles(logFile, jsonlFile string) error {
if err := os.WriteFile(logFile, []byte(buildLogHeader(logFile, jsonlFile)), 0o644); err != nil {
return err
}
return os.WriteFile(jsonlFile, nil, 0o644)
}
log.Infof("Logging flaky test runs to %s", logFile)
log.Infof("Structured run data: %s", jsonlFile)
run := 0
for {
run++
func logRunResult(run, exitCode int, summary runSummary, durationMs int64) {
log.Infof(
"Run %d: %s | %s (%.1fs) — logged",
run,
runResultLabel(exitCode),
formatRunCounts(summary),
float64(durationMs)/1000,
)
}
func executeRun(run int, rootDir, logFile, jsonlFile string) (exitCode int, stop bool) {
start := time.Now()
exitCode, summary, failures, err := runTestsOnce(rootDir)
durationMs := time.Since(start).Milliseconds()
@ -290,13 +380,29 @@ func main() {
log.WithError(logErr).Fatal("failed to append run log")
}
result := "PASS"
if exitCode != 0 {
result = "FAIL"
}
log.Infof("Run %d: %s | %s (%.1fs) — logged", run, result, formatRunCounts(summary), float64(durationMs)/1000)
logRunResult(run, exitCode, summary, durationMs)
return exitCode, exitCode != 0
}
if exitCode != 0 {
func main() {
initLog()
rootDir := serviceRoot()
logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))
if err := initRunFiles(logFile, jsonlFile); err != nil {
log.WithError(err).Fatal("failed to initialize output files")
}
log.Infof("Logging flaky test runs to %s", logFile)
log.Infof("Structured run data: %s", jsonlFile)
run := 0
for {
run++
exitCode, stop := executeRun(run, rootDir, logFile, jsonlFile)
if stop {
log.Errorf("Failure on run %d, stopping. See %s", run, logFile)
os.Exit(exitCode)
}

1
var/windows/OliveTin.wxs
View File

@ -22,6 +22,7 @@
<Icon Id="OliveTinIcon" SourceFile="$(var.SourceDir)/OliveTin.exe" />
<UIRef Id="WixUI_Minimal" />
<Property Id="WixUILicenseRtf" Value="License.rtf" />
<Feature Id="ProductFeature" Title="OliveTin" Level="1">
<ComponentGroupRef Id="CG.AppFiles" />