olivetin/service/scripts/find-flakey-tests-inf/main.go

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
)

const (
	defaultLogFile   = "flakey-test-runs.log"
	defaultJSONLFile = "flakey-test-runs.jsonl"
)

type testEvent struct {
	Time    time.Time `json:"Time"`
	Action  string    `json:"Action"`
	Package string    `json:"Package"`
	Test    string    `json:"Test"`
	Output  string    `json:"Output"`
	Elapsed float64   `json:"Elapsed"`
}

type testFailure struct {
	Package string `json:"package"`
	Test    string `json:"test"`
	Output  string `json:"output"`
}

type runSummary struct {
	Passes   int `json:"passes"`
	Failures int `json:"failures"`
	Skipped  int `json:"skipped"`
}

type jsonlRecord struct {
	Run            int           `json:"run"`
	Timestamp      string        `json:"timestamp"`
	ExitCode       int           `json:"exitCode"`
	DurationMs     int64         `json:"durationMs"`
	Passes         int           `json:"passes"`
	Failures       int           `json:"failures"`
	Skipped        int           `json:"skipped"`
	FailureDetails []testFailure `json:"failureDetails"`
}

func initLog() {
	logFormat := os.Getenv("OLIVETIN_LOG_FORMAT")

	if logFormat == "json" {
		log.SetFormatter(&log.JSONFormatter{})
	} else {
		log.SetFormatter(&log.TextFormatter{
			FullTimestamp: true,
			ForceQuote:    true,
		})
	}

	log.SetLevel(log.InfoLevel)
}

func serviceRoot() string {
	exe, err := os.Executable()
	if err == nil {
		candidate := filepath.Join(filepath.Dir(exe), "..", "..")
		if stat, statErr := os.Stat(filepath.Join(candidate, "go.mod")); statErr == nil && !stat.IsDir() {
			return candidate
		}
	}

	wd, err := os.Getwd()
	if err != nil {
		return "."
	}

	if stat, statErr := os.Stat(filepath.Join(wd, "go.mod")); statErr == nil && !stat.IsDir() {
		return wd
	}

	return filepath.Join(wd, "..")
}

func envOrDefault(name, fallback string) string {
	if value := strings.TrimSpace(os.Getenv(name)); value != "" {
		return value
	}
	return fallback
}

func formatFailure(failure testFailure) string {
	lines := []string{
		fmt.Sprintf("FAILURE: %s", failure.Test),
		fmt.Sprintf("  package: %s", failure.Package),
	}

	output := strings.TrimSpace(failure.Output)
	if output == "" {
		output = "unknown"
	}
	lines = append(lines, fmt.Sprintf("  output: %s", output))

	return strings.Join(lines, "\n")
}

func formatRunCounts(summary runSummary) string {
	return fmt.Sprintf("%d pass %d fail %d skip", summary.Passes, summary.Failures, summary.Skipped)
}

func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
	timestamp := time.Now().UTC().Format(time.RFC3339)
	passed := exitCode == 0
	result := "PASS"
	if !passed {
		result = "FAIL"
	}

	block := []string{
		fmt.Sprintf(
			"=== RUN %d | %s | %s | %d pass %d fail %d skip | %.1fs ===",
			run,
			timestamp,
			result,
			summary.Passes,
			summary.Failures,
			summary.Skipped,
			float64(durationMs)/1000,
		),
	}

	for _, failure := range failures {
		block = append(block, formatFailure(failure))
	}

	block = append(block, "")
	if err := appendFile(logFile, strings.Join(block, "\n")+"\n"); err != nil {
		return err
	}

	record := jsonlRecord{
		Run:            run,
		Timestamp:      timestamp,
		ExitCode:       exitCode,
		DurationMs:     durationMs,
		Passes:         summary.Passes,
		Failures:       summary.Failures,
		Skipped:        summary.Skipped,
		FailureDetails: failures,
	}

	encoded, err := json.Marshal(record)
	if err != nil {
		return err
	}

	return appendFile(jsonlFile, string(encoded)+"\n")
}

func appendFile(path, content string) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.WriteString(content)
	return err
}

func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
	cmd := exec.Command("go", "test", "./...", "-count=1", "-json")
	cmd.Dir = rootDir

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return 1, runSummary{}, nil, err
	}

	if err := cmd.Start(); err != nil {
		return 1, runSummary{}, nil, err
	}

	summary := runSummary{}
	failures := make([]testFailure, 0)
	failureOutput := make(map[string]*strings.Builder)

	scanner := bufio.NewScanner(stdout)
	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	for scanner.Scan() {
		var event testEvent
		if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
			continue
		}

		switch event.Action {
		case "pass":
			if event.Test != "" {
				summary.Passes++
			}
		case "fail":
			if event.Test != "" {
				summary.Failures++
				key := event.Package + "\x00" + event.Test
				failures = append(failures, testFailure{
					Package: event.Package,
					Test:    event.Test,
					Output:  "",
				})
				failureOutput[key] = &strings.Builder{}
			}
		case "skip":
			if event.Test != "" {
				summary.Skipped++
			}
		case "output":
			if event.Test == "" {
				continue
			}
			key := event.Package + "\x00" + event.Test
			if builder, ok := failureOutput[key]; ok {
				builder.WriteString(event.Output)
			}
		}
	}

	if err := scanner.Err(); err != nil {
		return 1, summary, failures, err
	}

	if err := cmd.Wait(); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			for index := range failures {
				key := failures[index].Package + "\x00" + failures[index].Test
				if builder, ok := failureOutput[key]; ok {
					failures[index].Output = strings.TrimSpace(builder.String())
				}
			}
			return exitErr.ExitCode(), summary, failures, nil
		}
		return 1, summary, failures, err
	}

	return 0, summary, failures, nil
}

func main() {
	initLog()

	rootDir := serviceRoot()
	logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
	jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))

	header := strings.Join([]string{
		fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
		fmt.Sprintf("# Log file: %s", logFile),
		fmt.Sprintf("# JSONL file: %s", jsonlFile),
		"",
	}, "\n") + "\n"

	if err := os.WriteFile(logFile, []byte(header), 0o644); err != nil {
		log.WithError(err).Fatal("failed to initialize log file")
	}
	if err := os.WriteFile(jsonlFile, nil, 0o644); err != nil {
		log.WithError(err).Fatal("failed to initialize jsonl file")
	}

	log.Infof("Logging flaky test runs to %s", logFile)
	log.Infof("Structured run data: %s", jsonlFile)

	run := 0
	for {
		run++

		start := time.Now()
		exitCode, summary, failures, err := runTestsOnce(rootDir)
		durationMs := time.Since(start).Milliseconds()

		if err != nil {
			log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
			_ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
			os.Exit(1)
		}

		if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
			log.WithError(logErr).Fatal("failed to append run log")
		}

		result := "PASS"
		if exitCode != 0 {
			result = "FAIL"
		}
		log.Infof("Run %d: %s | %s (%.1fs) — logged", run, result, formatRunCounts(summary), float64(durationMs)/1000)

		if exitCode != 0 {
			log.Errorf("Failure on run %d, stopping. See %s", run, logFile)
			os.Exit(exitCode)
		}
	}
}