From ac0f3ab6f8346157b1b2a47737ef237e0ae3af8f Mon Sep 17 00:00:00 2001 From: James Read Date: Fri, 31 May 2024 21:56:49 +0100 Subject: [PATCH] feature: Live command output! (#325) * feature: Live command output WIP * cicd: Fix live command output build errors * feature: Live command output WIP, remove stdout/stderr split * feature: Live command output * Restore output im stepExecAfter, rename so to ost for OutputStreamer * Update executor.go * feature: Remove output from log message, bad for long messages, binary output, and security * feature: Live command output WIP --- OliveTin.proto | 9 ++++- config.yaml | 8 ++-- internal/executor/executor.go | 70 ++++++++++++++++++--------------- internal/grpcapi/grpcApi.go | 3 +- internal/websocket/websocket.go | 14 ++++++- webui.dev/index.html | 38 ++++++------------ webui.dev/js/ExecutionDialog.js | 52 ++++++++++++------------ webui.dev/js/marshaller.js | 28 +++++++------ webui.dev/js/websocket.js | 1 + webui.dev/package-lock.json | 29 ++++++++++++++ webui.dev/package.json | 8 +++- webui.dev/style.css | 45 +++++++++++++-------- 12 files changed, 184 insertions(+), 121 deletions(-) diff --git a/OliveTin.proto b/OliveTin.proto index 121dc71..2c966e9 100644 --- a/OliveTin.proto +++ b/OliveTin.proto @@ -102,8 +102,7 @@ message GetLogsRequest{}; message LogEntry { string datetime_started = 1; string action_title = 2; - string stdout = 3; - string stderr = 4; + string output = 3; bool timed_out = 5; int32 exit_code = 6; string user = 7; @@ -185,6 +184,12 @@ message GetReadyzResponse { string status = 1; } +message EventOutputChunk { + string execution_tracking_id = 1; + + string output = 2; +} + message EventEntityChanged {} message EventConfigChanged {} message EventExecutionFinished { diff --git a/config.yaml b/config.yaml index 8836d94..33727ca 100644 --- a/config.yaml +++ b/config.yaml @@ -20,8 +20,9 @@ actions: # If you are running OliveTin in a container remember to pass through the # docker socket! https://docs.olivetin.app/action-container-control.html - title: Ping the Internet - shell: ping -c 1 1.1.1.1 + shell: ping -c 3 1.1.1.1 icon: ping + popupOnStart: execution-dialog-stdout-only # This uses `popupOnStart: execution-dialog-stdout-only` to simply show just # the command output. @@ -65,9 +66,10 @@ actions: shell: ping {{ host }} -c {{ count }} icon: ping timeout: 100 + popupOnStart: execution-dialog-stdout-only arguments: - name: host - title: host + title: Host type: ascii_identifier default: example.com description: The host that you want to ping @@ -75,7 +77,7 @@ actions: - name: count title: Count type: int - default: 1 + default: 3 description: How many times to do you want to ping? # OliveTin can control containers - docker is just a command line app. diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 2f572c9..a671ce7 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -14,7 +14,6 @@ import ( "bytes" "context" "fmt" - "io" "os" "os/exec" "path" @@ -76,10 +75,7 @@ type ExecutionRequest struct { type InternalLogEntry struct { DatetimeStarted time.Time DatetimeFinished time.Time - Stdout string - Stderr string - StdoutBuffer io.ReadCloser - StderrBuffer io.ReadCloser + Output string TimedOut bool Blocked bool ExitCode int32 @@ -130,6 +126,7 @@ func DefaultExecutor(cfg *config.Config) *Executor { type listener interface { OnExecutionStarted(actionTitle string) OnExecutionFinished(logEntry *InternalLogEntry) + OnOutputChunk(o []byte, executionTrackingId string) OnActionMapRebuilt() } @@ -148,8 +145,7 @@ func (e *Executor) ExecRequest(req *ExecutionRequest) (*sync.WaitGroup, string) req.logEntry = &InternalLogEntry{ DatetimeStarted: time.Now(), ExecutionTrackingID: req.TrackingID, - Stdout: "", - Stderr: "", + Output: "", ExitCode: -1337, // If an Action is not actually executed, this is the default exit code. ExecutionStarted: false, ExecutionFinished: false, @@ -214,7 +210,7 @@ func stepConcurrencyCheck(req *ExecutionRequest) bool { "actionTitle": req.logEntry.ActionTitle, }).Warnf(msg) - req.logEntry.Stdout = msg + req.logEntry.Output = msg req.logEntry.Blocked = true return false } @@ -262,7 +258,7 @@ func stepRateCheck(req *ExecutionRequest) bool { "actionTitle": req.logEntry.ActionTitle, }).Infof(msg) - req.logEntry.Stdout = msg + req.logEntry.Output = msg req.logEntry.Blocked = true return false } @@ -281,7 +277,7 @@ func stepParseArgs(req *ExecutionRequest) bool { req.finalParsedCommand, err = parseActionArguments(req.Action.Shell, req.Arguments, req.Action, req.logEntry.ActionTitle, req.EntityPrefix) if err != nil { - req.logEntry.Stdout = err.Error() + req.logEntry.Output = err.Error() log.Warnf(err.Error()) @@ -305,7 +301,7 @@ func stepRequestAction(req *ExecutionRequest) bool { "actionTitle": req.ActionTitle, }).Warnf("Action requested, but not found") - req.logEntry.Stderr = "Action not found: " + req.ActionTitle + req.logEntry.Output = "Action not found: " + req.ActionTitle return false } @@ -344,11 +340,10 @@ func stepLogFinish(req *ExecutionRequest) bool { req.logEntry.ExecutionFinished = true log.WithFields(log.Fields{ - "actionTitle": req.logEntry.ActionTitle, - "stdout": req.logEntry.Stdout, - "stderr": req.logEntry.Stderr, - "timedOut": req.logEntry.TimedOut, - "exit": req.logEntry.ExitCode, + "actionTitle": req.logEntry.ActionTitle, + "outputLength": len(req.logEntry.Output), + "timedOut": req.logEntry.TimedOut, + "exit": req.logEntry.ExitCode, }).Infof("Action finished") return true @@ -370,10 +365,27 @@ func wrapCommandInShell(ctx context.Context, finalParsedCommand string) *exec.Cm func appendErrorToStderr(err error, logEntry *InternalLogEntry) { if err != nil { - logEntry.Stderr = err.Error() + "\n\n" + logEntry.Stderr + logEntry.Output = err.Error() + "\n\n" + logEntry.Output } } +type OutputStreamer struct { + Req *ExecutionRequest + output bytes.Buffer +} + +func (ost *OutputStreamer) Write(o []byte) (n int, err error) { + for _, listener := range ost.Req.executor.listeners { + listener.OnOutputChunk(o, ost.Req.TrackingID) + } + + return ost.output.Write(o) +} + +func (ost *OutputStreamer) String() string { + return ost.output.String() +} + func buildEnv(req *ExecutionRequest) []string { ret := append(os.Environ(), "OLIVETIN=1") @@ -388,15 +400,12 @@ func stepExec(req *ExecutionRequest) bool { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(req.Action.Timeout)*time.Second) defer cancel() - var stdout bytes.Buffer - var stderr bytes.Buffer + streamer := &OutputStreamer{Req: req} cmd := wrapCommandInShell(ctx, req.finalParsedCommand) + cmd.Stdout = streamer + cmd.Stderr = streamer cmd.Env = buildEnv(req) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - req.logEntry.StdoutBuffer, _ = cmd.StdoutPipe() - req.logEntry.StderrBuffer, _ = cmd.StderrPipe() req.logEntry.ExecutionStarted = true @@ -407,8 +416,7 @@ func stepExec(req *ExecutionRequest) bool { waiterr := cmd.Wait() req.logEntry.ExitCode = int32(cmd.ProcessState.ExitCode()) - req.logEntry.Stdout = stdout.String() - req.logEntry.Stderr = stderr.String() + req.logEntry.Output = streamer.String() appendErrorToStderr(runerr, req.logEntry) appendErrorToStderr(waiterr, req.logEntry) @@ -435,7 +443,7 @@ func stepExecAfter(req *ExecutionRequest) bool { var stderr bytes.Buffer args := map[string]string{ - "stdout": req.logEntry.Stdout, + "output": req.logEntry.Output, "exitCode": fmt.Sprintf("%v", req.logEntry.ExitCode), } @@ -449,17 +457,17 @@ func stepExecAfter(req *ExecutionRequest) bool { waiterr := cmd.Wait() - req.logEntry.Stdout += "---\n" + stdout.String() - req.logEntry.Stderr += "---\n" + stderr.String() + req.logEntry.Output += "---\n" + stdout.String() + req.logEntry.Output += "---\n" + stderr.String() appendErrorToStderr(runerr, req.logEntry) appendErrorToStderr(waiterr, req.logEntry) if ctx.Err() == context.DeadlineExceeded { - req.logEntry.Stderr += "Your shellAfterCommand command timed out." + req.logEntry.Output += "Your shellAfterCommand command timed out." } - req.logEntry.Stdout += fmt.Sprintf("Your shellAfterCommand exited with code %v", cmd.ProcessState.ExitCode()) + req.logEntry.Output += fmt.Sprintf("Your shellAfterCommand exited with code %v", cmd.ProcessState.ExitCode()) return true } @@ -520,7 +528,7 @@ func saveLogOutput(req *ExecutionRequest, filename string) { dir := firstNonEmpty(req.Action.SaveLogs.OutputDirectory, req.Cfg.SaveLogs.OutputDirectory) if dir != "" { - data := req.logEntry.Stdout + "\n" + req.logEntry.Stderr + data := req.logEntry.Output filepath := path.Join(dir, filename+".log") err := os.WriteFile(filepath, []byte(data), 0644) diff --git a/internal/grpcapi/grpcApi.go b/internal/grpcapi/grpcApi.go index d11d889..0ec0c1e 100644 --- a/internal/grpcapi/grpcApi.go +++ b/internal/grpcapi/grpcApi.go @@ -154,8 +154,7 @@ func internalLogEntryToPb(logEntry *executor.InternalLogEntry) *pb.LogEntry { ActionId: logEntry.ActionId, DatetimeStarted: logEntry.DatetimeStarted.Format("2006-01-02 15:04:05"), DatetimeFinished: logEntry.DatetimeFinished.Format("2006-01-02 15:04:05"), - Stdout: logEntry.Stdout, - Stderr: logEntry.Stderr, + Output: logEntry.Output, TimedOut: logEntry.TimedOut, Blocked: logEntry.Blocked, ExitCode: logEntry.ExitCode, diff --git a/internal/websocket/websocket.go b/internal/websocket/websocket.go index 243d64a..dd8ab3f 100644 --- a/internal/websocket/websocket.go +++ b/internal/websocket/websocket.go @@ -68,6 +68,17 @@ func checkOriginPermissive(r *http.Request) bool { return true } +func (WebsocketExecutionListener) OnOutputChunk(chunk []byte, executionTrackingId string) { + log.Tracef("outputchunk: %s", string(chunk)) + + oc := &pb.EventOutputChunk{ + Output: string(chunk), + ExecutionTrackingId: executionTrackingId, + } + + broadcast(oc) +} + func (WebsocketExecutionListener) OnExecutionFinished(logEntry *executor.InternalLogEntry) { evt := &pb.EventExecutionFinished{ LogEntry: &pb.LogEntry{ @@ -76,8 +87,7 @@ func (WebsocketExecutionListener) OnExecutionFinished(logEntry *executor.Interna ActionId: logEntry.ActionId, DatetimeStarted: logEntry.DatetimeStarted.Format("2006-01-02 15:04:05"), DatetimeFinished: logEntry.DatetimeFinished.Format("2006-01-02 15:04:05"), - Stdout: logEntry.Stdout, - Stderr: logEntry.Stderr, + Output: logEntry.Output, TimedOut: logEntry.TimedOut, Blocked: logEntry.Blocked, ExitCode: logEntry.ExitCode, diff --git a/webui.dev/index.html b/webui.dev/index.html index 00b653b..1a4e317 100644 --- a/webui.dev/index.html +++ b/webui.dev/index.html @@ -2,13 +2,16 @@ - + + OliveTin + + @@ -84,12 +87,12 @@

- + -
+

@@ -98,10 +101,10 @@

-
+
Started: unknown
-
+

Finished: unknown

@@ -114,18 +117,13 @@
-
- Standard Output - 
?
-
- -
- Standard Error - 
?
+
+ Output +
-
+
@@ -136,7 +134,7 @@