From c13fe03168f971ae3468735034b21753c5b88d20 Mon Sep 17 00:00:00 2001 From: lvladika Date: Sun, 15 Nov 2026 20:55:03 +0100 Subject: [PATCH 1/2] feat(report): a runFlow's steps are in the report as they run A runFlow's, repeat's or retry's steps were written to the flow's file only when it ended, so a reader of the report (a live view of the run) saw a one-line runFlow for its whole run and then all its steps at once. Each nested step is now written as it starts (running) and as it ends, with the compound steps running around it, through FlowWriter.CommandSubs; the list written at the step's end (CommandEndWithSubs) is unchanged. --- pkg/executor/flow_runner.go | 83 ++++++++++++++++++++++++++++++ pkg/executor/nested_report_test.go | 74 ++++++++++++++++++++++++++ pkg/report/flow.go | 13 +++++ 3 files changed, 170 insertions(+) diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 16cd0e01..f99d3fa1 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "slices" "strconv" "strings" "time" @@ -40,6 +41,9 @@ type FlowRunner struct { conditionSkipped bool // Sub-command tracking for compound steps (runFlow, repeat, retry) subCommands []report.Command + // live is the top-level compound step that runs now, whose steps go to + // the report as they start and end (liveSubs); nil between them. + live *liveSteps // nestedArtifactSeq numbers failure artifacts captured for nested steps // (runFlow / repeat / retry), which have no top-level command index. nestedArtifactSeq int @@ -679,7 +683,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin if err := fr.script.ExpandStep(step); err != nil { result = expressionFailed(err) } else { + switch step.(type) { + case *flow.RepeatStep, *flow.RetryStep, *flow.RunFlowStep: + fr.live = &liveSteps{cmd: idx} + } result = fr.runStep(idx, step, &artifacts) + fr.live = nil } result = fr.maskTypedResult(step, result) @@ -1838,6 +1847,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { // inside a runFlow showed the password itself. describe, desc := fr.stepNames(step) + // The step as it starts, in the report while it runs (liveSubs). + running := report.Command{ + ID: fmt.Sprintf("sub-%d", len(fr.subCommands)), + Index: len(fr.subCommands), + Type: string(step.Type()), + Label: step.Label(), + YAML: describe, + Status: report.StatusRunning, + StartTime: &start, + } + // For nested compound steps, we need to track their sub-commands separately var nestedSubCommands, parentSubCommands []report.Command isCompoundStep := false @@ -1848,6 +1868,8 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { parentSubCommands = fr.subCommands fr.subCommands = nil } + fr.live.begin(isCompoundStep, parentSubCommands, running) + fr.liveSubs() // Expand once, before the handler, as executeStep does: a ${...} that // throws fails the step and nothing runs. @@ -1900,6 +1922,7 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { nestedSubCommands = fr.subCommands fr.subCommands = parentSubCommands } + fr.live.end(isCompoundStep) // Add to parent's sub-commands for report status := report.StatusPassed @@ -1952,10 +1975,70 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { } fr.subCommands = append(fr.subCommands, cmd) + fr.liveSubs() return result } +// liveSteps is a top-level runFlow, repeat or retry while it runs: the +// compound steps running inside it, outermost first, each with the list it +// goes into once it ends, and the step running in the innermost one. +type liveSteps struct { + cmd int + frames []liveFrame + step *report.Command +} + +type liveFrame struct { + parent []report.Command + cmd report.Command +} + +// begin records a nested step that starts: a compound one with the list it +// goes into (its parent's), another as the step running. +func (l *liveSteps) begin(compound bool, parent []report.Command, c report.Command) { + if l == nil { + return + } + if compound { + l.frames = append(l.frames, liveFrame{parent: parent, cmd: c}) + } else { + l.step = &c + } +} + +// end forgets the nested step begin recorded last, which ended. +func (l *liveSteps) end(compound bool) { + if l == nil { + return + } + if compound { + l.frames = l.frames[:len(l.frames)-1] + } else { + l.step = nil + } +} + +// liveSubs writes the steps of the top-level compound step that runs now to +// the report as far as they got: those that ended and those running, so a +// runFlow's steps can be followed while it runs and not only once it ended. +func (fr *FlowRunner) liveSubs() { + l := fr.live + if l == nil || fr.flowWriter == nil { + return + } + list := slices.Clone(fr.subCommands) + if l.step != nil { + list = append(list, *l.step) + } + for i := len(l.frames) - 1; i >= 0; i-- { + c := l.frames[i].cmd + c.SubCommands = list + list = append(slices.Clone(l.frames[i].parent), c) + } + fr.flowWriter.CommandSubs(l.cmd, list) +} + // executeSubFlow executes a sub-flow without separate report tracking. func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult { // Save current flow dir diff --git a/pkg/executor/nested_report_test.go b/pkg/executor/nested_report_test.go index 6031c260..0c2feafb 100644 --- a/pkg/executor/nested_report_test.go +++ b/pkg/executor/nested_report_test.go @@ -3,8 +3,10 @@ package executor import ( "context" "path/filepath" + "strings" "testing" + "github.com/devicelab-dev/maestro-runner/pkg/core" "github.com/devicelab-dev/maestro-runner/pkg/flow" "github.com/devicelab-dev/maestro-runner/pkg/report" ) @@ -104,3 +106,75 @@ func TestConsoleLogsAreReportedPerStep(t *testing.T) { t.Errorf("runFlow logs %q, tapOn logs %q: want none", cmds[1].Logs, cmds[2].Logs) } } + +// A runFlow's steps are in the report while they run, each as it starts and +// as it ends, the nested runFlow's too: they used to arrive with its end. +func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) { + dir := t.TempDir() + var shape func(cs []report.Command) string + shape = func(cs []report.Command) string { + var out []string + for _, c := range cs { + s := c.Type + ":" + string(c.Status) + if len(c.SubCommands) > 0 { + s += "[" + shape(c.SubCommands) + "]" + } + out = append(out, s) + } + return strings.Join(out, ",") + } + seen := map[string]string{} // a tap's id → its runFlow's steps in the report as it ran + driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult { + if tap, ok := step.(*flow.TapOnStep); ok { + d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + if err != nil { + t.Errorf("read the report while %s runs: %v", tap.Selector.ID, err) + } else { + seen[tap.Selector.ID] = shape(d.Commands[0].SubCommands) + } + } + return &core.CommandResult{Success: true} + }} + tap := func(id string) *flow.TapOnStep { + return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}} + } + runner := New(driver, RunnerConfig{ + OutputDir: dir, + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + }) + f := flow.Flow{ + SourcePath: "test.yaml", + Config: flow.Config{Name: "live steps"}, + Steps: []flow.Step{ + &flow.RunFlowStep{ + BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, + Steps: []flow.Step{ + tap("a"), + &flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: []flow.Step{tap("b")}}, + tap("c"), + }, + }, + }, + } + if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil { + t.Fatalf("Run() error = %v", err) + } + want := map[string]string{ + "a": "tapOn:running", + "b": "tapOn:passed,runFlow:running[tapOn:running]", + "c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running", + } + for id, w := range want { + if seen[id] != w { + t.Errorf("while %s ran the report had %q, want %q", id, seen[id], w) + } + } + d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + if err != nil { + t.Fatal(err) + } + if got, w := shape(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w { + t.Errorf("at the end the report has %q, want %q", got, w) + } +} diff --git a/pkg/report/flow.go b/pkg/report/flow.go index 48889cf0..46c1fb36 100644 --- a/pkg/report/flow.go +++ b/pkg/report/flow.go @@ -91,6 +91,19 @@ func (w *FlowWriter) CommandEndWithSubs(cmdIndex int, status Status, element *El w.updateIndexProgress() } +// CommandSubs writes a running command's sub-commands so far: the steps of a +// runFlow, repeat or retry that ended, and the ones running now, so a reader +// of the report sees them before the command ends. CommandEndWithSubs writes +// the final list. +func (w *FlowWriter) CommandSubs(cmdIndex int, subCommands []Command) { + if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { + return + } + w.flow.Commands[cmdIndex].SubCommands = subCommands + w.flush() + w.updateIndexProgress() +} + // CommandYAML replaces the command's description, written when the report // was built, with the one known once it ran. func (w *FlowWriter) CommandYAML(cmdIndex int, yaml string) { From 2c124d61748005fcf1c311f7ecddc1cd5f6f3390 Mon Sep 17 00:00:00 2001 From: lvladika Date: Wed, 18 Nov 2026 14:38:45 +0100 Subject: [PATCH 2/2] perf(report): a running runFlow's steps are written at most every 250 ms Each nested step's start and end rewrote the whole flow file (and copied every step so far into a new list), so the cost grew with the square of the steps: a repeat of 2 taps x 2000 took about 48 s and wrote the file 8,004 times, against 0.4 s before the live steps. FlowWriter.CommandSubs now writes at most once every LiveWriteInterval (250 ms). A call within that time leaves its list to a timer, which writes the last list given when the time is up, unless another write took it first, so steps that end quickly before a long one are in the file within the interval and not only once the long one ends. A nested runFlow's, repeat's or retry's end goes through the same limit (a repeat of a runFlow would otherwise write at each of its ends); the top-level step's start and end write at once, as before, and take a list left waiting; nothing is written for it after the flow's end. The interval starts once a write and its index update are done, so a write slower than the interval (a large flow on a slow machine, or under -race) does not let every step write again. The list is built only when it is written, on the timer's goroutine too: liveSubs passes the steps that ended as they are (they never change, and later ones are appended past their length) instead of copying them, and the FlowWriter's methods hold a lock, since the timer writes the flow too. The same repeat now takes 0.37 s and writes the file 4 times. --- pkg/executor/flow_runner.go | 39 ++++-- pkg/executor/nested_report_test.go | 201 ++++++++++++++++++++++++----- pkg/report/flow.go | 125 +++++++++++++++++- pkg/report/flow_test.go | 140 ++++++++++++++++++++ 4 files changed, 455 insertions(+), 50 deletions(-) diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index f99d3fa1..3675e581 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -62,7 +62,7 @@ func (fr *FlowRunner) Run() FlowResult { logger.Info("Total steps: %d", len(fr.flow.Steps)) // Create flow writer for this flow's updates - fr.flowWriter = report.NewFlowWriter(fr.detail, fr.config.OutputDir, fr.indexWriter) + fr.flowWriter = newFlowWriter(fr.detail, fr.config.OutputDir, fr.indexWriter) // Initialize script engine fr.script = NewScriptEngine() @@ -2001,7 +2001,7 @@ func (l *liveSteps) begin(compound bool, parent []report.Command, c report.Comma return } if compound { - l.frames = append(l.frames, liveFrame{parent: parent, cmd: c}) + l.frames = append(l.frames, liveFrame{parent: parent[:len(parent):len(parent)], cmd: c}) } else { l.step = &c } @@ -2022,23 +2022,40 @@ func (l *liveSteps) end(compound bool) { // liveSubs writes the steps of the top-level compound step that runs now to // the report as far as they got: those that ended and those running, so a // runFlow's steps can be followed while it runs and not only once it ended. +// +// The writer writes at most every so often (CommandSubs), and builds the +// list only then, maybe on its timer's goroutine, so what it is built from is +// taken here as it is now, without copying the steps: a step that ended is +// never changed, and the next ones are appended past the lengths taken here. func (fr *FlowRunner) liveSubs() { l := fr.live if l == nil || fr.flowWriter == nil { return } - list := slices.Clone(fr.subCommands) + done := fr.subCommands[:len(fr.subCommands):len(fr.subCommands)] + var step *report.Command if l.step != nil { - list = append(list, *l.step) - } - for i := len(l.frames) - 1; i >= 0; i-- { - c := l.frames[i].cmd - c.SubCommands = list - list = append(slices.Clone(l.frames[i].parent), c) - } - fr.flowWriter.CommandSubs(l.cmd, list) + s := *l.step + step = &s + } + frames := slices.Clone(l.frames) + fr.flowWriter.CommandSubs(l.cmd, func() []report.Command { + list := slices.Clone(done) + if step != nil { + list = append(list, *step) + } + for i := len(frames) - 1; i >= 0; i-- { + c := frames[i].cmd + c.SubCommands = list + list = append(slices.Clone(frames[i].parent), c) + } + return list + }) } +// newFlowWriter makes a flow's writer; a variable so tests can see it. +var newFlowWriter = report.NewFlowWriter + // executeSubFlow executes a sub-flow without separate report tracking. func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult { // Save current flow dir diff --git a/pkg/executor/nested_report_test.go b/pkg/executor/nested_report_test.go index 0c2feafb..8b9c28c0 100644 --- a/pkg/executor/nested_report_test.go +++ b/pkg/executor/nested_report_test.go @@ -3,8 +3,10 @@ package executor import ( "context" "path/filepath" + "strconv" "strings" "testing" + "time" "github.com/devicelab-dev/maestro-runner/pkg/core" "github.com/devicelab-dev/maestro-runner/pkg/flow" @@ -107,37 +109,68 @@ func TestConsoleLogsAreReportedPerStep(t *testing.T) { } } +// shapeOf is a report's commands as type:status, with their sub-commands in +// brackets. +func shapeOf(cs []report.Command) string { + var out []string + for _, c := range cs { + s := c.Type + ":" + string(c.Status) + if len(c.SubCommands) > 0 { + s += "[" + shapeOf(c.SubCommands) + "]" + } + out = append(out, s) + } + return strings.Join(out, ",") +} + +// waitForSubs reads the flow's file until its first command's sub-commands +// have the shape want, for at most within; it returns the last shape read. +func waitForSubs(path, want string, within time.Duration) (string, error) { + deadline := time.Now().Add(within) + for { + d, err := report.ReadFlowDetail(path) + got := "" + if err == nil && len(d.Commands) > 0 { + got = shapeOf(d.Commands[0].SubCommands) + } + if got == want || time.Now().After(deadline) { + return got, err + } + time.Sleep(5 * time.Millisecond) + } +} + +func liveTap(id string) *flow.TapOnStep { + return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}} +} + +func liveRunFlow(steps ...flow.Step) *flow.RunFlowStep { + return &flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: steps} +} + // A runFlow's steps are in the report while they run, each as it starts and // as it ends, the nested runFlow's too: they used to arrive with its end. +// The file is written at most every report.LiveWriteInterval, so each step +// waits for its state to reach the file. func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) { dir := t.TempDir() - var shape func(cs []report.Command) string - shape = func(cs []report.Command) string { - var out []string - for _, c := range cs { - s := c.Type + ":" + string(c.Status) - if len(c.SubCommands) > 0 { - s += "[" + shape(c.SubCommands) + "]" - } - out = append(out, s) - } - return strings.Join(out, ",") + path := filepath.Join(dir, "flows", "flow-000.json") + want := map[string]string{ + "a": "tapOn:running", + "b": "tapOn:passed,runFlow:running[tapOn:running]", + "c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running", } seen := map[string]string{} // a tap's id → its runFlow's steps in the report as it ran driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult { if tap, ok := step.(*flow.TapOnStep); ok { - d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + got, err := waitForSubs(path, want[tap.Selector.ID], 5*time.Second) if err != nil { t.Errorf("read the report while %s runs: %v", tap.Selector.ID, err) - } else { - seen[tap.Selector.ID] = shape(d.Commands[0].SubCommands) } + seen[tap.Selector.ID] = got } return &core.CommandResult{Success: true} }} - tap := func(id string) *flow.TapOnStep { - return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}} - } runner := New(driver, RunnerConfig{ OutputDir: dir, Artifacts: ArtifactNever, @@ -146,35 +179,135 @@ func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) { f := flow.Flow{ SourcePath: "test.yaml", Config: flow.Config{Name: "live steps"}, - Steps: []flow.Step{ - &flow.RunFlowStep{ - BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, - Steps: []flow.Step{ - tap("a"), - &flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: []flow.Step{tap("b")}}, - tap("c"), - }, - }, - }, + Steps: []flow.Step{liveRunFlow(liveTap("a"), liveRunFlow(liveTap("b")), liveTap("c"))}, } if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil { t.Fatalf("Run() error = %v", err) } - want := map[string]string{ - "a": "tapOn:running", - "b": "tapOn:passed,runFlow:running[tapOn:running]", - "c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running", - } for id, w := range want { if seen[id] != w { t.Errorf("while %s ran the report had %q, want %q", id, seen[id], w) } } - d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + d, err := report.ReadFlowDetail(path) if err != nil { t.Fatal(err) } - if got, w := shape(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w { + if got, w := shapeOf(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w { t.Errorf("at the end the report has %q, want %q", got, w) } } + +// Steps that end quickly before one that runs long are in the file while it +// runs, a runFlow that ended among them too: the last state is written once +// the interval is up, not only at the next step. And the top-level runFlow's +// end is in the file at once, for the step after it. +func TestQuickStepsReachTheFileWhileALongStepRuns(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "flows", "flow-000.json") + var whileLong, atNext string + var waited time.Duration + driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult { + tap, ok := step.(*flow.TapOnStep) + if !ok { + return &core.CommandResult{Success: true} + } + switch tap.Selector.ID { + case "long": + start := time.Now() + got, err := waitForSubs(path, "runFlow:passed[tapOn:passed],tapOn:running", 5*time.Second) + if err != nil { + t.Errorf("read the report while the long step runs: %v", err) + } + whileLong, waited = got, time.Since(start) + case "next": + d, err := report.ReadFlowDetail(path) + if err != nil { + t.Errorf("read the report at the next step: %v", err) + break + } + atNext = string(d.Commands[0].Status) + "[" + shapeOf(d.Commands[0].SubCommands) + "]" + } + return &core.CommandResult{Success: true} + }} + runner := New(driver, RunnerConfig{ + OutputDir: dir, + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + }) + f := flow.Flow{ + SourcePath: "test.yaml", + Config: flow.Config{Name: "quick then long"}, + Steps: []flow.Step{liveRunFlow(liveRunFlow(liveTap("quick")), liveTap("long")), liveTap("next")}, + } + if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil { + t.Fatalf("Run() error = %v", err) + } + if w := "runFlow:passed[tapOn:passed],tapOn:running"; whileLong != w { + t.Errorf("while the long step ran the report had %q after %v, want %q", whileLong, waited, w) + } + if w := "passed[runFlow:passed[tapOn:passed],tapOn:passed]"; atNext != w { + t.Errorf("at the next step the runFlow was %q, want %q", atNext, w) + } +} + +// A long repeat writes the flow's file at most every report.LiveWriteInterval +// while it runs, not at each of its steps' starts and ends: with 2 steps × +// 1000 that was about 4,000 writes of a file that grows to 0.7 MB, a cost +// that grew with the square of the steps. The same for a repeat of a +// runFlow. +func TestLongRepeatWritesTheFlowFileAtMostEveryInterval(t *testing.T) { + for _, tc := range []struct { + name string + steps []flow.Step + want string // the repeat's first two steps in the report at the end + count int // its steps in the report at the end + }{ + {"two taps", []flow.Step{liveTap("a"), liveTap("b")}, "tapOn:passed,tapOn:passed", 2000}, + {"a runFlow", []flow.Step{liveRunFlow(liveTap("a"))}, "runFlow:passed[tapOn:passed],runFlow:passed[tapOn:passed]", 1000}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + var w *report.FlowWriter + orig := newFlowWriter + newFlowWriter = func(d *report.FlowDetail, outputDir string, index *report.IndexWriter) *report.FlowWriter { + w = orig(d, outputDir, index) + return w + } + t.Cleanup(func() { newFlowWriter = orig }) + driver := &mockDriver{executeFunc: func(flow.Step) *core.CommandResult { return &core.CommandResult{Success: true} }} + runner := New(driver, RunnerConfig{ + OutputDir: dir, + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + }) + f := flow.Flow{ + SourcePath: "test.yaml", + Config: flow.Config{Name: "long repeat"}, + Steps: []flow.Step{&flow.RepeatStep{ + BaseStep: flow.BaseStep{StepType: flow.StepRepeat}, + Times: strconv.Itoa(1000), + Steps: tc.steps, + }}, + } + start := time.Now() + if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil { + t.Fatalf("Run() error = %v", err) + } + took := time.Since(start) + // The flow's own writes (its start, the repeat's start and end, + // its end) and one live write per interval at most. + if limit := 10 + int(took/report.LiveWriteInterval); w.Writes() > limit { + t.Errorf("the flow's file was written %d times in %v, want at most %d", w.Writes(), took, limit) + } + d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + if err != nil { + t.Fatal(err) + } + subs := d.Commands[0].SubCommands + if len(subs) != tc.count || shapeOf(subs[:2]) != tc.want { + t.Errorf("at the end the repeat has %d steps starting %q, want %d starting %q", len(subs), shapeOf(subs[:min(2, len(subs))]), tc.count, tc.want) + } + }) + } +} diff --git a/pkg/report/flow.go b/pkg/report/flow.go index 46c1fb36..a4dbdb99 100644 --- a/pkg/report/flow.go +++ b/pkg/report/flow.go @@ -5,18 +5,43 @@ import ( "fmt" "os" "path/filepath" + "sync" "time" "github.com/devicelab-dev/maestro-runner/pkg/logger" ) +// LiveWriteInterval is the shortest time between two writes of the flow's +// file for a running command's sub-commands (CommandSubs). +const LiveWriteInterval = 250 * time.Millisecond + // FlowWriter writes updates for a single flow. -// Each flow goroutine has its own FlowWriter - no locking needed. +// Each flow goroutine has its own FlowWriter. A running command's +// sub-commands may also be written from a timer (CommandSubs), so the methods +// that change or write the flow hold mu. type FlowWriter struct { + mu sync.Mutex flow *FlowDetail path string assetsDir string index *IndexWriter + + // The sub-commands CommandSubs was given last and has not written yet, + // and the timer that writes them once liveEvery has passed since the + // last write; liveGen tells a timer that fires after it was superseded. + liveEvery time.Duration + live *liveList + liveTimer *time.Timer + liveGen int + lastWrite time.Time + writes int + ended bool +} + +// liveList is a running command's sub-commands as CommandSubs was given them. +type liveList struct { + cmd int + subs func() []Command } // NewFlowWriter creates a new FlowWriter for a flow. @@ -34,11 +59,14 @@ func NewFlowWriter(flowDetail *FlowDetail, outputDir string, index *IndexWriter) path: flowPath, assetsDir: assetsDir, index: index, + liveEvery: LiveWriteInterval, } } // Start marks the flow as started. func (w *FlowWriter) Start() { + w.lock() + defer w.mu.Unlock() now := time.Now() w.flow.StartTime = now @@ -48,6 +76,8 @@ func (w *FlowWriter) Start() { // CommandStart marks a command as started. func (w *FlowWriter) CommandStart(cmdIndex int) { + w.lock() + defer w.mu.Unlock() if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { return } @@ -68,6 +98,8 @@ func (w *FlowWriter) CommandEnd(cmdIndex int, status Status, element *Element, e // CommandEndWithSubs marks a command as complete with optional sub-commands. func (w *FlowWriter) CommandEndWithSubs(cmdIndex int, status Status, element *Element, err *Error, artifacts CommandArtifacts, subCommands []Command) { + w.lock() + defer w.mu.Unlock() if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { return } @@ -95,18 +127,84 @@ func (w *FlowWriter) CommandEndWithSubs(cmdIndex int, status Status, element *El // runFlow, repeat or retry that ended, and the ones running now, so a reader // of the report sees them before the command ends. CommandEndWithSubs writes // the final list. -func (w *FlowWriter) CommandSubs(cmdIndex int, subCommands []Command) { - if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { +// +// The file is written at most once every liveEvery for them: a call within +// that time of the last write leaves its list to a timer, which writes the +// list of the last call when the time is up, unless another write took it +// first. So a long repeat does not rewrite the whole file at each of its +// steps, and a step that runs long after short ones still finds them in the +// file. subs builds the list when it is written, which may be later and on +// the timer's goroutine, so it must read only what no longer changes. +func (w *FlowWriter) CommandSubs(cmdIndex int, subs func() []Command) { + w.mu.Lock() + defer w.mu.Unlock() + if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) || w.ended { + return + } + w.live = &liveList{cmd: cmdIndex, subs: subs} + if wait := w.liveEvery - time.Since(w.lastWrite); wait > 0 { + if w.liveTimer == nil { + gen := w.liveGen + w.liveTimer = time.AfterFunc(wait, func() { w.writeLive(gen) }) + } + return + } + w.writeSubs() +} + +// writeLive is CommandSubs' timer: it writes the list waiting, unless a +// write took it meanwhile (liveGen moved) or the flow ended. +func (w *FlowWriter) writeLive(gen int) { + w.mu.Lock() + defer w.mu.Unlock() + if gen != w.liveGen || w.live == nil || w.ended { return } - w.flow.Commands[cmdIndex].SubCommands = subCommands + w.writeSubs() +} + +// writeSubs writes the list waiting and updates the index. The next interval +// starts once both are done: a write that takes longer than the interval (a +// large flow on a slow machine) still leaves the flow that long to run. +func (w *FlowWriter) writeSubs() { + w.applyLive() w.flush() w.updateIndexProgress() + w.lastWrite = time.Now() +} + +// lock takes mu and puts the list CommandSubs left waiting into the flow, so +// the write the caller makes carries it. +func (w *FlowWriter) lock() { + w.mu.Lock() + w.applyLive() +} + +// applyLive puts the waiting list, if any, into the flow and stops its timer. +func (w *FlowWriter) applyLive() { + if w.liveTimer != nil { + w.liveTimer.Stop() + w.liveTimer = nil + } + w.liveGen++ + if w.live != nil { + w.flow.Commands[w.live.cmd].SubCommands = w.live.subs() + w.live = nil + } +} + +// Writes returns how many times the flow's file was written so far. +func (w *FlowWriter) Writes() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.writes } // CommandYAML replaces the command's description, written when the report // was built, with the one known once it ran. func (w *FlowWriter) CommandYAML(cmdIndex int, yaml string) { + w.lock() + defer w.mu.Unlock() if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { return } @@ -117,6 +215,8 @@ func (w *FlowWriter) CommandYAML(cmdIndex int, yaml string) { // CommandLogs attaches what the command's scripts printed with console.log, // warn or error. func (w *FlowWriter) CommandLogs(cmdIndex int, logs []string) { + w.lock() + defer w.mu.Unlock() if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) || len(logs) == 0 { return } @@ -137,6 +237,9 @@ func (w *FlowWriter) CommandMessage(cmdIndex int, msg string) { // as the failure message when no individual command recorded one — covers // flow-level failures like failOnConsoleError. func (w *FlowWriter) End(status Status, flowLevelError ...string) { + w.lock() + defer w.mu.Unlock() + w.ended = true now := time.Now() w.flow.EndTime = &now @@ -187,6 +290,8 @@ func (w *FlowWriter) End(status Status, flowLevelError ...string) { // SetFlowArtifacts sets flow-level artifacts (video, logs). func (w *FlowWriter) SetFlowArtifacts(artifacts FlowArtifacts) { + w.lock() + defer w.mu.Unlock() w.flow.Artifacts = artifacts w.flush() } @@ -194,12 +299,16 @@ func (w *FlowWriter) SetFlowArtifacts(artifacts FlowArtifacts) { // SetConsoleLogs records browser console / page error entries captured during // the flow. Web flows only — mobile / native drivers leave this nil. func (w *FlowWriter) SetConsoleLogs(logs []ConsoleLog) { + w.lock() + defer w.mu.Unlock() w.flow.ConsoleLogs = logs w.flush() } // AddVideoTimestamp adds a video timestamp mapping. func (w *FlowWriter) AddVideoTimestamp(cmdIndex int, videoTimeMs int64) { + w.lock() + defer w.mu.Unlock() w.flow.Artifacts.VideoTimestamps = append(w.flow.Artifacts.VideoTimestamps, VideoTimestamp{ CommandIndex: cmdIndex, VideoTimeMs: videoTimeMs, @@ -305,6 +414,8 @@ func (w *FlowWriter) RecordingTarget() string { // SetVideo records that a screen recording exists at RecordingTarget, // storing the report-relative path alongside the other flow artifacts. func (w *FlowWriter) SetVideo() { + w.lock() + defer w.mu.Unlock() w.flow.Artifacts.Video = filepath.Join("assets", w.flow.ID, "recording.mp4") w.flush() } @@ -326,11 +437,13 @@ func (w *FlowWriter) GetFlowDetail() *FlowDetail { return w.flow } -// flush writes the flow detail to disk. +// flush writes the flow detail to disk. The caller holds mu. func (w *FlowWriter) flush() { + w.writes++ if err := atomicWriteJSON(w.path, w.flow); err != nil { logger.Warn("failed to write flow detail to %s: %v", w.path, err) } + w.lastWrite = time.Now() } // updateIndex updates the index with current flow state. @@ -381,6 +494,8 @@ func (w *FlowWriter) commandSummary() CommandSummary { // SkipRemainingCommands marks all pending commands as skipped. // Called when a command fails and we need to skip the rest. func (w *FlowWriter) SkipRemainingCommands(fromIndex int) { + w.lock() + defer w.mu.Unlock() for i := fromIndex; i < len(w.flow.Commands); i++ { if w.flow.Commands[i].Status == StatusPending { w.flow.Commands[i].Status = StatusSkipped diff --git a/pkg/report/flow_test.go b/pkg/report/flow_test.go index 01ccdd20..6f649dbb 100644 --- a/pkg/report/flow_test.go +++ b/pkg/report/flow_test.go @@ -549,6 +549,146 @@ func TestFlowWriter_commandSummary(t *testing.T) { } } +// subsOf is a CommandSubs list builder that counts its calls. +func subsOf(calls *int, cs ...Command) func() []Command { + return func() []Command { + *calls++ + return cs + } +} + +func readSubs(t *testing.T, tmpDir string) []Command { + t.Helper() + d, err := ReadFlowDetail(filepath.Join(tmpDir, "flows", "flow-000.json")) + if err != nil { + t.Fatal(err) + } + return d.Commands[1].SubCommands +} + +// A running command's sub-commands are written at most once every +// liveEvery, and built only when written: the end's write takes the last. +func TestFlowWriter_CommandSubs_WritesAtMostEveryInterval(t *testing.T) { + fw, iw, tmpDir := createTestFlowWriter(t) + defer iw.Close() + fw.liveEvery = time.Hour + + fw.Start() + fw.CommandStart(1) + before := fw.Writes() + var built int + for i := 0; i < 1000; i++ { + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusRunning, Index: i})) + } + if got := fw.Writes() - before; got != 0 { + t.Errorf("1000 calls within the interval wrote the file %d times, want 0", got) + } + if built != 0 { + t.Errorf("the lists were built %d times before a write, want 0", built) + } + fw.CommandEndWithSubs(1, StatusPassed, nil, nil, CommandArtifacts{}, []Command{{Type: "tapOn", Status: StatusPassed}}) + if got := fw.Writes() - before; got != 1 { + t.Errorf("the end wrote the file %d times, want 1", got) + } + if subs := readSubs(t, tmpDir); len(subs) != 1 || subs[0].Status != StatusPassed { + t.Errorf("after the end the file has %+v, want the end's list", subs) + } +} + +// A list given within the interval is written when it is up, without +// another call: the last one given. +func TestFlowWriter_CommandSubs_WritesTheLastListWhenTheIntervalIsUp(t *testing.T) { + fw, iw, tmpDir := createTestFlowWriter(t) + defer iw.Close() + fw.liveEvery = 50 * time.Millisecond + + fw.Start() + before := fw.Writes() + var built int + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusRunning})) + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusPassed}, Command{Type: "assertVisible", Status: StatusRunning})) + deadline := time.Now().Add(5 * time.Second) + for fw.Writes() == before && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) // a second write would come by now + if got := fw.Writes() - before; got != 1 { + t.Errorf("the file was written %d times, want 1", got) + } + if subs := readSubs(t, tmpDir); len(subs) != 2 || subs[1].Status != StatusRunning { + t.Errorf("the file has %+v, want the last list", subs) + } + if built != 1 { + t.Errorf("the lists were built %d times, want 1", built) + } +} + +// Nothing is written for a running command's list after the flow's end: a +// list left waiting goes into the end's write, and a later one is dropped. +func TestFlowWriter_CommandSubs_NoWriteAfterEnd(t *testing.T) { + fw, iw, tmpDir := createTestFlowWriter(t) + defer iw.Close() + fw.liveEvery = 50 * time.Millisecond + + fw.Start() + var built int + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusRunning})) + fw.End(StatusFailed) + after := fw.Writes() + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusPassed})) + time.Sleep(150 * time.Millisecond) + if got := fw.Writes() - after; got != 0 { + t.Errorf("the file was written %d times after the end, want 0", got) + } + if subs := readSubs(t, tmpDir); len(subs) != 1 || subs[0].Status != StatusRunning { + t.Errorf("the file has %+v, want the list the end wrote", subs) + } +} + +// With no interval each call writes, as before. +func TestFlowWriter_CommandSubs_NoInterval(t *testing.T) { + fw, iw, _ := createTestFlowWriter(t) + defer iw.Close() + fw.liveEvery = 0 + + fw.Start() + before := fw.Writes() + var built int + for i := 0; i < 3; i++ { + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusRunning})) + } + if got := fw.Writes() - before; got != 3 || built != 3 { + t.Errorf("3 calls wrote %d times and built %d lists, want 3 and 3", got, built) + } +} + +// The interval starts once a write is done, index included: a write slower +// than the interval (a large flow on a slow machine) does not let the next +// call write at once, which wrote at every step again. +func TestFlowWriter_CommandSubs_IntervalStartsAfterTheWrite(t *testing.T) { + fw, iw, _ := createTestFlowWriter(t) + defer iw.Close() + fw.liveEvery = 100 * time.Millisecond + + fw.Start() + time.Sleep(150 * time.Millisecond) // the next call writes at once + var built int + iw.mu.Lock() // the index is busy: the write's index update waits for it + done := make(chan struct{}) + go func() { + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusRunning})) + close(done) + }() + time.Sleep(300 * time.Millisecond) + iw.mu.Unlock() + <-done + before := fw.Writes() + fw.CommandSubs(1, subsOf(&built, Command{Type: "tapOn", Status: StatusPassed})) + if got := fw.Writes() - before; got != 0 { + t.Errorf("a call right after a write that took 300 ms wrote %d times, want 0 (it waits for the interval)", got) + } +} + // A passed command's outcome message reaches the flow detail (and so // report.json's flow files); an empty one or a bad index changes nothing. func TestFlowWriter_CommandMessage(t *testing.T) {