Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 101 additions & 1 deletion pkg/executor/flow_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"math"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -40,6 +41,9 @@ type FlowRunner struct {
conditionSkipped bool
// Sub-command tracking for compound steps (runFlow, repeat, retry)
subCommands []report.Command
// live is the top-level compound step that runs now, whose steps go to
// the report as they start and end (liveSubs); nil between them.
live *liveSteps
// nestedArtifactSeq numbers failure artifacts captured for nested steps
// (runFlow / repeat / retry), which have no top-level command index.
nestedArtifactSeq int
Expand All @@ -58,7 +62,7 @@ func (fr *FlowRunner) Run() FlowResult {
logger.Info("Total steps: %d", len(fr.flow.Steps))

// Create flow writer for this flow's updates
fr.flowWriter = report.NewFlowWriter(fr.detail, fr.config.OutputDir, fr.indexWriter)
fr.flowWriter = newFlowWriter(fr.detail, fr.config.OutputDir, fr.indexWriter)

// Initialize script engine
fr.script = NewScriptEngine()
Expand Down Expand Up @@ -679,7 +683,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin
if err := fr.script.ExpandStep(step); err != nil {
result = expressionFailed(err)
} else {
switch step.(type) {
case *flow.RepeatStep, *flow.RetryStep, *flow.RunFlowStep:
fr.live = &liveSteps{cmd: idx}
}
result = fr.runStep(idx, step, &artifacts)
fr.live = nil
}
result = fr.maskTypedResult(step, result)

Expand Down Expand Up @@ -1838,6 +1847,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
// inside a runFlow showed the password itself.
describe, desc := fr.stepNames(step)

// The step as it starts, in the report while it runs (liveSubs).
running := report.Command{
ID: fmt.Sprintf("sub-%d", len(fr.subCommands)),
Index: len(fr.subCommands),
Type: string(step.Type()),
Label: step.Label(),
YAML: describe,
Status: report.StatusRunning,
StartTime: &start,
}

// For nested compound steps, we need to track their sub-commands separately
var nestedSubCommands, parentSubCommands []report.Command
isCompoundStep := false
Expand All @@ -1848,6 +1868,8 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
parentSubCommands = fr.subCommands
fr.subCommands = nil
}
fr.live.begin(isCompoundStep, parentSubCommands, running)
fr.liveSubs()

// Expand once, before the handler, as executeStep does: a ${...} that
// throws fails the step and nothing runs.
Expand Down Expand Up @@ -1900,6 +1922,7 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
nestedSubCommands = fr.subCommands
fr.subCommands = parentSubCommands
}
fr.live.end(isCompoundStep)

// Add to parent's sub-commands for report
status := report.StatusPassed
Expand Down Expand Up @@ -1952,10 +1975,87 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
}

fr.subCommands = append(fr.subCommands, cmd)
fr.liveSubs()

return result
}

// liveSteps is a top-level runFlow, repeat or retry while it runs: the
// compound steps running inside it, outermost first, each with the list it
// goes into once it ends, and the step running in the innermost one.
type liveSteps struct {
cmd int
frames []liveFrame
step *report.Command
}

type liveFrame struct {
parent []report.Command
cmd report.Command
}

// begin records a nested step that starts: a compound one with the list it
// goes into (its parent's), another as the step running.
func (l *liveSteps) begin(compound bool, parent []report.Command, c report.Command) {
if l == nil {
return
}
if compound {
l.frames = append(l.frames, liveFrame{parent: parent[:len(parent):len(parent)], cmd: c})
} else {
l.step = &c
}
}

// end forgets the nested step begin recorded last, which ended.
func (l *liveSteps) end(compound bool) {
if l == nil {
return
}
if compound {
l.frames = l.frames[:len(l.frames)-1]
} else {
l.step = nil
}
}

// liveSubs writes the steps of the top-level compound step that runs now to
// the report as far as they got: those that ended and those running, so a
// runFlow's steps can be followed while it runs and not only once it ended.
//
// The writer writes at most every so often (CommandSubs), and builds the
// list only then, maybe on its timer's goroutine, so what it is built from is
// taken here as it is now, without copying the steps: a step that ended is
// never changed, and the next ones are appended past the lengths taken here.
func (fr *FlowRunner) liveSubs() {
l := fr.live
if l == nil || fr.flowWriter == nil {
return
}
done := fr.subCommands[:len(fr.subCommands):len(fr.subCommands)]
var step *report.Command
if l.step != nil {
s := *l.step
step = &s
}
frames := slices.Clone(l.frames)
fr.flowWriter.CommandSubs(l.cmd, func() []report.Command {
list := slices.Clone(done)
if step != nil {
list = append(list, *step)
}
for i := len(frames) - 1; i >= 0; i-- {
c := frames[i].cmd
c.SubCommands = list
list = append(slices.Clone(frames[i].parent), c)
}
return list
})
}

// newFlowWriter makes a flow's writer; a variable so tests can see it.
var newFlowWriter = report.NewFlowWriter

// executeSubFlow executes a sub-flow without separate report tracking.
func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult {
// Save current flow dir
Expand Down
207 changes: 207 additions & 0 deletions pkg/executor/nested_report_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@ package executor
import (
"context"
"path/filepath"
"strconv"
"strings"
"testing"
"time"

"github.com/devicelab-dev/maestro-runner/pkg/core"
"github.com/devicelab-dev/maestro-runner/pkg/flow"
"github.com/devicelab-dev/maestro-runner/pkg/report"
)
Expand Down Expand Up @@ -104,3 +108,206 @@ func TestConsoleLogsAreReportedPerStep(t *testing.T) {
t.Errorf("runFlow logs %q, tapOn logs %q: want none", cmds[1].Logs, cmds[2].Logs)
}
}

// shapeOf is a report's commands as type:status, with their sub-commands in
// brackets.
func shapeOf(cs []report.Command) string {
var out []string
for _, c := range cs {
s := c.Type + ":" + string(c.Status)
if len(c.SubCommands) > 0 {
s += "[" + shapeOf(c.SubCommands) + "]"
}
out = append(out, s)
}
return strings.Join(out, ",")
}

// waitForSubs reads the flow's file until its first command's sub-commands
// have the shape want, for at most within; it returns the last shape read.
func waitForSubs(path, want string, within time.Duration) (string, error) {
deadline := time.Now().Add(within)
for {
d, err := report.ReadFlowDetail(path)
got := ""
if err == nil && len(d.Commands) > 0 {
got = shapeOf(d.Commands[0].SubCommands)
}
if got == want || time.Now().After(deadline) {
return got, err
}
time.Sleep(5 * time.Millisecond)
}
}

func liveTap(id string) *flow.TapOnStep {
return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}}
}

func liveRunFlow(steps ...flow.Step) *flow.RunFlowStep {
return &flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: steps}
}

// A runFlow's steps are in the report while they run, each as it starts and
// as it ends, the nested runFlow's too: they used to arrive with its end.
// The file is written at most every report.LiveWriteInterval, so each step
// waits for its state to reach the file.
func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "flows", "flow-000.json")
want := map[string]string{
"a": "tapOn:running",
"b": "tapOn:passed,runFlow:running[tapOn:running]",
"c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running",
}
seen := map[string]string{} // a tap's id → its runFlow's steps in the report as it ran
driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult {
if tap, ok := step.(*flow.TapOnStep); ok {
got, err := waitForSubs(path, want[tap.Selector.ID], 5*time.Second)
if err != nil {
t.Errorf("read the report while %s runs: %v", tap.Selector.ID, err)
}
seen[tap.Selector.ID] = got
}
return &core.CommandResult{Success: true}
}}
runner := New(driver, RunnerConfig{
OutputDir: dir,
Artifacts: ArtifactNever,
Device: report.Device{ID: "test", Platform: "android"},
})
f := flow.Flow{
SourcePath: "test.yaml",
Config: flow.Config{Name: "live steps"},
Steps: []flow.Step{liveRunFlow(liveTap("a"), liveRunFlow(liveTap("b")), liveTap("c"))},
}
if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil {
t.Fatalf("Run() error = %v", err)
}
for id, w := range want {
if seen[id] != w {
t.Errorf("while %s ran the report had %q, want %q", id, seen[id], w)
}
}
d, err := report.ReadFlowDetail(path)
if err != nil {
t.Fatal(err)
}
if got, w := shapeOf(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w {
t.Errorf("at the end the report has %q, want %q", got, w)
}
}

// Steps that end quickly before one that runs long are in the file while it
// runs, a runFlow that ended among them too: the last state is written once
// the interval is up, not only at the next step. And the top-level runFlow's
// end is in the file at once, for the step after it.
func TestQuickStepsReachTheFileWhileALongStepRuns(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "flows", "flow-000.json")
var whileLong, atNext string
var waited time.Duration
driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult {
tap, ok := step.(*flow.TapOnStep)
if !ok {
return &core.CommandResult{Success: true}
}
switch tap.Selector.ID {
case "long":
start := time.Now()
got, err := waitForSubs(path, "runFlow:passed[tapOn:passed],tapOn:running", 5*time.Second)
if err != nil {
t.Errorf("read the report while the long step runs: %v", err)
}
whileLong, waited = got, time.Since(start)
case "next":
d, err := report.ReadFlowDetail(path)
if err != nil {
t.Errorf("read the report at the next step: %v", err)
break
}
atNext = string(d.Commands[0].Status) + "[" + shapeOf(d.Commands[0].SubCommands) + "]"
}
return &core.CommandResult{Success: true}
}}
runner := New(driver, RunnerConfig{
OutputDir: dir,
Artifacts: ArtifactNever,
Device: report.Device{ID: "test", Platform: "android"},
})
f := flow.Flow{
SourcePath: "test.yaml",
Config: flow.Config{Name: "quick then long"},
Steps: []flow.Step{liveRunFlow(liveRunFlow(liveTap("quick")), liveTap("long")), liveTap("next")},
}
if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil {
t.Fatalf("Run() error = %v", err)
}
if w := "runFlow:passed[tapOn:passed],tapOn:running"; whileLong != w {
t.Errorf("while the long step ran the report had %q after %v, want %q", whileLong, waited, w)
}
if w := "passed[runFlow:passed[tapOn:passed],tapOn:passed]"; atNext != w {
t.Errorf("at the next step the runFlow was %q, want %q", atNext, w)
}
}

// A long repeat writes the flow's file at most every report.LiveWriteInterval
// while it runs, not at each of its steps' starts and ends: with 2 steps ×
// 1000 that was about 4,000 writes of a file that grows to 0.7 MB, a cost
// that grew with the square of the steps. The same for a repeat of a
// runFlow.
func TestLongRepeatWritesTheFlowFileAtMostEveryInterval(t *testing.T) {
for _, tc := range []struct {
name string
steps []flow.Step
want string // the repeat's first two steps in the report at the end
count int // its steps in the report at the end
}{
{"two taps", []flow.Step{liveTap("a"), liveTap("b")}, "tapOn:passed,tapOn:passed", 2000},
{"a runFlow", []flow.Step{liveRunFlow(liveTap("a"))}, "runFlow:passed[tapOn:passed],runFlow:passed[tapOn:passed]", 1000},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
var w *report.FlowWriter
orig := newFlowWriter
newFlowWriter = func(d *report.FlowDetail, outputDir string, index *report.IndexWriter) *report.FlowWriter {
w = orig(d, outputDir, index)
return w
}
t.Cleanup(func() { newFlowWriter = orig })
driver := &mockDriver{executeFunc: func(flow.Step) *core.CommandResult { return &core.CommandResult{Success: true} }}
runner := New(driver, RunnerConfig{
OutputDir: dir,
Artifacts: ArtifactNever,
Device: report.Device{ID: "test", Platform: "android"},
})
f := flow.Flow{
SourcePath: "test.yaml",
Config: flow.Config{Name: "long repeat"},
Steps: []flow.Step{&flow.RepeatStep{
BaseStep: flow.BaseStep{StepType: flow.StepRepeat},
Times: strconv.Itoa(1000),
Steps: tc.steps,
}},
}
start := time.Now()
if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil {
t.Fatalf("Run() error = %v", err)
}
took := time.Since(start)
// The flow's own writes (its start, the repeat's start and end,
// its end) and one live write per interval at most.
if limit := 10 + int(took/report.LiveWriteInterval); w.Writes() > limit {
t.Errorf("the flow's file was written %d times in %v, want at most %d", w.Writes(), took, limit)
}
d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json"))
if err != nil {
t.Fatal(err)
}
subs := d.Commands[0].SubCommands
if len(subs) != tc.count || shapeOf(subs[:2]) != tc.want {
t.Errorf("at the end the repeat has %d steps starting %q, want %d starting %q", len(subs), shapeOf(subs[:min(2, len(subs))]), tc.count, tc.want)
}
})
}
}
Loading
Loading