From eb99c36ff626f9938392e7c24c38b09848e597e0 Mon Sep 17 00:00:00 2001 From: lvladika Date: Sun, 15 Nov 2026 23:58:36 +0100 Subject: [PATCH] fix(report): a step skipped by its when: is reported skipped A runFlow or runScript whose when: was false did not run, but the report said passed, the same as a step that ran, so a reader of the report could not tell the two apart. Maestro reports such a step as skipped (it throws CommandSkipped in Orchestra.kt's runFlowCommand and runScriptCommand). The step is now skipped in the report, at the top level and inside a runFlow, retry or repeat, and counts as skipped instead of passed. The console still prints it with a check mark: OnStepComplete gets passed for any step that did not fail, so a skipped step is not printed as a failure. That also applies to a step skipped by its platform:, which was printed with a cross. --- pkg/executor/flow_runner.go | 27 ++++++-- pkg/executor/when_skipped_test.go | 105 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 pkg/executor/when_skipped_test.go diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 5b5df9c5..faddeb82 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -34,8 +34,8 @@ type FlowRunner struct { stepsFailed int stepsSkipped int - // Set when a runFlow's or runScript's when: skipped it; markInteraction - // consumes it + // Set when a runFlow's or runScript's when: skipped it: the step is + // reported skipped, and markInteraction consumes it conditionSkipped bool // Sub-command tracking for compound steps (runFlow, repeat, retry) subCommands []report.Command @@ -401,7 +401,7 @@ func (fr *FlowRunner) runTopLevelStep(idx int, step flow.Step) (report.Status, s // Notify step complete if fr.config.OnStepComplete != nil { - fr.config.OnStepComplete(idx, desc, stepStatus == report.StatusPassed, stepDuration, stepError) + fr.config.OnStepComplete(idx, desc, stepStatus != report.StatusFailed, stepDuration, stepError) } // Track step counts (compound steps like runFlow/repeat/retry don't count themselves, @@ -695,7 +695,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin var errorInfo *report.Error var errorMsg string - if result.Success { + if result.Success && fr.conditionSkipped { + // Its when: was false. Maestro reports the step skipped, not passed + // (CommandSkipped in Orchestra.kt's runFlowCommand and runScriptCommand). + status = report.StatusSkipped + logger.Debug("Step %d skipped (when condition not met): %s", idx, fr.maskTypedName(step, step.Describe())) + } else if result.Success { status = report.StatusPassed logger.Debug("Step %d completed successfully (%dms): %s", idx, stepDuration, fr.maskTypedName(step, step.Describe())) } else { @@ -1799,11 +1804,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { result = fr.enrichTimeoutError(result) } + // Its when: was false: skipped, as at the top level. + skipped := result.Success && fr.conditionSkipped + // Track nested step counts (compound steps like runFlow/repeat/retry don't count themselves) if !isCompoundStep { - if result.Success { + switch { + case skipped: + fr.stepsSkipped++ + case result.Success: fr.stepsPassed++ - } else { + default: fr.stepsFailed++ } } @@ -1831,7 +1842,9 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { // Add to parent's sub-commands for report status := report.StatusPassed - if !result.Success { + if skipped { + status = report.StatusSkipped + } else if !result.Success { status = report.StatusFailed } diff --git a/pkg/executor/when_skipped_test.go b/pkg/executor/when_skipped_test.go new file mode 100644 index 00000000..71da7f8b --- /dev/null +++ b/pkg/executor/when_skipped_test.go @@ -0,0 +1,105 @@ +package executor + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/flow" + "github.com/devicelab-dev/maestro-runner/pkg/report" +) + +// A runFlow or runScript whose when: is false is skipped, and the report says +// so, at the top level and inside a runFlow, as Maestro's does (its +// CommandSkipped). It used to say passed, like a step that ran. +func TestWhenFalseIsReportedSkipped(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("set.js", "output.runs = (output.runs || 0) + 1;\n") + write("sub.yaml", `appId: x +--- +- tapOn: Sub +`) + write("main.yaml", `appId: x +--- +- evalScript: ${output.mode = 'local'} +- runScript: + file: set.js + when: + true: ${output.mode == 'ci'} +- runFlow: + file: sub.yaml + when: + true: ${output.mode == 'ci'} +- runFlow: + commands: + - runScript: + file: set.js + when: + true: ${output.mode == 'ci'} + - runFlow: + when: + true: ${output.mode == 'ci'} + commands: + - tapOn: Inline + - runScript: + file: set.js +- runScript: + file: set.js + when: + true: ${output.mode == 'local'} +`) + f, err := flow.ParseFile(filepath.Join(dir, "main.yaml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + out := t.TempDir() + var seen []flow.Step + runner := New(realDriverRejects(&seen), RunnerConfig{ + OutputDir: out, + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + }) + result, err := runner.Run(context.Background(), []flow.Flow{*f}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if result.Status != report.StatusPassed { + t.Fatalf("status = %s, want passed: %+v", result.Status, result.FlowResults) + } + if len(seen) != 0 { + t.Errorf("the device got %d steps, want none (both taps are skipped)", len(seen)) + } + detail, err := report.ReadFlowDetail(filepath.Join(out, "flows", "flow-000.json")) + if err != nil { + t.Fatal(err) + } + var walk func([]report.Command) string + walk = func(cmds []report.Command) string { + var s []string + for _, c := range cmds { + e := string(c.Status) + if len(c.SubCommands) > 0 { + e += "[" + walk(c.SubCommands) + "]" + } + s = append(s, e) + } + return strings.Join(s, ",") + } + got := walk(detail.Commands) + want := "passed,skipped,skipped,passed[skipped,skipped,passed],passed" + if got != want { + t.Errorf("statuses = %s, want %s", got, want) + } + // Two scripts skipped; the runFlows do not count themselves. + if fr := result.FlowResults[0]; fr.StepsSkipped != 2 || fr.StepsPassed != 3 { + t.Errorf("steps passed %d, skipped %d; want 3 and 2", fr.StepsPassed, fr.StepsSkipped) + } +}