Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions pkg/executor/flow_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ type FlowRunner struct {
stepsFailed int
stepsSkipped int

// Set when a runFlow's or runScript's when: skipped it; markInteraction
// consumes it
// Set when a runFlow's or runScript's when: skipped it: the step is
// reported skipped, and markInteraction consumes it
conditionSkipped bool
// Sub-command tracking for compound steps (runFlow, repeat, retry)
subCommands []report.Command
Expand Down Expand Up @@ -401,7 +401,7 @@ func (fr *FlowRunner) runTopLevelStep(idx int, step flow.Step) (report.Status, s

// Notify step complete
if fr.config.OnStepComplete != nil {
fr.config.OnStepComplete(idx, desc, stepStatus == report.StatusPassed, stepDuration, stepError)
fr.config.OnStepComplete(idx, desc, stepStatus != report.StatusFailed, stepDuration, stepError)
}

// Track step counts (compound steps like runFlow/repeat/retry don't count themselves,
Expand Down Expand Up @@ -695,7 +695,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin
var errorInfo *report.Error
var errorMsg string

if result.Success {
if result.Success && fr.conditionSkipped {
// Its when: was false. Maestro reports the step skipped, not passed
// (CommandSkipped in Orchestra.kt's runFlowCommand and runScriptCommand).
status = report.StatusSkipped
logger.Debug("Step %d skipped (when condition not met): %s", idx, fr.maskTypedName(step, step.Describe()))
} else if result.Success {
status = report.StatusPassed
logger.Debug("Step %d completed successfully (%dms): %s", idx, stepDuration, fr.maskTypedName(step, step.Describe()))
} else {
Expand Down Expand Up @@ -1799,11 +1804,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
result = fr.enrichTimeoutError(result)
}

// Its when: was false: skipped, as at the top level.
skipped := result.Success && fr.conditionSkipped

// Track nested step counts (compound steps like runFlow/repeat/retry don't count themselves)
if !isCompoundStep {
if result.Success {
switch {
case skipped:
fr.stepsSkipped++
case result.Success:
fr.stepsPassed++
} else {
default:
fr.stepsFailed++
}
}
Expand Down Expand Up @@ -1831,7 +1842,9 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {

// Add to parent's sub-commands for report
status := report.StatusPassed
if !result.Success {
if skipped {
status = report.StatusSkipped
} else if !result.Success {
status = report.StatusFailed
}

Expand Down
105 changes: 105 additions & 0 deletions pkg/executor/when_skipped_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package executor

import (
"context"
"os"
"path/filepath"
"strings"
"testing"

"github.com/devicelab-dev/maestro-runner/pkg/flow"
"github.com/devicelab-dev/maestro-runner/pkg/report"
)

// A runFlow or runScript whose when: is false is skipped, and the report says
// so, at the top level and inside a runFlow, as Maestro's does (its
// CommandSkipped). It used to say passed, like a step that ran.
func TestWhenFalseIsReportedSkipped(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
write("set.js", "output.runs = (output.runs || 0) + 1;\n")
write("sub.yaml", `appId: x
---
- tapOn: Sub
`)
write("main.yaml", `appId: x
---
- evalScript: ${output.mode = 'local'}
- runScript:
file: set.js
when:
true: ${output.mode == 'ci'}
- runFlow:
file: sub.yaml
when:
true: ${output.mode == 'ci'}
- runFlow:
commands:
- runScript:
file: set.js
when:
true: ${output.mode == 'ci'}
- runFlow:
when:
true: ${output.mode == 'ci'}
commands:
- tapOn: Inline
- runScript:
file: set.js
- runScript:
file: set.js
when:
true: ${output.mode == 'local'}
`)
f, err := flow.ParseFile(filepath.Join(dir, "main.yaml"))
if err != nil {
t.Fatalf("parse: %v", err)
}
out := t.TempDir()
var seen []flow.Step
runner := New(realDriverRejects(&seen), RunnerConfig{
OutputDir: out,
Artifacts: ArtifactNever,
Device: report.Device{ID: "test", Platform: "android"},
})
result, err := runner.Run(context.Background(), []flow.Flow{*f})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Status != report.StatusPassed {
t.Fatalf("status = %s, want passed: %+v", result.Status, result.FlowResults)
}
if len(seen) != 0 {
t.Errorf("the device got %d steps, want none (both taps are skipped)", len(seen))
}
detail, err := report.ReadFlowDetail(filepath.Join(out, "flows", "flow-000.json"))
if err != nil {
t.Fatal(err)
}
var walk func([]report.Command) string
walk = func(cmds []report.Command) string {
var s []string
for _, c := range cmds {
e := string(c.Status)
if len(c.SubCommands) > 0 {
e += "[" + walk(c.SubCommands) + "]"
}
s = append(s, e)
}
return strings.Join(s, ",")
}
got := walk(detail.Commands)
want := "passed,skipped,skipped,passed[skipped,skipped,passed],passed"
if got != want {
t.Errorf("statuses = %s, want %s", got, want)
}
// Two scripts skipped; the runFlows do not count themselves.
if fr := result.FlowResults[0]; fr.StepsSkipped != 2 || fr.StepsPassed != 3 {
t.Errorf("steps passed %d, skipped %d; want 3 and 2", fr.StepsPassed, fr.StepsSkipped)
}
}
Loading