Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,17 @@ waitForIdleTimeout: 3000 # Device idle wait (ms), 0 to disable
- assertVisible: "Welcome"
```

A flow file is one test, as in Maestro, also when its steps are all `runFlow`s: what one sub-flow keeps in `output` the next one reads, and a failing one ends the test. To run each `runFlow` of such a file as a test of its own instead, say so in its header (Maestro keeps the key and runs the file as one flow):

```yaml
suite: true # Each runFlow below runs as a flow of its own
---
- runFlow: login.yaml
- runFlow: checkout.yaml
```

A suite holds only `runFlow` steps of files, two or more, without `when:`, an else or inline `commands:`; a file that says `suite: true` and holds anything else fails validation. Each part takes the suite's `appId` and `env`, and its `runFlow`'s `env`. A folder of flows, or `flows:` in `config.yaml`, also runs several tests together.

## Visual Regression (`assertScreenshot`)

`assertScreenshot` compares the current screen (optionally cropped with `cropOn`) against a reference PNG.
Expand Down
45 changes: 38 additions & 7 deletions pkg/executor/flow_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -1125,7 +1125,11 @@ func (fr *FlowRunner) collectStepsForPrepare() []flow.Step {
}
seen[path] = true
if sub, err := flow.ParseFile(path); err == nil {
// A file run by runFlow or retry runs its own hooks around its
// steps (executeSubFlow), so a launchApp in them is scanned too.
expand(sub.Config.OnFlowStart, filepath.Dir(path), depth+1)
expand(sub.Steps, filepath.Dir(path), depth+1)
expand(sub.Config.OnFlowComplete, filepath.Dir(path), depth+1)
}
}
expand = func(steps []flow.Step, dir string, depth int) {
Expand Down Expand Up @@ -1897,9 +1901,37 @@ func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult {
return expressionFailed(err)
}

// Execute steps
// The file's own onFlowStart runs before its steps and its onFlowComplete
// after them, as Maestro's runSubFlow runs the file of a runFlow or a
// retry (Orchestra.kt): a failing onFlowStart skips the steps, the steps
// failing still run onFlowComplete, and the sub-flow fails when a hook
// fails. The first failure is the sub-flow's, as for a flow's own hooks.
// Once the run is cancelled or a runFlow's timeout: has passed,
// runSubFlowSteps runs nothing more, onFlowComplete included; Maestro too
// skips it on a cancel. Maestro also skips the steps when onFlowStart
// changed nothing on screen (runSubFlow reads executeSubflowCommands'
// "screen changed" as success); here they run.
result := fr.runSubFlowSteps(subFlow, subFlow.Config.OnFlowStart, "onFlowStart failed: ")
if result == nil {
result = fr.runSubFlowSteps(subFlow, subFlow.Steps, "")
}
if done := fr.runSubFlowSteps(subFlow, subFlow.Config.OnFlowComplete, "onFlowComplete failed: "); result == nil {
result = done
}
if result != nil {
return result
}
return &core.CommandResult{
Success: true,
Message: fmt.Sprintf("Sub-flow '%s' completed", subFlow.Config.Name),
}
}

// runSubFlowSteps runs steps of subFlow as nested steps and returns the
// result of the first that fails it (its message after prefix), or nil.
func (fr *FlowRunner) runSubFlowSteps(subFlow flow.Flow, steps []flow.Step, prefix string) *core.CommandResult {
var lastStepDesc string
for _, step := range subFlow.Steps {
for _, step := range steps {
if fr.ctx.Err() != nil {
msg := "Sub-flow cancelled"
if lastStepDesc != "" {
Expand Down Expand Up @@ -1935,14 +1967,13 @@ func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult {

result := fr.executeNestedStep(step)
if !result.Success && !optionalFailure(step, result) {
if prefix != "" {
result = &core.CommandResult{Success: false, Error: result.Error, Message: prefix + result.Message}
}
return result
}
}

return &core.CommandResult{
Success: true,
Message: fmt.Sprintf("Sub-flow '%s' completed", subFlow.Config.Name),
}
return nil
}

// executeSubFlowWithRetry runs a sub-flow up to attempts times, until it passes.
Expand Down
33 changes: 33 additions & 0 deletions pkg/executor/flow_runner_prepare_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,3 +243,36 @@ func TestCollectStepsForPrepare_NestedFileResolvesFromItsOwnDirectory(t *testing
}
}
}

// A file run by runFlow or retry runs its own onFlowStart and onFlowComplete
// (executeSubFlow), so the pre-session scan reads them too, in the order they
// run: a launchApp in a sub-file's onFlowStart comes before the file's steps.
func TestCollectStepsForPrepare_SubFlowFileHooks(t *testing.T) {
dir := t.TempDir()
part := "appId: x\nonFlowStart:\n - launchApp:\n permissions:\n all: deny\nonFlowComplete:\n - launchApp\n---\n- tapOn: a\n"
if err := os.WriteFile(filepath.Join(dir, "part.yaml"), []byte(part), 0o644); err != nil {
t.Fatal(err)
}
f := flow.Flow{
SourcePath: filepath.Join(dir, "main.yaml"),
Steps: []flow.Step{
&flow.RunFlowStep{File: "part.yaml"},
&flow.RetryStep{File: "part.yaml"},
&flow.TapOnStep{},
},
}
fr := newPrepareTestRunner(f)
defer fr.script.Close()

steps := fr.collectStepsForPrepare()
n, perms := countLaunchApps(steps)
if n != 2 {
t.Fatalf("expected the sub-file's two hook launchApps scanned, got %d", n)
}
if perms["all"] != "deny" {
t.Errorf("expected the onFlowStart launchApp {all: deny} first, got %v", perms)
}
if _, ok := steps[0].(*flow.LaunchAppStep); !ok {
t.Errorf("expected onFlowStart's launchApp before the file's steps, first is %T", steps[0])
}
}
2 changes: 2 additions & 0 deletions pkg/executor/parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ func (pr *ParallelRunner) Run(ctx context.Context, flows []flow.Flow) (*RunResul
if len(pr.workers) == 0 {
return nil, fmt.Errorf("no workers available")
}
// Split suite files (suite: true) as the sequential runner does
flows = expandSuites(flows)

// Read the previous run's timings before the skeleton below overwrites
// the report this reads.
Expand Down
18 changes: 13 additions & 5 deletions pkg/executor/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func New(driver core.Driver, cfg RunnerConfig) *Runner {

// Run executes all flows and generates reports.
func (r *Runner) Run(ctx context.Context, flows []flow.Flow) (*RunResult, error) {
// Expand suites into individual test case flows
// Split suite files (suite: true) into their test case flows
expandedFlows := expandSuites(flows)

// Build report skeleton
Expand Down Expand Up @@ -353,20 +353,28 @@ func (r *Runner) buildRunResult(flowResults []FlowResult) *RunResult {
}

// expandSuites expands suite files into individual test case flows.
// A suite is a flow that only contains runFlow steps - each runFlow becomes a separate flow.
// Non-suite flows pass through unchanged.
// A suite is a flow whose header says `suite: true` and that only contains runFlow steps of files
// (flow.IsSuite) - each runFlow becomes a separate flow. Other flows pass through unchanged: a flow made
// only of runFlow steps without the key is one flow, as in Maestro. Each case is logged, so a run that
// splits a file, or does not, says why.
func expandSuites(flows []flow.Flow) []flow.Flow {
var expanded []flow.Flow

for _, f := range flows {
if !f.IsSuite() {
// Not a suite, keep as-is
switch problem := f.SuiteProblem(); {
case f.Config.Suite:
logger.Warn("%s has suite: true, but %s: running it as one flow", f.SourcePath, problem)
case problem == "":
logger.Info("%s is made only of runFlow steps: running it as one flow (suite: true in its header runs each as a flow of its own)", f.SourcePath)
}
expanded = append(expanded, f)
continue
}

// Suite detected - expand each runFlow into a separate flow
// Suite - expand each runFlow into a separate flow
testCases := f.GetTestCases()
logger.Info("%s has suite: true: running its %d runFlow files as separate flows", f.SourcePath, len(testCases))
suiteDir := ""
if f.SourcePath != "" {
suiteDir = filepath.Dir(f.SourcePath)
Expand Down
131 changes: 131 additions & 0 deletions pkg/executor/subflow_hooks_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package executor

import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"

"github.com/devicelab-dev/maestro-runner/pkg/core"
"github.com/devicelab-dev/maestro-runner/pkg/flow"
"github.com/devicelab-dev/maestro-runner/pkg/report"
)

// runFiles writes files to a temp dir, parses main, runs it with cfg against
// a driver that records each step it gets, and returns the result, the steps
// the driver got and the report folder. The driver fails a tapOn on
// "Missing", and one on "Flaky" the first time only.
func runFiles(t *testing.T, files map[string]string, main string, cfg RunnerConfig) (*RunResult, []flow.Step, string) {
t.Helper()
dir := t.TempDir()
for name, body := range files {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
f, err := flow.ParseFile(filepath.Join(dir, main))
if err != nil {
t.Fatalf("parse: %v", err)
}
var seen []flow.Step
flaky := 0
driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult {
seen = append(seen, step)
if tap, ok := step.(*flow.TapOnStep); ok {
switch tap.Selector.Text {
case "Missing":
return &core.CommandResult{Success: false, Error: errors.New("element not found"), Message: "Element not found: Missing"}
case "Flaky":
if flaky++; flaky == 1 {
return &core.CommandResult{Success: false, Error: errors.New("element not found"), Message: "Element not found: Flaky"}
}
}
}
return &core.CommandResult{Success: true}
}}
cfg.OutputDir = filepath.Join(dir, "report")
cfg.Artifacts = ArtifactNever
cfg.Device = report.Device{ID: "test", Platform: "android"}
result, err := New(driver, cfg).Run(context.Background(), []flow.Flow{*f})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
return result, seen, cfg.OutputDir
}

// typed lists the taps and typed texts the driver got, in order.
func typed(steps []flow.Step) []string {
var out []string
for _, step := range steps {
switch s := step.(type) {
case *flow.TapOnStep:
out = append(out, "tapOn "+s.Selector.Text)
case *flow.InputTextStep:
out = append(out, "inputText "+s.Text)
}
}
return out
}

// hooked is a flow file with its own onFlowStart and onFlowComplete around
// steps.
func hooked(start, steps, complete string) string {
return "appId: x\nonFlowStart:\n" + start + "onFlowComplete:\n" + complete + "---\n" + steps
}

// A file reached by runFlow or retry runs its own onFlowStart before its
// steps and its onFlowComplete after them, as Maestro's runSubFlow runs them
// (Orchestra.kt); only its env: was applied. A failing onFlowStart skips the
// steps, failing steps still run onFlowComplete, and a failing hook fails the
// runFlow. Maestro also skips the steps when onFlowStart changed nothing on
// screen (an assertion, a failed optional step): here they run.
func TestRunFlowFileRunsItsOwnHooks(t *testing.T) {
start := " - tapOn: part start\n"
complete := " - tapOn: part end\n"
for _, c := range []struct {
name, main, part string
want report.Status
seen string
error string
}{
{"runFlow, twice", "- runFlow: part.yaml\n- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", complete),
report.StatusPassed, "tapOn part start, tapOn inside, tapOn part end, tapOn part start, tapOn inside, tapOn part end, tapOn after", ""},
{"retry", "- retry:\n maxRetries: 1\n file: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", complete),
report.StatusPassed, "tapOn part start, tapOn inside, tapOn part end, tapOn after", ""},
{"a retry runs the hooks again after a failed attempt", "- retry:\n maxRetries: 2\n file: part.yaml\n- tapOn: after\n",
hooked(start, "- tapOn: Flaky\n", complete), report.StatusPassed,
"tapOn part start, tapOn Flaky, tapOn part end, tapOn part start, tapOn Flaky, tapOn part end, tapOn after", ""},
{"a failing step still runs onFlowComplete", "- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: Missing\n- tapOn: skipped\n", complete),
report.StatusFailed, "tapOn part start, tapOn Missing, tapOn part end", "Missing"},
{"a failing onFlowStart skips the steps", "- runFlow: part.yaml\n- tapOn: after\n", hooked(" - tapOn: Missing\n", "- tapOn: inside\n", complete),
report.StatusFailed, "tapOn Missing, tapOn part end", "onFlowStart failed"},
{"a failing onFlowComplete fails the runFlow", "- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", " - tapOn: Missing\n"),
report.StatusFailed, "tapOn part start, tapOn inside, tapOn Missing", "onFlowComplete failed"},
{"an optional hook step that fails lets the steps run", "- runFlow: part.yaml\n- tapOn: after\n",
hooked(" - tapOn:\n text: Missing\n optional: true\n", "- tapOn: inside\n", complete), report.StatusPassed,
"tapOn Missing, tapOn inside, tapOn part end, tapOn after", ""},
{"a runFlow whose when: is false runs no hook", "- runFlow:\n file: part.yaml\n when:\n true: ${false}\n- tapOn: after\n",
hooked(start, "- tapOn: inside\n", complete), report.StatusPassed, "tapOn after", ""},
{"inline commands have no hooks", "- runFlow:\n commands:\n - tapOn: inline\n- tapOn: after\n", "",
report.StatusPassed, "tapOn inline, tapOn after", ""},
} {
t.Run(c.name, func(t *testing.T) {
files := map[string]string{"main.yaml": "appId: x\n---\n" + c.main}
if c.part != "" {
files["part.yaml"] = c.part
}
result, seen, _ := runFiles(t, files, "main.yaml", RunnerConfig{})
if result.Status != c.want {
t.Fatalf("status = %s, want %s: %+v", result.Status, c.want, result.FlowResults)
}
if got := strings.Join(typed(seen), ", "); got != c.seen {
t.Errorf("the driver got\n %s\nwant\n %s", got, c.seen)
}
if c.error != "" && !strings.Contains(result.FlowResults[0].Error, c.error) {
t.Errorf("error = %q, want it to name %q", result.FlowResults[0].Error, c.error)
}
})
}
}
Loading
Loading