From f42f3efbcc2813152eb27ae37e9c875dc7cc1511 Mon Sep 17 00:00:00 2001 From: lvladika Date: Mon, 16 Nov 2026 20:01:33 +0100 Subject: [PATCH 1/2] fix(executor): a runFlow's or a retry's file runs its own onFlowStart and onFlowComplete, as in Maestro A file reached by runFlow or retry applied only its env:; its header's onFlowStart and onFlowComplete never ran. Maestro runs them around the file's commands (Orchestra.kt runSubFlow, for runFlowCommand and retryCommand alike; the command's config is the file's header): the steps after onFlowStart, onFlowComplete in a finally. executeSubFlow now runs onFlowStart, then the steps unless it failed, then onFlowComplete, also after a failed step; the sub-flow fails when a hook fails, with the first failure, as a flow's own hooks report it. A retry runs them on each attempt. Once the run is cancelled or a runFlow's timeout: has passed, nothing more runs, onFlowComplete included (Maestro skips it on a cancel too). Unlike Maestro, the steps run when onFlowStart changed nothing on screen: runSubFlow takes executeSubflowCommands' "screen changed" result as onFlowStart's success, so a hook of only assertions skips the file's steps there. The pre-session scan for launchApp reads a sub-file's onFlowStart before its steps and its onFlowComplete after them, so a launchApp with permissions in a sub-file's hook arms the driver as one in its steps does. --- pkg/executor/flow_runner.go | 45 ++++++-- pkg/executor/flow_runner_prepare_test.go | 33 ++++++ pkg/executor/subflow_hooks_test.go | 131 +++++++++++++++++++++++ 3 files changed, 202 insertions(+), 7 deletions(-) create mode 100644 pkg/executor/subflow_hooks_test.go diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 5b5df9c5..8d555e37 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -1125,7 +1125,11 @@ func (fr *FlowRunner) collectStepsForPrepare() []flow.Step { } seen[path] = true if sub, err := flow.ParseFile(path); err == nil { + // A file run by runFlow or retry runs its own hooks around its + // steps (executeSubFlow), so a launchApp in them is scanned too. + expand(sub.Config.OnFlowStart, filepath.Dir(path), depth+1) expand(sub.Steps, filepath.Dir(path), depth+1) + expand(sub.Config.OnFlowComplete, filepath.Dir(path), depth+1) } } expand = func(steps []flow.Step, dir string, depth int) { @@ -1897,9 +1901,37 @@ func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult { return expressionFailed(err) } - // Execute steps + // The file's own onFlowStart runs before its steps and its onFlowComplete + // after them, as Maestro's runSubFlow runs the file of a runFlow or a + // retry (Orchestra.kt): a failing onFlowStart skips the steps, the steps + // failing still run onFlowComplete, and the sub-flow fails when a hook + // fails. The first failure is the sub-flow's, as for a flow's own hooks. + // Once the run is cancelled or a runFlow's timeout: has passed, + // runSubFlowSteps runs nothing more, onFlowComplete included; Maestro too + // skips it on a cancel. Maestro also skips the steps when onFlowStart + // changed nothing on screen (runSubFlow reads executeSubflowCommands' + // "screen changed" as success); here they run. + result := fr.runSubFlowSteps(subFlow, subFlow.Config.OnFlowStart, "onFlowStart failed: ") + if result == nil { + result = fr.runSubFlowSteps(subFlow, subFlow.Steps, "") + } + if done := fr.runSubFlowSteps(subFlow, subFlow.Config.OnFlowComplete, "onFlowComplete failed: "); result == nil { + result = done + } + if result != nil { + return result + } + return &core.CommandResult{ + Success: true, + Message: fmt.Sprintf("Sub-flow '%s' completed", subFlow.Config.Name), + } +} + +// runSubFlowSteps runs steps of subFlow as nested steps and returns the +// result of the first that fails it (its message after prefix), or nil. +func (fr *FlowRunner) runSubFlowSteps(subFlow flow.Flow, steps []flow.Step, prefix string) *core.CommandResult { var lastStepDesc string - for _, step := range subFlow.Steps { + for _, step := range steps { if fr.ctx.Err() != nil { msg := "Sub-flow cancelled" if lastStepDesc != "" { @@ -1935,14 +1967,13 @@ func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult { result := fr.executeNestedStep(step) if !result.Success && !optionalFailure(step, result) { + if prefix != "" { + result = &core.CommandResult{Success: false, Error: result.Error, Message: prefix + result.Message} + } return result } } - - return &core.CommandResult{ - Success: true, - Message: fmt.Sprintf("Sub-flow '%s' completed", subFlow.Config.Name), - } + return nil } // executeSubFlowWithRetry runs a sub-flow up to attempts times, until it passes. diff --git a/pkg/executor/flow_runner_prepare_test.go b/pkg/executor/flow_runner_prepare_test.go index 188152f6..4716417e 100644 --- a/pkg/executor/flow_runner_prepare_test.go +++ b/pkg/executor/flow_runner_prepare_test.go @@ -243,3 +243,36 @@ func TestCollectStepsForPrepare_NestedFileResolvesFromItsOwnDirectory(t *testing } } } + +// A file run by runFlow or retry runs its own onFlowStart and onFlowComplete +// (executeSubFlow), so the pre-session scan reads them too, in the order they +// run: a launchApp in a sub-file's onFlowStart comes before the file's steps. +func TestCollectStepsForPrepare_SubFlowFileHooks(t *testing.T) { + dir := t.TempDir() + part := "appId: x\nonFlowStart:\n - launchApp:\n permissions:\n all: deny\nonFlowComplete:\n - launchApp\n---\n- tapOn: a\n" + if err := os.WriteFile(filepath.Join(dir, "part.yaml"), []byte(part), 0o644); err != nil { + t.Fatal(err) + } + f := flow.Flow{ + SourcePath: filepath.Join(dir, "main.yaml"), + Steps: []flow.Step{ + &flow.RunFlowStep{File: "part.yaml"}, + &flow.RetryStep{File: "part.yaml"}, + &flow.TapOnStep{}, + }, + } + fr := newPrepareTestRunner(f) + defer fr.script.Close() + + steps := fr.collectStepsForPrepare() + n, perms := countLaunchApps(steps) + if n != 2 { + t.Fatalf("expected the sub-file's two hook launchApps scanned, got %d", n) + } + if perms["all"] != "deny" { + t.Errorf("expected the onFlowStart launchApp {all: deny} first, got %v", perms) + } + if _, ok := steps[0].(*flow.LaunchAppStep); !ok { + t.Errorf("expected onFlowStart's launchApp before the file's steps, first is %T", steps[0]) + } +} diff --git a/pkg/executor/subflow_hooks_test.go b/pkg/executor/subflow_hooks_test.go new file mode 100644 index 00000000..5fefb02a --- /dev/null +++ b/pkg/executor/subflow_hooks_test.go @@ -0,0 +1,131 @@ +package executor + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/core" + "github.com/devicelab-dev/maestro-runner/pkg/flow" + "github.com/devicelab-dev/maestro-runner/pkg/report" +) + +// runFiles writes files to a temp dir, parses main, runs it with cfg against +// a driver that records each step it gets, and returns the result, the steps +// the driver got and the report folder. The driver fails a tapOn on +// "Missing", and one on "Flaky" the first time only. +func runFiles(t *testing.T, files map[string]string, main string, cfg RunnerConfig) (*RunResult, []flow.Step, string) { + t.Helper() + dir := t.TempDir() + for name, body := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + f, err := flow.ParseFile(filepath.Join(dir, main)) + if err != nil { + t.Fatalf("parse: %v", err) + } + var seen []flow.Step + flaky := 0 + driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult { + seen = append(seen, step) + if tap, ok := step.(*flow.TapOnStep); ok { + switch tap.Selector.Text { + case "Missing": + return &core.CommandResult{Success: false, Error: errors.New("element not found"), Message: "Element not found: Missing"} + case "Flaky": + if flaky++; flaky == 1 { + return &core.CommandResult{Success: false, Error: errors.New("element not found"), Message: "Element not found: Flaky"} + } + } + } + return &core.CommandResult{Success: true} + }} + cfg.OutputDir = filepath.Join(dir, "report") + cfg.Artifacts = ArtifactNever + cfg.Device = report.Device{ID: "test", Platform: "android"} + result, err := New(driver, cfg).Run(context.Background(), []flow.Flow{*f}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + return result, seen, cfg.OutputDir +} + +// typed lists the taps and typed texts the driver got, in order. +func typed(steps []flow.Step) []string { + var out []string + for _, step := range steps { + switch s := step.(type) { + case *flow.TapOnStep: + out = append(out, "tapOn "+s.Selector.Text) + case *flow.InputTextStep: + out = append(out, "inputText "+s.Text) + } + } + return out +} + +// hooked is a flow file with its own onFlowStart and onFlowComplete around +// steps. +func hooked(start, steps, complete string) string { + return "appId: x\nonFlowStart:\n" + start + "onFlowComplete:\n" + complete + "---\n" + steps +} + +// A file reached by runFlow or retry runs its own onFlowStart before its +// steps and its onFlowComplete after them, as Maestro's runSubFlow runs them +// (Orchestra.kt); only its env: was applied. A failing onFlowStart skips the +// steps, failing steps still run onFlowComplete, and a failing hook fails the +// runFlow. Maestro also skips the steps when onFlowStart changed nothing on +// screen (an assertion, a failed optional step): here they run. +func TestRunFlowFileRunsItsOwnHooks(t *testing.T) { + start := " - tapOn: part start\n" + complete := " - tapOn: part end\n" + for _, c := range []struct { + name, main, part string + want report.Status + seen string + error string + }{ + {"runFlow, twice", "- runFlow: part.yaml\n- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", complete), + report.StatusPassed, "tapOn part start, tapOn inside, tapOn part end, tapOn part start, tapOn inside, tapOn part end, tapOn after", ""}, + {"retry", "- retry:\n maxRetries: 1\n file: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", complete), + report.StatusPassed, "tapOn part start, tapOn inside, tapOn part end, tapOn after", ""}, + {"a retry runs the hooks again after a failed attempt", "- retry:\n maxRetries: 2\n file: part.yaml\n- tapOn: after\n", + hooked(start, "- tapOn: Flaky\n", complete), report.StatusPassed, + "tapOn part start, tapOn Flaky, tapOn part end, tapOn part start, tapOn Flaky, tapOn part end, tapOn after", ""}, + {"a failing step still runs onFlowComplete", "- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: Missing\n- tapOn: skipped\n", complete), + report.StatusFailed, "tapOn part start, tapOn Missing, tapOn part end", "Missing"}, + {"a failing onFlowStart skips the steps", "- runFlow: part.yaml\n- tapOn: after\n", hooked(" - tapOn: Missing\n", "- tapOn: inside\n", complete), + report.StatusFailed, "tapOn Missing, tapOn part end", "onFlowStart failed"}, + {"a failing onFlowComplete fails the runFlow", "- runFlow: part.yaml\n- tapOn: after\n", hooked(start, "- tapOn: inside\n", " - tapOn: Missing\n"), + report.StatusFailed, "tapOn part start, tapOn inside, tapOn Missing", "onFlowComplete failed"}, + {"an optional hook step that fails lets the steps run", "- runFlow: part.yaml\n- tapOn: after\n", + hooked(" - tapOn:\n text: Missing\n optional: true\n", "- tapOn: inside\n", complete), report.StatusPassed, + "tapOn Missing, tapOn inside, tapOn part end, tapOn after", ""}, + {"a runFlow whose when: is false runs no hook", "- runFlow:\n file: part.yaml\n when:\n true: ${false}\n- tapOn: after\n", + hooked(start, "- tapOn: inside\n", complete), report.StatusPassed, "tapOn after", ""}, + {"inline commands have no hooks", "- runFlow:\n commands:\n - tapOn: inline\n- tapOn: after\n", "", + report.StatusPassed, "tapOn inline, tapOn after", ""}, + } { + t.Run(c.name, func(t *testing.T) { + files := map[string]string{"main.yaml": "appId: x\n---\n" + c.main} + if c.part != "" { + files["part.yaml"] = c.part + } + result, seen, _ := runFiles(t, files, "main.yaml", RunnerConfig{}) + if result.Status != c.want { + t.Fatalf("status = %s, want %s: %+v", result.Status, c.want, result.FlowResults) + } + if got := strings.Join(typed(seen), ", "); got != c.seen { + t.Errorf("the driver got\n %s\nwant\n %s", got, c.seen) + } + if c.error != "" && !strings.Contains(result.FlowResults[0].Error, c.error) { + t.Errorf("error = %q, want it to name %q", result.FlowResults[0].Error, c.error) + } + }) + } +} From 43d76105c195b8bc64b150d225fb65dd5353f545 Mon Sep 17 00:00:00 2001 From: lvladika Date: Wed, 18 Nov 2026 14:13:42 +0100 Subject: [PATCH 2/2] fix(executor): a flow file made only of runFlows is one flow, as in Maestro; suite: true splits it A test file whose steps are all runFlows of files was split into a flow per runFlow (IsSuite, expandSuites), in a sequential run only: the parallel runner never split one, so the same file gave a different result with --parallel. Maestro has no such rule. A test file is one flow and a runFlow a sub-flow of it (Orchestra.kt runFlowCommand, runSubFlow, executeSubflowCommands): what one sub-flow keeps in output the next one reads, since the output binding lives as long as the flow's JS engine; a failing sub-flow ends the test; and the test is reported under its own name. Split, the next part read output as undefined, the parts after a failure still ran, and the report and the JUnit listed the parts but not the test. The split is now asked for in the file's header, suite: true, and both runners honour it. The file says what it is, so it means the same on every run, from the command line, CI or a library caller. Maestro keeps an unknown header key (YamlConfig's @JsonAnySetter into ext), so such a file still runs there, as one flow. A file that says suite: true and cannot be split (another step, a runFlow with when:, an else or inline commands, a single runFlow) fails validation, naming the step, rather than running as one flow against what its header says. Reached by a runFlow, a file's suite: true changes nothing. A library caller that skips the validator gets such a file run as one flow, with a warning in the log. The run's log says how each runFlow-only file runs: split by its key, or one flow, with the key that would split it. This changes a default: a file that relied on the split is one test now unless its header says suite: true. The README's Flow Config section shows the key. --- README.md | 11 ++ pkg/executor/parallel.go | 2 + pkg/executor/runner.go | 18 ++- pkg/executor/suite_file_test.go | 264 ++++++++++++++++++++++++++++++++ pkg/flow/flow.go | 47 ++++-- pkg/flow/parser_test.go | 59 ++++++- pkg/validator/validator.go | 11 +- pkg/validator/validator_test.go | 42 +++++ 8 files changed, 428 insertions(+), 26 deletions(-) create mode 100644 pkg/executor/suite_file_test.go diff --git a/README.md b/README.md index b46dedd4..f1cb3fbc 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,17 @@ waitForIdleTimeout: 3000 # Device idle wait (ms), 0 to disable - assertVisible: "Welcome" ``` +A flow file is one test, as in Maestro, also when its steps are all `runFlow`s: what one sub-flow keeps in `output` the next one reads, and a failing one ends the test. To run each `runFlow` of such a file as a test of its own instead, say so in its header (Maestro keeps the key and runs the file as one flow): + +```yaml +suite: true # Each runFlow below runs as a flow of its own +--- +- runFlow: login.yaml +- runFlow: checkout.yaml +``` + +A suite holds only `runFlow` steps of files, two or more, without `when:`, an else or inline `commands:`; a file that says `suite: true` and holds anything else fails validation. Each part takes the suite's `appId` and `env`, and its `runFlow`'s `env`. A folder of flows, or `flows:` in `config.yaml`, also runs several tests together. + ## Visual Regression (`assertScreenshot`) `assertScreenshot` compares the current screen (optionally cropped with `cropOn`) against a reference PNG. diff --git a/pkg/executor/parallel.go b/pkg/executor/parallel.go index 9a0d5a73..0c500f1a 100644 --- a/pkg/executor/parallel.go +++ b/pkg/executor/parallel.go @@ -99,6 +99,8 @@ func (pr *ParallelRunner) Run(ctx context.Context, flows []flow.Flow) (*RunResul if len(pr.workers) == 0 { return nil, fmt.Errorf("no workers available") } + // Split suite files (suite: true) as the sequential runner does + flows = expandSuites(flows) // Read the previous run's timings before the skeleton below overwrites // the report this reads. diff --git a/pkg/executor/runner.go b/pkg/executor/runner.go index d8038d86..e63e0350 100644 --- a/pkg/executor/runner.go +++ b/pkg/executor/runner.go @@ -141,7 +141,7 @@ func New(driver core.Driver, cfg RunnerConfig) *Runner { // Run executes all flows and generates reports. func (r *Runner) Run(ctx context.Context, flows []flow.Flow) (*RunResult, error) { - // Expand suites into individual test case flows + // Split suite files (suite: true) into their test case flows expandedFlows := expandSuites(flows) // Build report skeleton @@ -353,20 +353,28 @@ func (r *Runner) buildRunResult(flowResults []FlowResult) *RunResult { } // expandSuites expands suite files into individual test case flows. -// A suite is a flow that only contains runFlow steps - each runFlow becomes a separate flow. -// Non-suite flows pass through unchanged. +// A suite is a flow whose header says `suite: true` and that only contains runFlow steps of files +// (flow.IsSuite) - each runFlow becomes a separate flow. Other flows pass through unchanged: a flow made +// only of runFlow steps without the key is one flow, as in Maestro. Each case is logged, so a run that +// splits a file, or does not, says why. func expandSuites(flows []flow.Flow) []flow.Flow { var expanded []flow.Flow for _, f := range flows { if !f.IsSuite() { - // Not a suite, keep as-is + switch problem := f.SuiteProblem(); { + case f.Config.Suite: + logger.Warn("%s has suite: true, but %s: running it as one flow", f.SourcePath, problem) + case problem == "": + logger.Info("%s is made only of runFlow steps: running it as one flow (suite: true in its header runs each as a flow of its own)", f.SourcePath) + } expanded = append(expanded, f) continue } - // Suite detected - expand each runFlow into a separate flow + // Suite - expand each runFlow into a separate flow testCases := f.GetTestCases() + logger.Info("%s has suite: true: running its %d runFlow files as separate flows", f.SourcePath, len(testCases)) suiteDir := "" if f.SourcePath != "" { suiteDir = filepath.Dir(f.SourcePath) diff --git a/pkg/executor/suite_file_test.go b/pkg/executor/suite_file_test.go new file mode 100644 index 00000000..52fd0d9c --- /dev/null +++ b/pkg/executor/suite_file_test.go @@ -0,0 +1,264 @@ +package executor + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/core" + "github.com/devicelab-dev/maestro-runner/pkg/flow" + "github.com/devicelab-dev/maestro-runner/pkg/logger" + "github.com/devicelab-dev/maestro-runner/pkg/report" +) + +// A test made of runFlows of other files: the first logs in and keeps the +// user in output, the second reads it. +var suiteFiles = map[string]string{ + "checkout.yaml": `appId: x +--- +- runFlow: login.yaml +- runFlow: + file: cart.yaml + env: + ITEM: socks +`, + "login.yaml": `appId: x +env: + USER: alice +--- +- evalScript: ${output.user = USER} +- tapOn: Log in +`, + "cart.yaml": `appId: x +--- +- inputText: ${output.user + ' buys ' + ITEM} +`, +} + +func readJSON(t *testing.T, path string) map[string]any { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatalf("%s: %v", path, err) + } + return doc +} + +func keys(m map[string]any) string { + var out []string + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return strings.Join(out, ",") +} + +// Maestro runs a test file as one flow, whatever its steps: a runFlow's +// steps run nested in it (Orchestra.kt runFlowCommand, runSubFlow), what one +// sub-flow keeps in output the next one reads (the JS engine's output +// binding lives as long as the flow), and the file is one test in the +// reports. A file made only of runFlows was split into a flow per runFlow, +// so the second sub-flow read output.user as undefined and the test never +// came back under its own name. +func TestRunFlowOnlyFileIsOneFlow(t *testing.T) { + result, seen, out := runFiles(t, suiteFiles, "checkout.yaml", RunnerConfig{}) + if result.Status != report.StatusPassed || result.TotalFlows != 1 { + t.Fatalf("status = %s with %d flows, want passed with 1: %+v", result.Status, result.TotalFlows, result.FlowResults) + } + want := "tapOn Log in, inputText alice buys socks" + if got := strings.Join(typed(seen), ", "); got != want { + t.Errorf("the driver got %q, want %q", got, want) + } + + // The report a reader takes: one flow under the test's name and path, + // its runFlows with their steps nested. + index := readJSON(t, filepath.Join(out, "report.json")) + flows, _ := index["flows"].([]any) + if len(flows) != 1 { + t.Fatalf("report.json has %d flows, want 1", len(flows)) + } + entry := flows[0].(map[string]any) + if entry["name"] != "checkout" || !strings.HasSuffix(entry["sourceFile"].(string), "checkout.yaml") || entry["status"] != "passed" { + t.Errorf("report.json flow = name %v, sourceFile %v, status %v; want checkout, checkout.yaml, passed", + entry["name"], entry["sourceFile"], entry["status"]) + } + detail := readJSON(t, filepath.Join(out, entry["dataFile"].(string))) + commands, _ := detail["commands"].([]any) + if len(commands) != 2 { + t.Fatalf("the flow has %d commands, want its 2 runFlows", len(commands)) + } + for i, wantSubs := range []int{2, 1} { + cmd := commands[i].(map[string]any) + subs, _ := cmd["subCommands"].([]any) + if cmd["type"] != "runFlow" || cmd["status"] != "passed" || len(subs) != wantSubs { + t.Errorf("command %d = %v %v with %d sub-commands, want a passed runFlow with %d", i, cmd["type"], cmd["status"], len(subs), wantSubs) + } + } + junit, err := os.ReadFile(filepath.Join(out, "junit-report.xml")) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(string(junit), " 0 { - return false + switch { + case !ok || rf.File == "": + return fmt.Sprintf("step %d is a runFlow with inline commands", i+1) + case rf.When != nil: + return fmt.Sprintf("step %d is a runFlow with a when: condition", i+1) + case rf.ElseFile != "" || len(rf.ElseSteps) > 0: + return fmt.Sprintf("step %d is a runFlow with an else", i+1) } runFlowWithFileCount++ } - // Must have at least 2 runFlow steps with file references to be a suite - // A single runFlow is just a wrapper, not a test suite - return runFlowWithFileCount >= 2 + if runFlowWithFileCount < 2 { + return "it has only one runFlow" + } + return "" } // GetTestCases returns the runFlow steps if this is a suite, empty slice otherwise. @@ -79,6 +91,7 @@ type Config struct { StepDelay *int `yaml:"stepDelay"` // Pause between top-level steps in ms (nil = use global, 0 = none) Properties map[string]string `yaml:"properties"` // Custom key/values emitted as JUnit testcase properties (#84) FailOnConsoleError bool `yaml:"failOnConsoleError"` // Web only: auto-fail the flow if any console.error / uncaught JS exception is captured + Suite bool `yaml:"suite"` // Run each runFlow as a flow of its own (see Flow.IsSuite); Maestro keeps the key and runs the file as one flow OnFlowStart []Step `yaml:"-"` // Lifecycle hook: runs before commands OnFlowComplete []Step `yaml:"-"` // Lifecycle hook: runs after commands } diff --git a/pkg/flow/parser_test.go b/pkg/flow/parser_test.go index 9607155c..eb4566e1 100644 --- a/pkg/flow/parser_test.go +++ b/pkg/flow/parser_test.go @@ -2142,14 +2142,66 @@ func TestFlow_IsSuite(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := tt.flow.IsSuite() - if got != tt.expected { - t.Errorf("IsSuite() = %v, want %v", got, tt.expected) + // Without suite: true in the header no flow is a suite, whatever + // its steps: it runs as one flow, as in Maestro. + if tt.flow.IsSuite() { + t.Errorf("IsSuite() = true without suite: true") + } + f := tt.flow + f.Config.Suite = true + if got := f.IsSuite(); got != tt.expected { + t.Errorf("IsSuite() with suite: true = %v, want %v", got, tt.expected) + } + if problem := f.SuiteProblem(); (problem == "") != tt.expected { + t.Errorf("SuiteProblem() = %q, want a reason exactly when it is not a suite", problem) } }) } } +// SuiteProblem names the step that keeps a file from being a suite, for the +// validator's error. +func TestFlow_SuiteProblem(t *testing.T) { + runFlow := func(file string) Step { return &RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, File: file} } + tests := []struct { + steps []Step + want string + }{ + {nil, "it has no steps"}, + {[]Step{runFlow("a.yaml"), &TapOnStep{BaseStep: BaseStep{StepType: StepTapOn}}}, "step 2 is tapOn, not a runFlow"}, + {[]Step{runFlow("a.yaml"), &RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, Steps: []Step{&TapOnStep{}}}}, "step 2 is a runFlow with inline commands"}, + {[]Step{&RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, File: "a.yaml", When: &Condition{Visible: &Selector{Text: "Menu"}}}, runFlow("b.yaml")}, "step 1 is a runFlow with a when: condition"}, + {[]Step{runFlow("a.yaml"), &RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, File: "b.yaml", ElseFile: "c.yaml"}}, "step 2 is a runFlow with an else"}, + {[]Step{runFlow("a.yaml")}, "it has only one runFlow"}, + {[]Step{runFlow("a.yaml"), runFlow("b.yaml")}, ""}, + } + for _, tt := range tests { + f := Flow{Steps: tt.steps} + if got := f.SuiteProblem(); got != tt.want { + t.Errorf("SuiteProblem() = %q, want %q", got, tt.want) + } + } +} + +// The header key parses; a file without it is no suite. +func TestParse_SuiteHeader(t *testing.T) { + body := "appId: com.example\nsuite: true\n---\n- runFlow: a.yaml\n- runFlow: b.yaml\n" + f, err := Parse([]byte(body), "suite.yaml") + if err != nil { + t.Fatal(err) + } + if !f.Config.Suite || !f.IsSuite() { + t.Errorf("Config.Suite = %v, IsSuite() = %v; want both true", f.Config.Suite, f.IsSuite()) + } + f, err = Parse([]byte(strings.Replace(body, "suite: true\n", "", 1)), "suite.yaml") + if err != nil { + t.Fatal(err) + } + if f.Config.Suite || f.IsSuite() { + t.Errorf("without the key: Config.Suite = %v, IsSuite() = %v; want both false", f.Config.Suite, f.IsSuite()) + } +} + func TestFlow_GetTestCases(t *testing.T) { // Non-suite should return nil nonSuite := Flow{ @@ -2163,6 +2215,7 @@ func TestFlow_GetTestCases(t *testing.T) { // Suite should return runFlow steps suite := Flow{ + Config: Config{Suite: true}, Steps: []Step{ &RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, File: "login.yaml"}, &RunFlowStep{BaseStep: BaseStep{StepType: StepRunFlow}, File: "checkout.yaml"}, diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index abdca65a..027d9c07 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -448,8 +448,17 @@ func (v *Validator) validateFile(filePath string, result *Result, validated map[ } // Check tag filters if flow.ShouldIncludeFlow(f, v.includeTags, v.excludeTags) { - result.TestCases = append(result.TestCases, filePath) testCasesAdded[filePath] = true + // A file that says it is a suite and cannot be split would run + // as one flow against what its header says. + if problem := f.SuiteProblem(); f.Config.Suite && problem != "" { + result.Errors = append(result.Errors, &ValidationError{ + File: filePath, + Message: fmt.Sprintf("suite: true, but %s; a suite has only runFlow steps of files, two or more, without when:, else or inline commands", problem), + }) + return + } + result.TestCases = append(result.TestCases, filePath) } } } diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index 1f0443c3..6c438a1e 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -1546,3 +1546,45 @@ func writeFile(t *testing.T, path, body string) { t.Fatal(err) } } + +// suite: true says the file's runFlows run as flows of their own. A file +// that cannot be split that way is refused before the run, naming the step +// that stops it; one that can is a test case like any other. +func TestValidate_SuiteKey(t *testing.T) { + dir := t.TempDir() + files := map[string]string{ + "suite.yaml": "appId: x\nsuite: true\n---\n- runFlow: a.yaml\n- runFlow: b.yaml\n", + "broken.yaml": "appId: x\nsuite: true\n---\n- runFlow: a.yaml\n- runFlow: b.yaml\n- tapOn: Done\n", + "a.yaml": "appId: x\n---\n- tapOn: A\n", + "b.yaml": "appId: x\n---\n- tapOn: B\n", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + result := New(nil, nil).Validate(filepath.Join(dir, "suite.yaml")) + if !result.IsValid() || len(result.TestCases) != 1 { + t.Errorf("suite.yaml: errors %v, %d test cases; want valid with 1", result.Errors, len(result.TestCases)) + } + + result = New(nil, nil).Validate(filepath.Join(dir, "broken.yaml")) + if result.IsValid() || len(result.TestCases) != 0 { + t.Fatalf("broken.yaml: errors %v, %d test cases; want an error and none", result.Errors, len(result.TestCases)) + } + want := "suite: true, but step 3 is tapOn, not a runFlow; a suite has only runFlow steps of files, two or more, without when:, else or inline commands" + if got := result.Errors[0].Error(); !strings.HasSuffix(got, want) { + t.Errorf("error = %q, want it to end %q", got, want) + } + + // Reached by runFlow, a file's suite: true changes nothing: it runs + // nested, as any runFlow's file does. + if err := os.WriteFile(filepath.Join(dir, "main.yaml"), []byte("appId: x\n---\n- runFlow: broken.yaml\n- tapOn: C\n"), 0o644); err != nil { + t.Fatal(err) + } + result = New(nil, nil).Validate(filepath.Join(dir, "main.yaml")) + if !result.IsValid() { + t.Errorf("main.yaml: errors %v, want none", result.Errors) + } +}