diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 5b5df9c5..d9bb871a 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -448,9 +448,15 @@ type expressionError struct{ error } func (e expressionError) Unwrap() error { return e.error } // expressionFailed is the result of a step whose ${...} threw: the step fails -// with the JavaScript error and does not run, as in Maestro. +// with the JavaScript error and does not run, as in Maestro. A timeout whose +// ${...} is not a number fails it the same way, but an optional step's +// failure lets the flow go on, as Maestro's InvalidCommand does (a warning +// there): see numberError. func expressionFailed(err error) *core.CommandResult { - err = expressionError{err} + var number numberError + if !errors.As(err, &number) || !number.optionalForgives { + err = expressionError{err} + } return &core.CommandResult{Success: false, Error: err, Message: err.Error()} } diff --git a/pkg/executor/number_expression_test.go b/pkg/executor/number_expression_test.go new file mode 100644 index 00000000..3736c355 --- /dev/null +++ b/pkg/executor/number_expression_test.go @@ -0,0 +1,187 @@ +package executor + +import ( + "fmt" + "strings" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/flow" + "github.com/devicelab-dev/maestro-runner/pkg/report" +) + +// timeoutsSeen lists the timeout (and a scrollUntilVisible's speed) of each +// step the driver got, in order. +func timeoutsSeen(seen []flow.Step) []string { + var out []string + for _, step := range seen { + switch s := step.(type) { + case *flow.WaitUntilStep: + out = append(out, fmt.Sprintf("wait %d", s.TimeoutMs)) + case *flow.WaitForAnimationToEndStep: + out = append(out, fmt.Sprintf("animation %d", s.TimeoutMs)) + case *flow.ScrollUntilVisibleStep: + out = append(out, fmt.Sprintf("scroll %d speed %d", s.TimeoutMs, s.Speed)) + } + } + return out +} + +// Maestro reads extendedWaitUntil's, waitForAnimationToEnd's and +// scrollUntilVisible's timeout, and scrollUntilVisible's speed, as text and +// evaluates them with the step's other scripts (Commands.kt evaluateScripts). +// The step gets the value its variables hold when it runs: at the top level, +// in a sub-flow run twice with different env, and in each pass of a repeat. +func TestNumberFieldExpressionsReachTheDriver(t *testing.T) { + result, seen := runFlowYAML(t, map[string]string{ + "main.yaml": `appId: x +env: + PAUSE_MS: 3000 + SPEED: 40 + PADDED: " 1500 " +--- +- extendedWaitUntil: + visible: Done + timeout: ${PAUSE_MS} +- waitForAnimationToEnd: + timeout: ${PAUSE_MS * 2} +- scrollUntilVisible: + element: Row + timeout: ${PAUSE_MS} + speed: ${SPEED} +- runFlow: + file: wait.yaml + env: + WAIT_MS: 1000 +- runFlow: + file: wait.yaml + env: + WAIT_MS: 2000 +- evalScript: ${output.n = 0} +- repeat: + times: 2 + commands: + - evalScript: ${output.n = output.n + 1} + - waitForAnimationToEnd: + timeout: ${output.n * 500} +- extendedWaitUntil: + visible: Done + timeout: ${PADDED} +`, + "wait.yaml": `appId: x +--- +- extendedWaitUntil: + notVisible: Spinner + timeout: ${WAIT_MS} +`, + }) + if result.Status != report.StatusPassed { + t.Fatalf("status = %s, want passed: %+v", result.Status, result.FlowResults) + } + want := []string{ + "wait 3000", + "animation 6000", + "scroll 3000 speed 40", + "wait 1000", + "wait 2000", + "animation 500", + "animation 1000", + "wait 1500", + } + got := timeoutsSeen(seen) + if strings.Join(got, ", ") != strings.Join(want, ", ") { + t.Errorf("the driver got\n %s\nwant\n %s", strings.Join(got, ", "), strings.Join(want, ", ")) + } +} + +// A value that is not a number fails the step, which does not run. Maestro +// lets an optional extendedWaitUntil or scrollUntilVisible go on (a warning +// there: its timeout error is an InvalidCommand; here the step is reported +// failed and the flow continues), but never a bad scroll +// speed (InvalidNumericFieldValue) or waitForAnimationToEnd timeout (a +// NumberFormatException): Orchestra.kt executeCommands. +func TestNumberFieldExpressionThatIsNotANumber(t *testing.T) { + for _, c := range []struct { + name, step string + want report.Status + error string + header string + }{ + {"extendedWaitUntil", `- extendedWaitUntil: + visible: Done + timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon", not a number of milliseconds`, ""}, + {"optional extendedWaitUntil", `- extendedWaitUntil: + visible: Done + timeout: ${LATER} + optional: true`, report.StatusPassed, "", ""}, + {"optional scrollUntilVisible timeout", `- scrollUntilVisible: + element: Row + timeout: ${LATER} + optional: true`, report.StatusPassed, "", ""}, + {"optional waitForAnimationToEnd", `- waitForAnimationToEnd: + timeout: ${LATER} + optional: true`, report.StatusFailed, `waitForAnimationToEnd timeout: ${LATER} is "soon", not a number of milliseconds`, ""}, + {"optional scrollUntilVisible speed", `- scrollUntilVisible: + element: Row + speed: ${FAST} + optional: true`, report.StatusFailed, `scrollUntilVisible speed: ${FAST} is "400", not a number between 0 and 100`, ""}, + // Maestro evaluates the speed first, so its error is the step's. + {"optional scrollUntilVisible, speed and timeout both bad", `- scrollUntilVisible: + element: Row + timeout: ${LATER} + speed: ${FAST} + optional: true`, report.StatusFailed, `scrollUntilVisible speed: ${FAST} is "400"`, ""}, + // A ${...} that throws fails even an optional step (JsEvaluationException). + {"optional extendedWaitUntil whose script throws", `- extendedWaitUntil: + visible: Done + timeout: ${output.wait.ms} + optional: true`, report.StatusFailed, "TypeError", ""}, + {"extendedWaitUntil in a runFlow", `- runFlow: + commands: + - extendedWaitUntil: + visible: Done + timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`, ""}, + {"optional extendedWaitUntil in a runFlow", `- runFlow: + commands: + - extendedWaitUntil: + visible: Done + timeout: ${LATER} + optional: true`, report.StatusPassed, "", ""}, + {"waitForAnimationToEnd in a repeat", `- repeat: + times: 2 + commands: + - waitForAnimationToEnd: + timeout: ${LATER}`, report.StatusFailed, `waitForAnimationToEnd timeout: ${LATER} is "soon"`, ""}, + {"optional extendedWaitUntil in a repeat", `- repeat: + times: 2 + commands: + - extendedWaitUntil: + visible: Done + timeout: ${LATER} + optional: true`, report.StatusPassed, "", ""}, + {"extendedWaitUntil in a retry", `- retry: + maxRetries: 2 + commands: + - extendedWaitUntil: + visible: Done + timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`, ""}, + {"extendedWaitUntil in onFlowStart", `- tapOn: Next`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`, + "onFlowStart:\n - extendedWaitUntil:\n visible: Done\n timeout: ${LATER}\n"}, + } { + t.Run(c.name, func(t *testing.T) { + result, seen := runFlowYAML(t, map[string]string{ + "main.yaml": "appId: x\nenv:\n LATER: soon\n FAST: 400\n" + c.header + "---\n" + c.step + "\n", + }) + if result.Status != c.want { + t.Fatalf("status = %s, want %s: %+v", result.Status, c.want, result.FlowResults) + } + if c.error != "" && !strings.Contains(result.FlowResults[0].Error, c.error) { + t.Errorf("error = %q, want %q", result.FlowResults[0].Error, c.error) + } + for _, step := range seen { + if _, tap := step.(*flow.TapOnStep); !tap { + t.Errorf("the driver got %s; want no step with the bad value", step.Describe()) + } + } + }) + } +} diff --git a/pkg/executor/scripting.go b/pkg/executor/scripting.go index 6ff775c7..20db9ec8 100644 --- a/pkg/executor/scripting.go +++ b/pkg/executor/scripting.go @@ -846,6 +846,85 @@ func (x *expansion) text(s string) string { return out } +// numberError is a number field whose ${...} resolved to something that is +// not a number the field takes. The step fails with it without running. +// optionalForgives is true where Maestro lets an optional step's failure go +// (a warning there; here the step is reported failed and the flow goes on): +// its timeout error is an InvalidCommand, which optional downgrades; a bad +// scroll speed is an InvalidNumericFieldValue, which it never does, and +// waitForAnimationToEnd's timeout fails as a NumberFormatException +// (Orchestra.kt executeCommands, Commands.kt). +type numberError struct { + msg string + optionalForgives bool +} + +func (e numberError) Error() string { return e.msg } + +// number expands a number field's ${...} expression and returns its value, +// trimmed. ok is false when the expression threw; the error is kept. +func (x *expansion) number(expr string) (string, bool) { + out, err := x.se.expand(expr) + if err != nil { + if x.err == nil { + x.err = err + } + return "", false + } + return strings.TrimSpace(out), true +} + +func (x *expansion) fail(err error) { + if x.err == nil { + x.err = err + } +} + +// timeout resolves a `timeout:` written as a ${...} expression into +// TimeoutMs, read as Maestro's parseTimeoutMs reads it: a whole number of +// milliseconds, `_` allowed (Maestro's extendedWaitUntil drops it; its +// scrollUntilVisible and waitForAnimationToEnd fail on it, so there this is +// lenient), surrounding spaces trimmed. A non-positive one is then the +// step's default, as when it is written as a literal. The expression is +// kept, so the step resolves it again each time it runs. +func (x *expansion) timeout(b *flow.BaseStep, optionalForgives bool) { + if b.TimeoutExpr == "" { + return + } + v, ok := x.number(b.TimeoutExpr) + if !ok { + return + } + ms, err := strconv.Atoi(strings.ReplaceAll(v, "_", "")) + if err != nil { + x.fail(numberError{ + msg: fmt.Sprintf("%s timeout: %s is %q, not a number of milliseconds", b.StepType, b.TimeoutExpr, v), + optionalForgives: optionalForgives, + }) + return + } + b.TimeoutMs = ms +} + +// scrollSpeed resolves scrollUntilVisible's `speed:` written as a ${...} +// expression into Speed: a whole number from 0 to 100, as Maestro's +// parseScrollSpeed takes and as a literal speed is checked at parse time. +func (x *expansion) scrollSpeed(s *flow.ScrollUntilVisibleStep) { + if s.SpeedExpr == "" { + return + } + v, ok := x.number(s.SpeedExpr) + if !ok { + return + } + speed, err := strconv.Atoi(v) + if err != nil || speed < 0 || speed > 100 { + x.fail(numberError{msg: fmt.Sprintf("scrollUntilVisible speed: %s is %q, not a number between 0 and 100", s.SpeedExpr, v)}) + return + } + s.Speed = speed +} + // envMap expands `${VAR}` / `${VAR || "default"}` in every value of a YAML // `env:` map, in place. func (x *expansion) envMap(env map[string]string) { @@ -899,9 +978,16 @@ func (se *ScriptEngine) ExpandStep(step flow.Step) error { if s.NotVisible != nil { s.NotVisible = x.selector(s.NotVisible) } + x.timeout(&s.BaseStep, true) case *flow.ScrollUntilVisibleStep: s.Element = *x.selector(&s.Element) s.Direction = x.text(s.Direction) + // Speed first, as Maestro evaluates it (ScrollUntilVisibleCommand + // evaluateScripts): with both bad, the speed's error is the step's. + x.scrollSpeed(s) + x.timeout(&s.BaseStep, true) + case *flow.WaitForAnimationToEndStep: + x.timeout(&s.BaseStep, false) case *flow.SetAirplaneModeStep: if str, ok := s.EnabledRaw.(string); ok { s.Enabled = parseBoolExpr(x.text(str)) diff --git a/pkg/flow/number_expression_test.go b/pkg/flow/number_expression_test.go new file mode 100644 index 00000000..3bfc7bcf --- /dev/null +++ b/pkg/flow/number_expression_test.go @@ -0,0 +1,125 @@ +package flow + +import "testing" + +// Maestro reads extendedWaitUntil's, waitForAnimationToEnd's and +// scrollUntilVisible's timeout, and scrollUntilVisible's speed, as text and +// evaluates them when the step runs, so a flow can set them from a variable +// or quote them. Here they are ints, and such a flow failed to parse, which +// stopped every flow of the run before a device was touched. +func TestParse_NumberFieldExpressions(t *testing.T) { + t.Run("a ${...} is kept for the expand pass", func(t *testing.T) { + f, err := Parse([]byte(` +- extendedWaitUntil: + visible: "Done" + timeout: ${PAUSE_MS} +- waitForAnimationToEnd: + timeout: ${PAUSE_MS * 2} +- scrollUntilVisible: + element: "Row" + timeout: ${PAUSE_MS} + speed: ${SPEED} +`), "test.yaml") + if err != nil { + t.Fatalf("parse: %v", err) + } + wait := f.Steps[0].(*WaitUntilStep) + if wait.TimeoutExpr != "${PAUSE_MS}" || wait.TimeoutMs != 0 { + t.Errorf("extendedWaitUntil: TimeoutExpr=%q TimeoutMs=%d, want ${PAUSE_MS} and 0", wait.TimeoutExpr, wait.TimeoutMs) + } + if wait.Visible == nil || wait.Visible.Text != "Done" { + t.Errorf("extendedWaitUntil lost its selector: %+v", wait.Visible) + } + anim := f.Steps[1].(*WaitForAnimationToEndStep) + if anim.TimeoutExpr != "${PAUSE_MS * 2}" { + t.Errorf("waitForAnimationToEnd: TimeoutExpr=%q, want ${PAUSE_MS * 2}", anim.TimeoutExpr) + } + scroll := f.Steps[2].(*ScrollUntilVisibleStep) + if scroll.TimeoutExpr != "${PAUSE_MS}" || scroll.SpeedExpr != "${SPEED}" || scroll.Speed != 0 { + t.Errorf("scrollUntilVisible: TimeoutExpr=%q SpeedExpr=%q Speed=%d", scroll.TimeoutExpr, scroll.SpeedExpr, scroll.Speed) + } + if scroll.Element.Text != "Row" { + t.Errorf("scrollUntilVisible lost its element: %+v", scroll.Element) + } + }) + + t.Run("a quoted number is the number", func(t *testing.T) { + f, err := Parse([]byte(` +- extendedWaitUntil: + visible: "Done" + timeout: "3000" +- waitForAnimationToEnd: + timeout: '1500' +- scrollUntilVisible: + element: "Row" + timeout: "8000" + speed: "40" +`), "test.yaml") + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := f.Steps[0].(*WaitUntilStep).TimeoutMs; got != 3000 { + t.Errorf("extendedWaitUntil TimeoutMs = %d, want 3000", got) + } + if got := f.Steps[1].(*WaitForAnimationToEndStep).TimeoutMs; got != 1500 { + t.Errorf("waitForAnimationToEnd TimeoutMs = %d, want 1500", got) + } + scroll := f.Steps[2].(*ScrollUntilVisibleStep) + if scroll.TimeoutMs != 8000 || scroll.Speed != 40 || scroll.TimeoutExpr != "" || scroll.SpeedExpr != "" { + t.Errorf("scrollUntilVisible: TimeoutMs=%d Speed=%d TimeoutExpr=%q SpeedExpr=%q, want 8000, 40 and no expressions", + scroll.TimeoutMs, scroll.Speed, scroll.TimeoutExpr, scroll.SpeedExpr) + } + }) + + t.Run("a number is read in decimal, a timeout's underscores dropped", func(t *testing.T) { + // yaml.v3 reads a leading zero as octal ("0100" = 64) and "08" as a + // float; Maestro reads these fields with toLong and parseTimeoutMs. + // scrollUntilVisible's "3_000" is a leniency: Maestro checks it with + // parseTimeoutMs, then runs it with toLong, which refuses the `_` + // (Orchestra.kt scrollUntilVisible); only its extendedWaitUntil + // drops it. + f, err := Parse([]byte(` +- extendedWaitUntil: + visible: "Done" + timeout: "0100" +- waitForAnimationToEnd: + timeout: 0100 +- scrollUntilVisible: + element: "Row" + timeout: "3_000" + speed: "040" +- extendedWaitUntil: + visible: "Done" + timeout: 08 +`), "test.yaml") + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := f.Steps[0].(*WaitUntilStep).TimeoutMs; got != 100 { + t.Errorf(`timeout: "0100" = %d, want 100`, got) + } + if got := f.Steps[1].(*WaitForAnimationToEndStep).TimeoutMs; got != 100 { + t.Errorf("timeout: 0100 = %d, want 100", got) + } + scroll := f.Steps[2].(*ScrollUntilVisibleStep) + if scroll.TimeoutMs != 3000 || scroll.Speed != 40 { + t.Errorf(`timeout: "3_000", speed: "040" = %d, %d; want 3000, 40`, scroll.TimeoutMs, scroll.Speed) + } + if got := f.Steps[3].(*WaitUntilStep).TimeoutMs; got != 8 { + t.Errorf("timeout: 08 = %d, want 8", got) + } + }) + + t.Run("a literal that is not a number is still refused", func(t *testing.T) { + for _, steps := range []string{ + "- extendedWaitUntil:\n visible: x\n timeout: soon\n", + "- waitForAnimationToEnd:\n timeout: \"soon\"\n", + "- scrollUntilVisible:\n element: x\n speed: 400\n", + "- scrollUntilVisible:\n element: x\n speed: \"4_0\"\n", + } { + if _, err := Parse([]byte(steps), "test.yaml"); err == nil { + t.Errorf("parse of %q succeeded, want an error", steps) + } + } + }) +} diff --git a/pkg/flow/parser.go b/pkg/flow/parser.go index 7f3f8900..1f5cc25d 100644 --- a/pkg/flow/parser.go +++ b/pkg/flow/parser.go @@ -342,6 +342,46 @@ func dropNonNumericScalar(node *yaml.Node, key string) { } } +// numberExpression prepares a number field that Maestro reads as text and +// evaluates when the step runs: extendedWaitUntil's, waitForAnimationToEnd's +// and scrollUntilVisible's timeout, and scrollUntilVisible's speed (Strings +// in Maestro's YAML model, resolved by the commands' evaluateScripts). Here +// those fields are ints, so `timeout: ${PAUSE_MS}` failed the whole flow at +// parse time, and with it every other flow of the run. +// +// A value holding a ${...} expression is taken out of the mapping and +// returned, for the expand pass to resolve. A number, quoted ("3000") or not, +// is read as Maestro reads it: in decimal, so "0100" is 100 where YAML would +// read octal 64, and for a timeout (underscores true) with its underscores +// dropped ("3_000"), as Maestro's extendedWaitUntil drops them +// (parseTimeoutMs); its scrollUntilVisible and waitForAnimationToEnd fail on +// one when the step runs (toLong), so there this is lenient. Anything else +// is left for the decoder, which refuses it before a device is touched. +func numberExpression(node *yaml.Node, key string, underscores bool) string { + if node == nil || node.Kind != yaml.MappingNode { + return "" + } + for i := 0; i+1 < len(node.Content); i += 2 { + k, v := node.Content[i], node.Content[i+1] + if k.Value != key || v.Kind != yaml.ScalarNode { + continue + } + if strings.Contains(v.Value, "${") { + node.Content = append(node.Content[:i], node.Content[i+2:]...) + return v.Value + } + digits := v.Value + if underscores { + digits = strings.ReplaceAll(digits, "_", "") + } + if n, err := strconv.Atoi(digits); err == nil { + v.Tag, v.Style, v.Value = "!!int", 0, strconv.Itoa(n) + } + return "" + } + return "" +} + func decodeStep(stepType StepType, valueNode *yaml.Node, sourcePath string) (Step, error) { switch stepType { case StepTapOn: @@ -411,6 +451,8 @@ func decodeStep(stepType StepType, valueNode *yaml.Node, sourcePath string) (Ste case StepScrollUntilVisible: var s ScrollUntilVisibleStep + s.TimeoutExpr = numberExpression(valueNode, "timeout", true) + s.SpeedExpr = numberExpression(valueNode, "speed", false) if valueNode.Kind == yaml.ScalarNode { s.Element.Text = valueNode.Value } else if err := valueNode.Decode(&s); err != nil { @@ -620,6 +662,7 @@ func decodeStep(stepType StepType, valueNode *yaml.Node, sourcePath string) (Ste case StepWaitUntil: var s WaitUntilStep + s.TimeoutExpr = numberExpression(valueNode, "timeout", true) if err := valueNode.Decode(&s); err != nil { return nil, wrapParseError(sourcePath, valueNode.Line, err) } @@ -1133,6 +1176,7 @@ func decodeStep(stepType StepType, valueNode *yaml.Node, sourcePath string) (Ste case StepWaitForAnimationToEnd: var s WaitForAnimationToEndStep + s.TimeoutExpr = numberExpression(valueNode, "timeout", true) if err := valueNode.Decode(&s); err != nil { return nil, wrapParseError(sourcePath, valueNode.Line, err) } diff --git a/pkg/flow/step.go b/pkg/flow/step.go index 2865a943..925de755 100644 --- a/pkg/flow/step.go +++ b/pkg/flow/step.go @@ -143,6 +143,12 @@ type BaseStep struct { Optional bool `yaml:"optional"` StepLabel string `yaml:"label"` TimeoutMs int `yaml:"timeout"` + // TimeoutExpr is `timeout:` written as a ${...} expression, kept for the + // expand pass to resolve into TimeoutMs when the step runs. Maestro reads + // extendedWaitUntil's, waitForAnimationToEnd's and scrollUntilVisible's + // timeout as text and evaluates it with the step's other scripts, so only + // those steps set it (see numberExpression). + TimeoutExpr string `yaml:"-"` // Platform restricts this step to a single platform; when set and it // doesn't match the running driver, the step is skipped (Maestro #1353). Platform string `yaml:"platform"` @@ -341,6 +347,9 @@ type ScrollUntilVisibleStep struct { WaitToSettleTimeoutMs int `yaml:"waitToSettleTimeoutMs"` // Engine selects the scroll backend. See ScrollStep.Engine. Engine string `yaml:"engine"` + // SpeedExpr is `speed:` written as a ${...} expression, which the expand + // pass resolves into Speed when the step runs, as Maestro evaluates it. + SpeedExpr string `yaml:"-"` } // BackStep presses back.