Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions pkg/executor/flow_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,9 +448,15 @@ type expressionError struct{ error }
func (e expressionError) Unwrap() error { return e.error }

// expressionFailed is the result of a step whose ${...} threw: the step fails
// with the JavaScript error and does not run, as in Maestro.
// with the JavaScript error and does not run, as in Maestro. A timeout whose
// ${...} is not a number fails it the same way, but an optional step's
// failure lets the flow go on, as Maestro's InvalidCommand does (a warning
// there): see numberError.
func expressionFailed(err error) *core.CommandResult {
err = expressionError{err}
var number numberError
if !errors.As(err, &number) || !number.optionalForgives {
err = expressionError{err}
}
return &core.CommandResult{Success: false, Error: err, Message: err.Error()}
}

Expand Down
187 changes: 187 additions & 0 deletions pkg/executor/number_expression_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
package executor

import (
"fmt"
"strings"
"testing"

"github.com/devicelab-dev/maestro-runner/pkg/flow"
"github.com/devicelab-dev/maestro-runner/pkg/report"
)

// timeoutsSeen lists the timeout (and a scrollUntilVisible's speed) of each
// step the driver got, in order.
func timeoutsSeen(seen []flow.Step) []string {
var out []string
for _, step := range seen {
switch s := step.(type) {
case *flow.WaitUntilStep:
out = append(out, fmt.Sprintf("wait %d", s.TimeoutMs))
case *flow.WaitForAnimationToEndStep:
out = append(out, fmt.Sprintf("animation %d", s.TimeoutMs))
case *flow.ScrollUntilVisibleStep:
out = append(out, fmt.Sprintf("scroll %d speed %d", s.TimeoutMs, s.Speed))
}
}
return out
}

// Maestro reads extendedWaitUntil's, waitForAnimationToEnd's and
// scrollUntilVisible's timeout, and scrollUntilVisible's speed, as text and
// evaluates them with the step's other scripts (Commands.kt evaluateScripts).
// The step gets the value its variables hold when it runs: at the top level,
// in a sub-flow run twice with different env, and in each pass of a repeat.
func TestNumberFieldExpressionsReachTheDriver(t *testing.T) {
result, seen := runFlowYAML(t, map[string]string{
"main.yaml": `appId: x
env:
PAUSE_MS: 3000
SPEED: 40
PADDED: " 1500 "
---
- extendedWaitUntil:
visible: Done
timeout: ${PAUSE_MS}
- waitForAnimationToEnd:
timeout: ${PAUSE_MS * 2}
- scrollUntilVisible:
element: Row
timeout: ${PAUSE_MS}
speed: ${SPEED}
- runFlow:
file: wait.yaml
env:
WAIT_MS: 1000
- runFlow:
file: wait.yaml
env:
WAIT_MS: 2000
- evalScript: ${output.n = 0}
- repeat:
times: 2
commands:
- evalScript: ${output.n = output.n + 1}
- waitForAnimationToEnd:
timeout: ${output.n * 500}
- extendedWaitUntil:
visible: Done
timeout: ${PADDED}
`,
"wait.yaml": `appId: x
---
- extendedWaitUntil:
notVisible: Spinner
timeout: ${WAIT_MS}
`,
})
if result.Status != report.StatusPassed {
t.Fatalf("status = %s, want passed: %+v", result.Status, result.FlowResults)
}
want := []string{
"wait 3000",
"animation 6000",
"scroll 3000 speed 40",
"wait 1000",
"wait 2000",
"animation 500",
"animation 1000",
"wait 1500",
}
got := timeoutsSeen(seen)
if strings.Join(got, ", ") != strings.Join(want, ", ") {
t.Errorf("the driver got\n %s\nwant\n %s", strings.Join(got, ", "), strings.Join(want, ", "))
}
}

// A value that is not a number fails the step, which does not run. Maestro
// lets an optional extendedWaitUntil or scrollUntilVisible go on (a warning
// there: its timeout error is an InvalidCommand; here the step is reported
// failed and the flow continues), but never a bad scroll
// speed (InvalidNumericFieldValue) or waitForAnimationToEnd timeout (a
// NumberFormatException): Orchestra.kt executeCommands.
func TestNumberFieldExpressionThatIsNotANumber(t *testing.T) {
for _, c := range []struct {
name, step string
want report.Status
error string
header string
}{
{"extendedWaitUntil", `- extendedWaitUntil:
visible: Done
timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon", not a number of milliseconds`, ""},
{"optional extendedWaitUntil", `- extendedWaitUntil:
visible: Done
timeout: ${LATER}
optional: true`, report.StatusPassed, "", ""},
{"optional scrollUntilVisible timeout", `- scrollUntilVisible:
element: Row
timeout: ${LATER}
optional: true`, report.StatusPassed, "", ""},
{"optional waitForAnimationToEnd", `- waitForAnimationToEnd:
timeout: ${LATER}
optional: true`, report.StatusFailed, `waitForAnimationToEnd timeout: ${LATER} is "soon", not a number of milliseconds`, ""},
{"optional scrollUntilVisible speed", `- scrollUntilVisible:
element: Row
speed: ${FAST}
optional: true`, report.StatusFailed, `scrollUntilVisible speed: ${FAST} is "400", not a number between 0 and 100`, ""},
// Maestro evaluates the speed first, so its error is the step's.
{"optional scrollUntilVisible, speed and timeout both bad", `- scrollUntilVisible:
element: Row
timeout: ${LATER}
speed: ${FAST}
optional: true`, report.StatusFailed, `scrollUntilVisible speed: ${FAST} is "400"`, ""},
// A ${...} that throws fails even an optional step (JsEvaluationException).
{"optional extendedWaitUntil whose script throws", `- extendedWaitUntil:
visible: Done
timeout: ${output.wait.ms}
optional: true`, report.StatusFailed, "TypeError", ""},
{"extendedWaitUntil in a runFlow", `- runFlow:
commands:
- extendedWaitUntil:
visible: Done
timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`, ""},
{"optional extendedWaitUntil in a runFlow", `- runFlow:
commands:
- extendedWaitUntil:
visible: Done
timeout: ${LATER}
optional: true`, report.StatusPassed, "", ""},
{"waitForAnimationToEnd in a repeat", `- repeat:
times: 2
commands:
- waitForAnimationToEnd:
timeout: ${LATER}`, report.StatusFailed, `waitForAnimationToEnd timeout: ${LATER} is "soon"`, ""},
{"optional extendedWaitUntil in a repeat", `- repeat:
times: 2
commands:
- extendedWaitUntil:
visible: Done
timeout: ${LATER}
optional: true`, report.StatusPassed, "", ""},
{"extendedWaitUntil in a retry", `- retry:
maxRetries: 2
commands:
- extendedWaitUntil:
visible: Done
timeout: ${LATER}`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`, ""},
{"extendedWaitUntil in onFlowStart", `- tapOn: Next`, report.StatusFailed, `extendedWaitUntil timeout: ${LATER} is "soon"`,
"onFlowStart:\n - extendedWaitUntil:\n visible: Done\n timeout: ${LATER}\n"},
} {
t.Run(c.name, func(t *testing.T) {
result, seen := runFlowYAML(t, map[string]string{
"main.yaml": "appId: x\nenv:\n LATER: soon\n FAST: 400\n" + c.header + "---\n" + c.step + "\n",
})
if result.Status != c.want {
t.Fatalf("status = %s, want %s: %+v", result.Status, c.want, result.FlowResults)
}
if c.error != "" && !strings.Contains(result.FlowResults[0].Error, c.error) {
t.Errorf("error = %q, want %q", result.FlowResults[0].Error, c.error)
}
for _, step := range seen {
if _, tap := step.(*flow.TapOnStep); !tap {
t.Errorf("the driver got %s; want no step with the bad value", step.Describe())
}
}
})
}
}
86 changes: 86 additions & 0 deletions pkg/executor/scripting.go
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,85 @@ func (x *expansion) text(s string) string {
return out
}

// numberError is a number field whose ${...} resolved to something that is
// not a number the field takes. The step fails with it without running.
// optionalForgives is true where Maestro lets an optional step's failure go
// (a warning there; here the step is reported failed and the flow goes on):
// its timeout error is an InvalidCommand, which optional downgrades; a bad
// scroll speed is an InvalidNumericFieldValue, which it never does, and
// waitForAnimationToEnd's timeout fails as a NumberFormatException
// (Orchestra.kt executeCommands, Commands.kt).
type numberError struct {
msg string
optionalForgives bool
}

func (e numberError) Error() string { return e.msg }

// number expands a number field's ${...} expression and returns its value,
// trimmed. ok is false when the expression threw; the error is kept.
func (x *expansion) number(expr string) (string, bool) {
out, err := x.se.expand(expr)
if err != nil {
if x.err == nil {
x.err = err
}
return "", false
}
return strings.TrimSpace(out), true
}

func (x *expansion) fail(err error) {
if x.err == nil {
x.err = err
}
}

// timeout resolves a `timeout:` written as a ${...} expression into
// TimeoutMs, read as Maestro's parseTimeoutMs reads it: a whole number of
// milliseconds, `_` allowed (Maestro's extendedWaitUntil drops it; its
// scrollUntilVisible and waitForAnimationToEnd fail on it, so there this is
// lenient), surrounding spaces trimmed. A non-positive one is then the
// step's default, as when it is written as a literal. The expression is
// kept, so the step resolves it again each time it runs.
func (x *expansion) timeout(b *flow.BaseStep, optionalForgives bool) {
if b.TimeoutExpr == "" {
return
}
v, ok := x.number(b.TimeoutExpr)
if !ok {
return
}
ms, err := strconv.Atoi(strings.ReplaceAll(v, "_", ""))
if err != nil {
x.fail(numberError{
msg: fmt.Sprintf("%s timeout: %s is %q, not a number of milliseconds", b.StepType, b.TimeoutExpr, v),
optionalForgives: optionalForgives,
})
return
}
b.TimeoutMs = ms
}

// scrollSpeed resolves scrollUntilVisible's `speed:` written as a ${...}
// expression into Speed: a whole number from 0 to 100, as Maestro's
// parseScrollSpeed takes and as a literal speed is checked at parse time.
func (x *expansion) scrollSpeed(s *flow.ScrollUntilVisibleStep) {
if s.SpeedExpr == "" {
return
}
v, ok := x.number(s.SpeedExpr)
if !ok {
return
}
speed, err := strconv.Atoi(v)
if err != nil || speed < 0 || speed > 100 {
x.fail(numberError{msg: fmt.Sprintf("scrollUntilVisible speed: %s is %q, not a number between 0 and 100", s.SpeedExpr, v)})
return
}
s.Speed = speed
}

// envMap expands `${VAR}` / `${VAR || "default"}` in every value of a YAML
// `env:` map, in place.
func (x *expansion) envMap(env map[string]string) {
Expand Down Expand Up @@ -899,9 +978,16 @@ func (se *ScriptEngine) ExpandStep(step flow.Step) error {
if s.NotVisible != nil {
s.NotVisible = x.selector(s.NotVisible)
}
x.timeout(&s.BaseStep, true)
case *flow.ScrollUntilVisibleStep:
s.Element = *x.selector(&s.Element)
s.Direction = x.text(s.Direction)
// Speed first, as Maestro evaluates it (ScrollUntilVisibleCommand
// evaluateScripts): with both bad, the speed's error is the step's.
x.scrollSpeed(s)
x.timeout(&s.BaseStep, true)
case *flow.WaitForAnimationToEndStep:
x.timeout(&s.BaseStep, false)
case *flow.SetAirplaneModeStep:
if str, ok := s.EnabledRaw.(string); ok {
s.Enabled = parseBoolExpr(x.text(str))
Expand Down
Loading
Loading