Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion backtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ func readError(last marketdata.Bar, err error) error {
// stops included; pending is what is still waiting when the data ends.
type result struct {
scenario string
strategy string // the name of the strategy that traded
bars int

submitted, filled, rejected, cancelled, pending int
Expand All @@ -136,8 +137,12 @@ func newResult(scenario string, bars int, tr *trader) (*result, error) {
if err != nil {
return nil, err
}
name := ""
if tr.strategy != nil { // a trader built without one, as some tests do
name = tr.strategy.name()
}
r := &result{
scenario: scenario, bars: bars, rejected: len(tr.rejected),
scenario: scenario, strategy: name, bars: bars, rejected: len(tr.rejected),
balance: act.balance, unrealized: unrealized, equity: equity,
realized: realized,
}
Expand Down
11 changes: 10 additions & 1 deletion cmd/trader/backtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type backtestFlags struct {
statusFlags
source string
resultsRoot string
worker string
}

func newBacktestCmd(a *app) *cobra.Command {
Expand All @@ -43,7 +44,13 @@ Every run is journaled: it gets a RunID and a directory of that name under the
results root, holding run.json (what the run was, and how it ended) and
events.jsonl (every order transition). The results root is --results-root, then
` + envResultsRoot + `, then results.root in ` + defaultConfigPath + `, then the default ` + service.DefaultResultsRoot + `.
A run that fails or is cancelled is journaled too, with its status and why.`,
A run that fails or is cancelled is journaled too, with its status and why.

--worker host:port trades with an external strategy process listening on that
loopback address (examples/emax is the canonical one) instead of the built-in EMAX:
every group of bars goes to it over TCP and JSON and it answers with the orders it
wants; Trader sizes, admits and fills them exactly as for the built-in strategy. A
worker is not warmed up, so the months before --from are not loaded for it.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return a.runBacktest(cmd, f)
Expand All @@ -57,6 +64,7 @@ A run that fails or is cancelled is journaled too, with its status and why.`,
fl.StringVar(&f.interval, "interval", "", "h1 or d1 (default: marketdata.interval)")
fl.StringVar(&f.from, "from", "", "first instant, UTC: YYYY-MM-DD or RFC 3339 (required)")
fl.StringVar(&f.to, "to", "", "end of the range, exclusive, UTC: YYYY-MM-DD or RFC 3339 (required)")
fl.StringVar(&f.worker, "worker", "", "loopback host:port of a worker process that trades instead of the built-in EMAX (see examples/emax); it is not warmed up")
fl.StringVar(&f.resultsRoot, "results-root", "", "results root (default: "+envResultsRoot+", then results.root in the config file, then "+service.DefaultResultsRoot+")")
fl.BoolVar(&f.allowBad, "allow-bad", false, "accept months that have bad records (their good bars load)")
fl.BoolVar(&f.allowMissing, "allow-missing", false, "accept months with no canonical file (a gap in the series)")
Expand All @@ -76,6 +84,7 @@ func (a *app) runBacktest(cmd *cobra.Command, f backtestFlags) error {
res, err := service.RunBacktest(ctx, logging.WithComponent(slog.Default(), logging.ComponentMarketData), service.BacktestRequest{
Source: f.source, Instruments: opts.symbols, Interval: opts.interval, From: opts.from, To: opts.to,
AllowBad: f.allowBad, AllowMissing: f.allowMissing,
Worker: f.worker,
ResultsRoot: orDefault(resolveRoot(f.resultsRoot, a.getenv(envResultsRoot), opts.resultsRoot), service.DefaultResultsRoot),
})
if err != nil {
Expand Down
68 changes: 68 additions & 0 deletions cmd/trader/backtest_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package main

import (
"bufio"
"fmt"
"go/parser"
"go/token"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
Expand Down Expand Up @@ -197,3 +199,69 @@ func TestBacktestCommand_IsTransportOnly(t *testing.T) {
}
}
}

// startEMAXProcess builds examples/emax and runs it as a separate process, as an
// operator would, and returns the address it prints.
func startEMAXProcess(t *testing.T) string {
t.Helper()
bin := filepath.Join(t.TempDir(), "emax")
out, err := exec.Command("go", "build", "-o", bin, "../../examples/emax").CombinedOutput()
require.NoError(t, err, string(out))

cmd := exec.Command(bin, "-addr", "127.0.0.1:0")
stdout, err := cmd.StdoutPipe()
require.NoError(t, err)
require.NoError(t, cmd.Start())
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })

line, err := bufio.NewReader(stdout).ReadString('\n')
require.NoError(t, err)
return strings.TrimSpace(strings.TrimPrefix(line, "listening on "))
}

// TestBacktest_WorkerTradesLikeTheBuiltInStrategy runs the command twice on the same
// stored data: with the built-in EMAX, and with the EMAX example as a separate process
// over loopback TCP. The summaries are the same.
func TestBacktest_WorkerTradesLikeTheBuiltInStrategy(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
putWave(t, root, "GBPUSD", 125000)
builtin, _, err := runData(t, nil, backtestArgs(t, root, "2024-02-01")...)
require.NoError(t, err)

addr := startEMAXProcess(t)
results := t.TempDir()
args := backtestArgs(t, root, "2024-02-01", "--worker", addr)
args[len(args)-3] = results // --results-root's value, which comes before --worker
remote, _, err := runData(t, nil, args...)

require.NoError(t, err)
summary := func(out string) string { return out[:strings.LastIndex(strings.TrimSpace(out), "\nrun ")] }
assert.Equal(t, summary(builtin), summary(remote), "the same orders, trades and money")
assert.Contains(t, remote, "orders: submitted")
assert.NotContains(t, remote, "submitted 0,", "it traded")
dirs := runDirs(t, results)
require.Len(t, dirs, 1)
rec, rerr := os.ReadFile(filepath.Join(results, dirs[0], "run.json"))
require.NoError(t, rerr)
assert.Contains(t, string(rec), `"transport": "tcp"`)
assert.Contains(t, string(rec), `"addr": "`+addr+`"`)
}

func TestBacktest_AWorkerThatIsNotListeningIsAFailedRun(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
putWave(t, root, "GBPUSD", 125000)
results := t.TempDir()
args := backtestArgs(t, root, "2024-02-01", "--worker", "127.0.0.1:1")
args[len(args)-3] = results

_, _, err := runData(t, nil, args...)

require.Error(t, err)
dirs := runDirs(t, results)
require.Len(t, dirs, 1, "the run exists, and says it failed")
rec, rerr := os.ReadFile(filepath.Join(results, dirs[0], "run.json"))
require.NoError(t, rerr)
assert.Contains(t, string(rec), `"status": "failed"`)
}
5 changes: 4 additions & 1 deletion docs/DevelopersGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,10 @@ account and broker (`replay.go`, `backtest.go`):
and the run directory (`internal/results`, the one package that opens those files),
then `LoadSeries` (the market-data service's policy for missing, bad and stale
months), then `RunSeriesWith`, then finalizes `run.json`; `trader backtest` only
parses flags and prints.
parses flags and prints. `RunOptions.Worker` (`--worker`) swaps the built-in strategy
for a remote one (`strategy_remote.go`, the transport in package `strategy`) with
everything else unchanged; `examples/emax` is the canonical worker, and its parity
with the built-in EMAX is tested (`trader_test.go`, `service/backtest_test.go`).
- Mid prices and ticks (#667). A canonical mid is the exact midpoint of bid and
ask, so it can sit half a tick from a whole tick (1.100005 for ticks of 0.00001).
`trader.RunSeries` rounds every bar price to the nearest tick of the market (a
Expand Down
12 changes: 12 additions & 0 deletions docs/UsersGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,18 @@ stored data is not changed.
Output: the engine's summary for the run (bars, orders, trades, money, open
positions), then `run <RunID>: <directory>`.

### An external strategy

`--worker host:port` trades with a strategy running as its own process on a loopback
address instead of the built-in EMAX. Trader sends each group of completed bars over TCP
and JSON; the worker answers with the orders it wants (it never says how many units);
Trader sizes, admits, fills and journals them as it does for the built-in strategy. A
worker is not warmed up: the month before `--from` is not loaded for it, and it starts
cold. [`examples/emax`](../examples/emax/README.md) is the canonical worker (the same
EMAX, written against the public SDK) and its README says how to run it and read the
journal. A worker that cannot be reached, disconnects or crashes ends the run as
`failed`, with the events recorded until then.

### Results

Every run is journaled, whether it completes, fails or is cancelled. It gets a RunID
Expand Down
85 changes: 85 additions & 0 deletions examples/emax/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# EMAX as an external strategy

The canonical example of a strategy that runs as its own process. It is the EMAX
strategy Trader has built in, written against the public Go SDK only (`strategy`,
`marketdata`, `num`), so it shows what an outside worker looks like: Trader sends
each group of completed bars over loopback TCP and JSON, the worker answers with the
orders it wants, and Trader sizes, admits, fills and journals them exactly as it
does for the built-in strategy.

| File | What it is |
|---|---|
| `main.go` | The program: flags, then `strategy.ListenAndServe`. |
| `emax/emax.go` | The strategy: `emax.New(cfg)` and `OnBars`, plus the EMA. |

## The rule

Long while the short EMA of the close is above the long one, short while it is
below. A change of side is a close on the crossover bar and the opposite open on the
next. Every open carries a stop a fixed fraction of the entry price away, rounded
down to the price tick (at least one tick). Each instrument has its own EMAs and
signal state. The strategy asks only; it never says how many units.

## Run it

Start the worker. It listens on a loopback address (a non-loopback one is refused)
and prints the address it bound:

```sh
go run ./examples/emax -addr 127.0.0.1:9100
```

| Flag | Default | Meaning |
|---|---|---|
| `-addr` | `127.0.0.1:0` | loopback address to listen on (`:0` picks a free port) |
| `-short`, `-long` | `2`, `4` | EMA windows (the built-in EMAX uses 2 and 4) |
| `-stop` | `0.01` | stop distance as a fraction of the price (1%) |
| `-tick` | `0.00001` | price tick of the instruments traded (EURUSD, GBPUSD) |

Then run a backtest that trades with it, in another terminal. The data is the
canonical bars in the store, as for any `trader backtest`:

```sh
trader backtest --worker 127.0.0.1:9100 \
--instrument EURUSD,GBPUSD --interval h1 --from 2024-01-01 --to 2024-04-01
```

The worker serves one run and exits when Trader closes the connection. To compare
with the built-in strategy, run the same command without `--worker`: the summaries
are the same (see Parity).

A worker is not warmed up. The built-in EMAX is prefilled with the month before
`--from`; a worker is only shown bars from `--from` on, so it starts cold. For the
comparison, use a range whose earlier month is not stored, or accept that the first
few bars may differ.

## Where the results are

Every run is journaled under the results root (`--results-root`, `TRADER_RESULTS_ROOT`,
`results.root`, default `/srv/trading/backtest/results`), in a directory named by its
RunID. The command prints it at the end:

```text
run run_01KB5Q3V7T2N4XG9YB6HFA1V7Z: /srv/trading/backtest/results/run_01KB5Q3V7T2N4XG9YB6HFA1V7Z
```

```sh
dir=/srv/trading/backtest/results/run_01KB5Q3V7T2N4XG9YB6HFA1V7Z
jq '{status, error, strategy, result: .result.orders}' $dir/run.json # transport tcp, status completed
jq -c 'select(.state=="filled")' $dir/events.jsonl # every fill, stops included
```

A run whose worker disconnects or crashes is journaled too: `run.json` says
`failed` and why, and `events.jsonl` keeps what had happened until then.

## Parity

The worker must trade exactly as the built-in EMAX does. The root package's tests
hold it to that: the six EMAX scenarios, a 1,500-bar two-instrument walk, and a real
month built through the store (`service`) are each run with the built-in strategy and
with this worker over TCP from the same data, settings and account, and the
journals (every order transition, fill and protective exit, in order, to the price),
the summaries and the final positions must be identical. A run that disagrees is a
regression to explain, not a difference in protocol. The worker's EMA is a copy of
the built-in one (a worker cannot import Trader's private code), and a test checks the
two give the same value to the last bit.
Loading
Loading