Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions broker.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ type broker struct {
// it is not in completed and is not counted as a fill; each exit it changed
// also carries the change in its own history.
amended []*order

// journal, when not nil, is given to every order the broker makes itself (its
// protective exits and closeouts) and to the orders it is handed: see
// trader.handleOrder. A run sets it before any order exists.
journal *journal
}

func newBroker(id string, markets *catalog) *broker {
Expand Down Expand Up @@ -235,8 +240,8 @@ func (br *broker) amend(ctx context.Context, act *account, o *order) error {
}
if !o.takeProfit.IsZero() {
if target == nil {
br.placeExit(&order{accountID: o.accountID, instrument: o.instrument, direction: pos.direction, Time: o.Time},
limitOrder, reasonTakeProfit, o.takeProfit)
br.placeExit(&order{accountID: o.accountID, instrument: o.instrument, direction: pos.direction, Time: o.Time, journal: o.journal},
limitOrder, reasonTakeProfit, o.takeProfit, num.Price{})
changes = append(changes, "target set to "+o.takeProfit.String())
} else {
changes = append(changes, "target "+target.price.String()+" to "+o.takeProfit.String())
Expand Down Expand Up @@ -546,14 +551,14 @@ func (br *broker) enforceMargin(ctx context.Context, f frame, act *account) erro
// open o carries them, as resting closes the broker owns.
func (br *broker) placeExits(o *order) {
if !o.stopLoss.IsZero() {
br.placeExit(o, stopOrder, reasonStopLoss, o.stopLoss).trail = o.trail
br.placeExit(o, stopOrder, reasonStopLoss, o.stopLoss, o.trail)
}
if !o.takeProfit.IsZero() {
br.placeExit(o, limitOrder, reasonTakeProfit, o.takeProfit)
br.placeExit(o, limitOrder, reasonTakeProfit, o.takeProfit, num.Price{})
}
}

func (br *broker) placeExit(o *order, typ orderType, why orderReason, p num.Price) *order {
func (br *broker) placeExit(o *order, typ orderType, why orderReason, p, trail num.Price) *order {
x := &order{
id: br.nextID(),
accountID: o.accountID,
Expand All @@ -563,7 +568,9 @@ func (br *broker) placeExit(o *order, typ orderType, why orderReason, p num.Pric
orderType: typ,
reason: why,
price: p,
trail: trail, // part of the order before its first record, so the journal's requested line has it
Time: o.Time,
journal: o.journal, // before the first record: the requested line is the order's first
}
x.record(stateRequested, x.Time, "")
br.enqueue(x)
Expand Down Expand Up @@ -613,6 +620,7 @@ func (br *broker) closePosition(b marketdata.Bar, act *account, pos *position, w
reason: why,
price: p,
Time: b.Time,
journal: br.journal,
}
o.record(stateRequested, b.Time, "")
o.record(stateSubmitted, b.Time, "")
Expand Down
27 changes: 24 additions & 3 deletions cmd/trader/backtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ import (
// backtestFlags are the options of `trader backtest`: status's, plus the source.
type backtestFlags struct {
statusFlags
source string
source string
resultsRoot string
}

func newBacktestCmd(a *app) *cobra.Command {
Expand All @@ -36,7 +37,13 @@ those two (a stale month is never accepted). The command reads only, needs no
token and makes no network call. The data root is --data-root, then
` + envDataRoot + `, then marketdata.root in ` + defaultConfigPath + `, then the default ` + store.DefaultRoot + `;
--instrument and --interval also fall back to that file. h1 and d1 only; the
engine trades the instruments it has a market for.`,
engine trades the instruments it has a market for.

Every run is journaled: it gets a RunID and a directory of that name under the
results root, holding run.json (what the run was, and how it ended) and
events.jsonl (every order transition). The results root is --results-root, then
` + envResultsRoot + `, then results.root in ` + defaultConfigPath + `, then the default ` + service.DefaultResultsRoot + `.
A run that fails or is cancelled is journaled too, with its status and why.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return a.runBacktest(cmd, f)
Expand All @@ -50,6 +57,7 @@ engine trades the instruments it has a market for.`,
fl.StringVar(&f.interval, "interval", "", "h1 or d1 (default: marketdata.interval)")
fl.StringVar(&f.from, "from", "", "first instant, UTC: YYYY-MM-DD or RFC 3339 (required)")
fl.StringVar(&f.to, "to", "", "end of the range, exclusive, UTC: YYYY-MM-DD or RFC 3339 (required)")
fl.StringVar(&f.resultsRoot, "results-root", "", "results root (default: "+envResultsRoot+", then results.root in the config file, then "+service.DefaultResultsRoot+")")
fl.BoolVar(&f.allowBad, "allow-bad", false, "accept months that have bad records (their good bars load)")
fl.BoolVar(&f.allowMissing, "allow-missing", false, "accept months with no canonical file (a gap in the series)")
return cmd
Expand All @@ -68,12 +76,25 @@ func (a *app) runBacktest(cmd *cobra.Command, f backtestFlags) error {
res, err := service.RunBacktest(ctx, logging.WithComponent(slog.Default(), logging.ComponentMarketData), service.BacktestRequest{
Source: f.source, Instruments: opts.symbols, Interval: opts.interval, From: opts.from, To: opts.to,
AllowBad: f.allowBad, AllowMissing: f.allowMissing,
ResultsRoot: orDefault(resolveRoot(f.resultsRoot, a.getenv(envResultsRoot), opts.resultsRoot), service.DefaultResultsRoot),
})
if err != nil {
return err
}
if err := res.Write(cmd.OutOrStdout()); err != nil {
out := cmd.OutOrStdout()
if err := res.Write(out); err != nil {
return fmt.Errorf("write summary: %w", err)
}
if _, err := fmt.Fprintf(out, "run %s: %s\n", res.RunID, res.Dir); err != nil {
return fmt.Errorf("write summary: %w", err)
}
return nil
}

// orDefault is v, or def when v is empty.
func orDefault(v, def string) string {
if v == "" {
return def
}
return v
}
88 changes: 87 additions & 1 deletion cmd/trader/backtest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"
"go/parser"
"go/token"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
Expand Down Expand Up @@ -43,7 +45,7 @@ func putWave(t *testing.T, root, symbol string, base int64) {
func backtestArgs(t *testing.T, root, to string, extra ...string) []string {
t.Helper()
return append([]string{"backtest", "--config", writeConfig(t, ""), "--data-root", root, "--instrument", "EURUSD,GBPUSD",
"--interval", "h1", "--from", "2024-01-01", "--to", to}, extra...)
"--interval", "h1", "--from", "2024-01-01", "--to", to, "--results-root", t.TempDir()}, extra...)
}

func TestBacktest_RunsTwoInstrumentsFromTheStoreAndPrintsTheSummaryWithoutAToken(t *testing.T) {
Expand All @@ -59,6 +61,90 @@ func TestBacktest_RunsTwoInstrumentsFromTheStoreAndPrintsTheSummaryWithoutAToken
assert.NotEmpty(t, logs)
}

// runDirs lists the run directories under root (each named by a RunID).
func runDirs(t *testing.T, root string) []string {
t.Helper()
entries, err := os.ReadDir(root)
require.NoError(t, err)
var names []string
for _, e := range entries {
names = append(names, e.Name())
}
return names
}

func TestBacktest_JournalsTheRunAndSaysWhere(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
putWave(t, root, "GBPUSD", 125000)
results := t.TempDir()
args := backtestArgs(t, root, "2024-02-01")
args[len(args)-1] = results // --results-root

out, _, err := runData(t, nil, args...)

require.NoError(t, err)
dirs := runDirs(t, results)
require.Len(t, dirs, 1)
assert.Contains(t, out, "run "+dirs[0]+": "+filepath.Join(results, dirs[0]), "the summary ends by saying where the run is")
assert.FileExists(t, filepath.Join(results, dirs[0], "run.json"))
assert.FileExists(t, filepath.Join(results, dirs[0], "events.jsonl"))
}

// TestBacktest_ResultsRootIsFlagThenEnvironmentThenFile checks the order the
// results root is resolved in, the same as the data root's.
func TestBacktest_ResultsRootIsFlagThenEnvironmentThenFile(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
putWave(t, root, "GBPUSD", 125000)
flagDir, envDir, fileDir := t.TempDir(), t.TempDir(), t.TempDir()
withFile := func() []string {
args := backtestArgs(t, root, "2024-02-01")
args = args[:len(args)-2] // no --results-root
args[2] = writeConfig(t, "results:\n root: "+fileDir+"\n")
return args
}

_, _, err := runData(t, nil, withFile()...)
require.NoError(t, err)
assert.Len(t, runDirs(t, fileDir), 1, "the config file's root, with no flag or environment")

_, _, err = runData(t, map[string]string{envResultsRoot: envDir}, withFile()...)
require.NoError(t, err)
assert.Len(t, runDirs(t, envDir), 1, "the environment beats the file")
assert.Len(t, runDirs(t, fileDir), 1, "and the file got nothing more")

_, _, err = runData(t, map[string]string{envResultsRoot: envDir}, append(withFile(), "--results-root", flagDir)...)
require.NoError(t, err)
assert.Len(t, runDirs(t, flagDir), 1, "the flag beats both")
assert.Len(t, runDirs(t, envDir), 1)
}

func TestOrDefault(t *testing.T) {
assert.Equal(t, "x", orDefault("x", "d"))
assert.Equal(t, "d", orDefault("", "d"))
assert.Equal(t, "/srv/trading/backtest/results", service.DefaultResultsRoot)
}

func TestBacktest_AFailedRunIsJournaledToo(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
putWave(t, root, "GBPUSD", 125000)
results := t.TempDir()
args := backtestArgs(t, root, "2024-03-01") // February is missing
args[len(args)-1] = results

_, _, err := runData(t, nil, args...)

require.ErrorIs(t, err, service.ErrNotLoadable)
dirs := runDirs(t, results)
require.Len(t, dirs, 1)
assert.ErrorContains(t, err, filepath.Join(results, dirs[0]), "the error says where the run is")
data, rerr := os.ReadFile(filepath.Join(results, dirs[0], "run.json"))
require.NoError(t, rerr)
assert.Contains(t, string(data), `"status": "failed"`)
}

func TestBacktest_FailsOnAMissingMonthUnlessAllowed(t *testing.T) {
root := storetest.Root(t)
putWave(t, root, "EURUSD", 110000)
Expand Down
9 changes: 5 additions & 4 deletions cmd/trader/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,10 @@ h1 and d1 only.`,
// rangeOptions are the options build and status share: what to read, and the data
// root, resolved from flag, environment and file.
type rangeOptions struct {
symbols []string
interval string
from, to time.Time
symbols []string
interval string
from, to time.Time
resultsRoot string // results.root of the config file; the flag and the environment are not here
}

// resolveRange resolves the shared options (flag, environment, file) and points
Expand Down Expand Up @@ -99,7 +100,7 @@ func (a *app) resolveRange(cmd *cobra.Command, f buildFlags) (rangeOptions, erro
if err != nil {
return rangeOptions{}, err
}
return rangeOptions{symbols: symbols, interval: interval, from: from, to: to}, nil
return rangeOptions{symbols: symbols, interval: interval, from: from, to: to, resultsRoot: cfg.Results.Root}, nil
}

// runBuild resolves every option and hands the request to the application
Expand Down
6 changes: 6 additions & 0 deletions cmd/trader/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ const defaultConfigPath = "etc/trader.yml"
// secret is an error rather than ignored.
type fileConfig struct {
Marketdata marketdataConfig `yaml:"marketdata"`
Results resultsConfig `yaml:"results"`
}

// resultsConfig is where the results of runs are kept.
type resultsConfig struct {
Root string `yaml:"root"`
}

type marketdataConfig struct {
Expand Down
10 changes: 10 additions & 0 deletions cmd/trader/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ func TestLoadConfig_ReadsTheNestedMarketdataBlock(t *testing.T) {
assert.Equal(t, marketdataConfig{Root: "/somewhere", Provider: "oanda", BaseURL: "https://example.test", Instruments: []string{"EURUSD", "GBPUSD"}, Interval: "h1"}, cfg.Marketdata)
}

func TestLoadConfig_ReadsTheResultsRoot(t *testing.T) {
cfg, err := loadConfig(writeConfig(t, "results:\n root: /srv/elsewhere/results\n"), true)

require.NoError(t, err)
assert.Equal(t, resultsConfig{Root: "/srv/elsewhere/results"}, cfg.Results)

_, err = loadConfig(writeConfig(t, "results:\n root: /x\n token: nope\n"), true)
assert.Error(t, err, "a results key that is not known, or looks like a secret, is refused like any other")
}

func TestLoadConfig_AMissingFileIsFineUnlessNamed(t *testing.T) {
missing := filepath.Join(t.TempDir(), "nope.yml")

Expand Down
3 changes: 3 additions & 0 deletions cmd/trader/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ const (
envOANDAToken = "TRADER_OANDA_TOKEN"
envDataRoot = "TRADER_DATA_ROOT"

// envResultsRoot is where a run's results (its RunID directory) go.
envResultsRoot = "TRADER_RESULTS_ROOT"

// defaultBaseURL is OANDA's practice host, so a run without a base URL can
// only read demo data; the live host is chosen explicitly.
defaultBaseURL = "https://api-fxpractice.oanda.com"
Expand Down
14 changes: 10 additions & 4 deletions docs/DevelopersGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,10 +145,16 @@ account and broker (`replay.go`, `backtest.go`):
- `trader.RunSeries(ctx, name, series...)` is the engine's entry for series a reader
loaded: it works on copies, normalizes instrument keys, refuses an instrument it
has no market for, runs EMAX on each instrument with the scenarios' fixed
settings and returns a `Result` (`Write` prints the summary). The engine opens no
file. `service.RunBacktest` is the application service that joins the two:
`LoadSeries` (the market-data service's policy for missing, bad and stale
months), then `RunSeries`; `trader backtest` only parses flags and prints.
settings and returns a `Result` (`Write` prints the summary, `Summary` has the numbers).
The engine opens no file; `RunSeriesWith` takes an `io.Writer` for the journal, which
`order.record` feeds (`journal.go`; only `service.RunBacktest` journals every run, a direct
`RunSeries` call writes one only if given a writer) so every order transition of every order,
the broker's exits included, is written as it happens. `service.RunBacktest` is the
application service that joins the two and journals every run: it makes the RunID
and the run directory (`internal/results`, the one package that opens those files),
then `LoadSeries` (the market-data service's policy for missing, bad and stale
months), then `RunSeriesWith`, then finalizes `run.json`; `trader backtest` only
parses flags and prints.
- Mid prices and ticks (#667). A canonical mid is the exact midpoint of bid and
ask, so it can sit half a tick from a whole tick (1.100005 for ticks of 0.00001).
`trader.RunSeries` rounds every bar price to the nearest tick of the market (a
Expand Down
30 changes: 29 additions & 1 deletion docs/UsersGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ default `/srv/data`.
| Option | Flag | Environment | Config key |
|---|---|---|---|
| data root | `--data-root` | `TRADER_DATA_ROOT` | `marketdata.root` (default `/srv/data`) |
| results root | `--results-root` (backtest) | `TRADER_RESULTS_ROOT` | `results.root` (default `/srv/trading/backtest/results`) |
| instruments | `--instrument` (repeat or comma-separated) | | `marketdata.instruments` |
| interval | `--interval` | | `marketdata.interval` |
| OANDA host | `--base-url` (download only) | | `marketdata.base_url` |
Expand Down Expand Up @@ -285,7 +286,34 @@ prices to the nearest tick of the market (a mid between two ticks rounds up); th
stored data is not changed.

Output: the engine's summary for the run (bars, orders, trades, money, open
positions).
positions), then `run <RunID>: <directory>`.

### Results

Every run is journaled, whether it completes, fails or is cancelled. It gets a RunID
(`run_` and a sortable ULID that carries the time it began) and a directory of that
name under the results root (`--results-root`, then `TRADER_RESULTS_ROOT`, then
`results.root`, then `/srv/trading/backtest/results`; the root is created if it is
missing). Two files are enough to read a run:

- `run.json`: the RunID, kind (`backtest`), strategy, source, interval, instruments,
range, wall-clock `started_at` and `ended_at`, `status` (`running` while it runs,
then `completed`, `failed` or `cancelled`), `error` and, for a completed run, the
engine's `result` (orders, trades, balance, equity, open positions). A run that was
cut off, say by a crash, still says `running`.
- `events.jsonl`: one JSON line for every order transition, in the order it
happened: `requested` (what was asked: instrument, action, direction, type, price,
stop, target), `sized` (the account's size), `submitted` (the broker's id), `filled`
(units, fill price, and what the order was), `rejected` or `cancelled` (with the
reason). The broker's own orders, protective stops, targets and margin closeouts,
are in it with their `cause`. Lines carry the simulated time only, so two runs that
trade the same way write the same lines. A failed run keeps the events it had.

```sh
ls /srv/trading/backtest/results # one directory per run
jq . /srv/trading/backtest/results/run_*/run.json # status, error, result
jq -c 'select(.state=="filled")' /srv/trading/backtest/results/run_*/events.jsonl
```

## Backtests

Expand Down
5 changes: 5 additions & 0 deletions etc/trader.example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,8 @@ marketdata:
# Defaults for `trader data download`; --instrument and --interval win.
instruments: [EURUSD]
interval: h1
results:
# Where each run's results go, in a directory named by its RunID: run.json and
# events.jsonl. --results-root, then TRADER_RESULTS_ROOT, then this value, then
# /srv/trading/backtest/results.
root: /srv/trading/backtest/results
Loading
Loading