diff --git a/broker.go b/broker.go index b1031d49..11f95558 100644 --- a/broker.go +++ b/broker.go @@ -46,6 +46,11 @@ type broker struct { // it is not in completed and is not counted as a fill; each exit it changed // also carries the change in its own history. amended []*order + + // journal, when not nil, is given to every order the broker makes itself (its + // protective exits and closeouts) and to the orders it is handed: see + // trader.handleOrder. A run sets it before any order exists. + journal *journal } func newBroker(id string, markets *catalog) *broker { @@ -235,8 +240,8 @@ func (br *broker) amend(ctx context.Context, act *account, o *order) error { } if !o.takeProfit.IsZero() { if target == nil { - br.placeExit(&order{accountID: o.accountID, instrument: o.instrument, direction: pos.direction, Time: o.Time}, - limitOrder, reasonTakeProfit, o.takeProfit) + br.placeExit(&order{accountID: o.accountID, instrument: o.instrument, direction: pos.direction, Time: o.Time, journal: o.journal}, + limitOrder, reasonTakeProfit, o.takeProfit, num.Price{}) changes = append(changes, "target set to "+o.takeProfit.String()) } else { changes = append(changes, "target "+target.price.String()+" to "+o.takeProfit.String()) @@ -546,14 +551,14 @@ func (br *broker) enforceMargin(ctx context.Context, f frame, act *account) erro // open o carries them, as resting closes the broker owns. func (br *broker) placeExits(o *order) { if !o.stopLoss.IsZero() { - br.placeExit(o, stopOrder, reasonStopLoss, o.stopLoss).trail = o.trail + br.placeExit(o, stopOrder, reasonStopLoss, o.stopLoss, o.trail) } if !o.takeProfit.IsZero() { - br.placeExit(o, limitOrder, reasonTakeProfit, o.takeProfit) + br.placeExit(o, limitOrder, reasonTakeProfit, o.takeProfit, num.Price{}) } } -func (br *broker) placeExit(o *order, typ orderType, why orderReason, p num.Price) *order { +func (br *broker) placeExit(o *order, typ orderType, why orderReason, p, trail num.Price) *order { x := &order{ id: br.nextID(), accountID: o.accountID, @@ -563,7 +568,9 @@ func (br *broker) placeExit(o *order, typ orderType, why orderReason, p num.Pric orderType: typ, reason: why, price: p, + trail: trail, // part of the order before its first record, so the journal's requested line has it Time: o.Time, + journal: o.journal, // before the first record: the requested line is the order's first } x.record(stateRequested, x.Time, "") br.enqueue(x) @@ -613,6 +620,7 @@ func (br *broker) closePosition(b marketdata.Bar, act *account, pos *position, w reason: why, price: p, Time: b.Time, + journal: br.journal, } o.record(stateRequested, b.Time, "") o.record(stateSubmitted, b.Time, "") diff --git a/cmd/trader/backtest.go b/cmd/trader/backtest.go index 582f5266..d9fd3f6a 100644 --- a/cmd/trader/backtest.go +++ b/cmd/trader/backtest.go @@ -16,7 +16,8 @@ import ( // backtestFlags are the options of `trader backtest`: status's, plus the source. type backtestFlags struct { statusFlags - source string + source string + resultsRoot string } func newBacktestCmd(a *app) *cobra.Command { @@ -36,7 +37,13 @@ those two (a stale month is never accepted). The command reads only, needs no token and makes no network call. The data root is --data-root, then ` + envDataRoot + `, then marketdata.root in ` + defaultConfigPath + `, then the default ` + store.DefaultRoot + `; --instrument and --interval also fall back to that file. h1 and d1 only; the -engine trades the instruments it has a market for.`, +engine trades the instruments it has a market for. + +Every run is journaled: it gets a RunID and a directory of that name under the +results root, holding run.json (what the run was, and how it ended) and +events.jsonl (every order transition). The results root is --results-root, then +` + envResultsRoot + `, then results.root in ` + defaultConfigPath + `, then the default ` + service.DefaultResultsRoot + `. +A run that fails or is cancelled is journaled too, with its status and why.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return a.runBacktest(cmd, f) @@ -50,6 +57,7 @@ engine trades the instruments it has a market for.`, fl.StringVar(&f.interval, "interval", "", "h1 or d1 (default: marketdata.interval)") fl.StringVar(&f.from, "from", "", "first instant, UTC: YYYY-MM-DD or RFC 3339 (required)") fl.StringVar(&f.to, "to", "", "end of the range, exclusive, UTC: YYYY-MM-DD or RFC 3339 (required)") + fl.StringVar(&f.resultsRoot, "results-root", "", "results root (default: "+envResultsRoot+", then results.root in the config file, then "+service.DefaultResultsRoot+")") fl.BoolVar(&f.allowBad, "allow-bad", false, "accept months that have bad records (their good bars load)") fl.BoolVar(&f.allowMissing, "allow-missing", false, "accept months with no canonical file (a gap in the series)") return cmd @@ -68,12 +76,25 @@ func (a *app) runBacktest(cmd *cobra.Command, f backtestFlags) error { res, err := service.RunBacktest(ctx, logging.WithComponent(slog.Default(), logging.ComponentMarketData), service.BacktestRequest{ Source: f.source, Instruments: opts.symbols, Interval: opts.interval, From: opts.from, To: opts.to, AllowBad: f.allowBad, AllowMissing: f.allowMissing, + ResultsRoot: orDefault(resolveRoot(f.resultsRoot, a.getenv(envResultsRoot), opts.resultsRoot), service.DefaultResultsRoot), }) if err != nil { return err } - if err := res.Write(cmd.OutOrStdout()); err != nil { + out := cmd.OutOrStdout() + if err := res.Write(out); err != nil { + return fmt.Errorf("write summary: %w", err) + } + if _, err := fmt.Fprintf(out, "run %s: %s\n", res.RunID, res.Dir); err != nil { return fmt.Errorf("write summary: %w", err) } return nil } + +// orDefault is v, or def when v is empty. +func orDefault(v, def string) string { + if v == "" { + return def + } + return v +} diff --git a/cmd/trader/backtest_test.go b/cmd/trader/backtest_test.go index 9f6c9af2..5275be1e 100644 --- a/cmd/trader/backtest_test.go +++ b/cmd/trader/backtest_test.go @@ -4,6 +4,8 @@ import ( "fmt" "go/parser" "go/token" + "os" + "path/filepath" "strconv" "strings" "testing" @@ -43,7 +45,7 @@ func putWave(t *testing.T, root, symbol string, base int64) { func backtestArgs(t *testing.T, root, to string, extra ...string) []string { t.Helper() return append([]string{"backtest", "--config", writeConfig(t, ""), "--data-root", root, "--instrument", "EURUSD,GBPUSD", - "--interval", "h1", "--from", "2024-01-01", "--to", to}, extra...) + "--interval", "h1", "--from", "2024-01-01", "--to", to, "--results-root", t.TempDir()}, extra...) } func TestBacktest_RunsTwoInstrumentsFromTheStoreAndPrintsTheSummaryWithoutAToken(t *testing.T) { @@ -59,6 +61,90 @@ func TestBacktest_RunsTwoInstrumentsFromTheStoreAndPrintsTheSummaryWithoutAToken assert.NotEmpty(t, logs) } +// runDirs lists the run directories under root (each named by a RunID). +func runDirs(t *testing.T, root string) []string { + t.Helper() + entries, err := os.ReadDir(root) + require.NoError(t, err) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + return names +} + +func TestBacktest_JournalsTheRunAndSaysWhere(t *testing.T) { + root := storetest.Root(t) + putWave(t, root, "EURUSD", 110000) + putWave(t, root, "GBPUSD", 125000) + results := t.TempDir() + args := backtestArgs(t, root, "2024-02-01") + args[len(args)-1] = results // --results-root + + out, _, err := runData(t, nil, args...) + + require.NoError(t, err) + dirs := runDirs(t, results) + require.Len(t, dirs, 1) + assert.Contains(t, out, "run "+dirs[0]+": "+filepath.Join(results, dirs[0]), "the summary ends by saying where the run is") + assert.FileExists(t, filepath.Join(results, dirs[0], "run.json")) + assert.FileExists(t, filepath.Join(results, dirs[0], "events.jsonl")) +} + +// TestBacktest_ResultsRootIsFlagThenEnvironmentThenFile checks the order the +// results root is resolved in, the same as the data root's. +func TestBacktest_ResultsRootIsFlagThenEnvironmentThenFile(t *testing.T) { + root := storetest.Root(t) + putWave(t, root, "EURUSD", 110000) + putWave(t, root, "GBPUSD", 125000) + flagDir, envDir, fileDir := t.TempDir(), t.TempDir(), t.TempDir() + withFile := func() []string { + args := backtestArgs(t, root, "2024-02-01") + args = args[:len(args)-2] // no --results-root + args[2] = writeConfig(t, "results:\n root: "+fileDir+"\n") + return args + } + + _, _, err := runData(t, nil, withFile()...) + require.NoError(t, err) + assert.Len(t, runDirs(t, fileDir), 1, "the config file's root, with no flag or environment") + + _, _, err = runData(t, map[string]string{envResultsRoot: envDir}, withFile()...) + require.NoError(t, err) + assert.Len(t, runDirs(t, envDir), 1, "the environment beats the file") + assert.Len(t, runDirs(t, fileDir), 1, "and the file got nothing more") + + _, _, err = runData(t, map[string]string{envResultsRoot: envDir}, append(withFile(), "--results-root", flagDir)...) + require.NoError(t, err) + assert.Len(t, runDirs(t, flagDir), 1, "the flag beats both") + assert.Len(t, runDirs(t, envDir), 1) +} + +func TestOrDefault(t *testing.T) { + assert.Equal(t, "x", orDefault("x", "d")) + assert.Equal(t, "d", orDefault("", "d")) + assert.Equal(t, "/srv/trading/backtest/results", service.DefaultResultsRoot) +} + +func TestBacktest_AFailedRunIsJournaledToo(t *testing.T) { + root := storetest.Root(t) + putWave(t, root, "EURUSD", 110000) + putWave(t, root, "GBPUSD", 125000) + results := t.TempDir() + args := backtestArgs(t, root, "2024-03-01") // February is missing + args[len(args)-1] = results + + _, _, err := runData(t, nil, args...) + + require.ErrorIs(t, err, service.ErrNotLoadable) + dirs := runDirs(t, results) + require.Len(t, dirs, 1) + assert.ErrorContains(t, err, filepath.Join(results, dirs[0]), "the error says where the run is") + data, rerr := os.ReadFile(filepath.Join(results, dirs[0], "run.json")) + require.NoError(t, rerr) + assert.Contains(t, string(data), `"status": "failed"`) +} + func TestBacktest_FailsOnAMissingMonthUnlessAllowed(t *testing.T) { root := storetest.Root(t) putWave(t, root, "EURUSD", 110000) diff --git a/cmd/trader/build.go b/cmd/trader/build.go index ec3f6cc4..209d745e 100644 --- a/cmd/trader/build.go +++ b/cmd/trader/build.go @@ -67,9 +67,10 @@ h1 and d1 only.`, // rangeOptions are the options build and status share: what to read, and the data // root, resolved from flag, environment and file. type rangeOptions struct { - symbols []string - interval string - from, to time.Time + symbols []string + interval string + from, to time.Time + resultsRoot string // results.root of the config file; the flag and the environment are not here } // resolveRange resolves the shared options (flag, environment, file) and points @@ -99,7 +100,7 @@ func (a *app) resolveRange(cmd *cobra.Command, f buildFlags) (rangeOptions, erro if err != nil { return rangeOptions{}, err } - return rangeOptions{symbols: symbols, interval: interval, from: from, to: to}, nil + return rangeOptions{symbols: symbols, interval: interval, from: from, to: to, resultsRoot: cfg.Results.Root}, nil } // runBuild resolves every option and hands the request to the application diff --git a/cmd/trader/config.go b/cmd/trader/config.go index ee2efcc6..dae2e730 100644 --- a/cmd/trader/config.go +++ b/cmd/trader/config.go @@ -23,6 +23,12 @@ const defaultConfigPath = "etc/trader.yml" // secret is an error rather than ignored. type fileConfig struct { Marketdata marketdataConfig `yaml:"marketdata"` + Results resultsConfig `yaml:"results"` +} + +// resultsConfig is where the results of runs are kept. +type resultsConfig struct { + Root string `yaml:"root"` } type marketdataConfig struct { diff --git a/cmd/trader/config_test.go b/cmd/trader/config_test.go index 26d6d299..c78baffe 100644 --- a/cmd/trader/config_test.go +++ b/cmd/trader/config_test.go @@ -19,6 +19,16 @@ func TestLoadConfig_ReadsTheNestedMarketdataBlock(t *testing.T) { assert.Equal(t, marketdataConfig{Root: "/somewhere", Provider: "oanda", BaseURL: "https://example.test", Instruments: []string{"EURUSD", "GBPUSD"}, Interval: "h1"}, cfg.Marketdata) } +func TestLoadConfig_ReadsTheResultsRoot(t *testing.T) { + cfg, err := loadConfig(writeConfig(t, "results:\n root: /srv/elsewhere/results\n"), true) + + require.NoError(t, err) + assert.Equal(t, resultsConfig{Root: "/srv/elsewhere/results"}, cfg.Results) + + _, err = loadConfig(writeConfig(t, "results:\n root: /x\n token: nope\n"), true) + assert.Error(t, err, "a results key that is not known, or looks like a secret, is refused like any other") +} + func TestLoadConfig_AMissingFileIsFineUnlessNamed(t *testing.T) { missing := filepath.Join(t.TempDir(), "nope.yml") diff --git a/cmd/trader/data.go b/cmd/trader/data.go index a23ef24c..9ae201d3 100644 --- a/cmd/trader/data.go +++ b/cmd/trader/data.go @@ -24,6 +24,9 @@ const ( envOANDAToken = "TRADER_OANDA_TOKEN" envDataRoot = "TRADER_DATA_ROOT" + // envResultsRoot is where a run's results (its RunID directory) go. + envResultsRoot = "TRADER_RESULTS_ROOT" + // defaultBaseURL is OANDA's practice host, so a run without a base URL can // only read demo data; the live host is chosen explicitly. defaultBaseURL = "https://api-fxpractice.oanda.com" diff --git a/docs/DevelopersGuide.md b/docs/DevelopersGuide.md index bfff9708..67bb3ad9 100644 --- a/docs/DevelopersGuide.md +++ b/docs/DevelopersGuide.md @@ -145,10 +145,16 @@ account and broker (`replay.go`, `backtest.go`): - `trader.RunSeries(ctx, name, series...)` is the engine's entry for series a reader loaded: it works on copies, normalizes instrument keys, refuses an instrument it has no market for, runs EMAX on each instrument with the scenarios' fixed - settings and returns a `Result` (`Write` prints the summary). The engine opens no - file. `service.RunBacktest` is the application service that joins the two: - `LoadSeries` (the market-data service's policy for missing, bad and stale - months), then `RunSeries`; `trader backtest` only parses flags and prints. + settings and returns a `Result` (`Write` prints the summary, `Summary` has the numbers). + The engine opens no file; `RunSeriesWith` takes an `io.Writer` for the journal, which + `order.record` feeds (`journal.go`; only `service.RunBacktest` journals every run, a direct + `RunSeries` call writes one only if given a writer) so every order transition of every order, + the broker's exits included, is written as it happens. `service.RunBacktest` is the + application service that joins the two and journals every run: it makes the RunID + and the run directory (`internal/results`, the one package that opens those files), + then `LoadSeries` (the market-data service's policy for missing, bad and stale + months), then `RunSeriesWith`, then finalizes `run.json`; `trader backtest` only + parses flags and prints. - Mid prices and ticks (#667). A canonical mid is the exact midpoint of bid and ask, so it can sit half a tick from a whole tick (1.100005 for ticks of 0.00001). `trader.RunSeries` rounds every bar price to the nearest tick of the market (a diff --git a/docs/UsersGuide.md b/docs/UsersGuide.md index e5b7748d..a232dbc9 100644 --- a/docs/UsersGuide.md +++ b/docs/UsersGuide.md @@ -53,6 +53,7 @@ default `/srv/data`. | Option | Flag | Environment | Config key | |---|---|---|---| | data root | `--data-root` | `TRADER_DATA_ROOT` | `marketdata.root` (default `/srv/data`) | +| results root | `--results-root` (backtest) | `TRADER_RESULTS_ROOT` | `results.root` (default `/srv/trading/backtest/results`) | | instruments | `--instrument` (repeat or comma-separated) | | `marketdata.instruments` | | interval | `--interval` | | `marketdata.interval` | | OANDA host | `--base-url` (download only) | | `marketdata.base_url` | @@ -285,7 +286,34 @@ prices to the nearest tick of the market (a mid between two ticks rounds up); th stored data is not changed. Output: the engine's summary for the run (bars, orders, trades, money, open -positions). +positions), then `run : `. + +### Results + +Every run is journaled, whether it completes, fails or is cancelled. It gets a RunID +(`run_` and a sortable ULID that carries the time it began) and a directory of that +name under the results root (`--results-root`, then `TRADER_RESULTS_ROOT`, then +`results.root`, then `/srv/trading/backtest/results`; the root is created if it is +missing). Two files are enough to read a run: + +- `run.json`: the RunID, kind (`backtest`), strategy, source, interval, instruments, + range, wall-clock `started_at` and `ended_at`, `status` (`running` while it runs, + then `completed`, `failed` or `cancelled`), `error` and, for a completed run, the + engine's `result` (orders, trades, balance, equity, open positions). A run that was + cut off, say by a crash, still says `running`. +- `events.jsonl`: one JSON line for every order transition, in the order it + happened: `requested` (what was asked: instrument, action, direction, type, price, + stop, target), `sized` (the account's size), `submitted` (the broker's id), `filled` + (units, fill price, and what the order was), `rejected` or `cancelled` (with the + reason). The broker's own orders, protective stops, targets and margin closeouts, + are in it with their `cause`. Lines carry the simulated time only, so two runs that + trade the same way write the same lines. A failed run keeps the events it had. + +```sh +ls /srv/trading/backtest/results # one directory per run +jq . /srv/trading/backtest/results/run_*/run.json # status, error, result +jq -c 'select(.state=="filled")' /srv/trading/backtest/results/run_*/events.jsonl +``` ## Backtests diff --git a/etc/trader.example.yml b/etc/trader.example.yml index 1e6d3a1b..f2a87ae8 100644 --- a/etc/trader.example.yml +++ b/etc/trader.example.yml @@ -10,3 +10,8 @@ marketdata: # Defaults for `trader data download`; --instrument and --interval win. instruments: [EURUSD] interval: h1 +results: + # Where each run's results go, in a directory named by its RunID: run.json and + # events.jsonl. --results-root, then TRADER_RESULTS_ROOT, then this value, then + # /srv/trading/backtest/results. + root: /srv/trading/backtest/results diff --git a/historical.go b/historical.go index 9f5d981c..f842f064 100644 --- a/historical.go +++ b/historical.go @@ -19,16 +19,92 @@ type Result struct{ r *result } // nothing that varies between runs. func (r *Result) Write(w io.Writer) error { return r.r.write(w) } +// Summary is what a finished run came to, as data: the numbers Write prints, exact, +// for a run record to keep. It is read from the result, not calculated again. +type Summary struct { + Strategy string `json:"strategy"` + Account string `json:"account"` + InitialBalance num.Money `json:"initial_balance"` + Bars int `json:"bars"` + Orders OrderCount `json:"orders"` + Trades int `json:"trades"` + Wins int `json:"wins"` + Losses int `json:"losses"` + BreakEven int `json:"break_even"` + Realized num.Money `json:"realized"` + Unrealized num.Money `json:"unrealized"` + Balance num.Money `json:"balance"` + Equity num.Money `json:"equity"` + Open []Position `json:"open"` +} + +// OrderCount is how many orders ended where, as in Write's orders line. +type OrderCount struct { + Submitted int `json:"submitted"` + Filled int `json:"filled"` + Rejected int `json:"rejected"` + Cancelled int `json:"cancelled"` + Pending int `json:"pending"` +} + +// Position is a position still open when the run ended. +type Position struct { + Instrument string `json:"instrument"` + Direction string `json:"direction"` + Units num.Quantity `json:"units"` + EntryPrice num.Price `json:"entry_price"` + MarkPrice num.Price `json:"mark_price"` +} + +// Summary returns the run's numbers. +func (r *Result) Summary() Summary { + x := r.r + s := Summary{ + Strategy: runStrategy, Account: runAccount, InitialBalance: num.MustParseMoney(runBalance, usdCurrency), + Bars: x.bars, + Orders: OrderCount{Submitted: x.submitted, Filled: x.filled, Rejected: x.rejected, Cancelled: x.cancelled, Pending: x.pending}, + Trades: x.trades, Wins: x.wins, Losses: x.losses, BreakEven: x.breakEven, + Realized: x.realized, Unrealized: x.unrealized, Balance: x.balance, Equity: x.equity, + Open: []Position{}, + } + for _, p := range x.open { + s.Open = append(s.Open, Position{Instrument: p.instrument, Direction: p.direction.String(), Units: p.units, + EntryPrice: p.entryPrice, MarkPrice: p.markPrice}) + } + return s +} + // The fixed configuration of RunSeries: the one the EMAX scenarios were worked out // for, applied to each instrument of the run. Choosing a strategy and an account // is a later piece of work. const ( - runShort = 2 - runLong = 4 - runBalance = "10000" // USD - runStrategy = "emax" + runShort = 2 + runLong = 4 + runBalance = "10000" // USD + + // StrategyEMAX names the strategy RunSeries trades. + StrategyEMAX = "emax" + runStrategy = StrategyEMAX + runAccount = "sim123" ) +// RunOptions are the optional parts of a run. +// +// Journaling is the application's rule, not the engine's: every run started through +// service.RunBacktest (which is what `trader backtest` calls) is journaled, with a +// RunID and a results directory. RunSeries and RunSeriesWith are the engine's +// library entry points and write a journal only if handed a writer, which is how +// tests and any other caller that does not want a run directory use them. The +// scenario regression (Run, the six EMAX scenarios) is not a run in that sense: it +// is the baseline the journaled runs are compared with, and has no directory. +type RunOptions struct { + // Journal, when not nil, is written one JSON line for every transition of every + // order of the run, in the order they happen (see journal). The engine opens no + // file; the caller owns the writer. If a write fails the run stops with that + // error, and what was written before it stays. + Journal io.Writer +} + // RunSeries backtests the series, which a reader has already loaded, on one // shared account and broker: EMAX on each instrument, one strategy callback per // distinct availability instant (see backtest.runSeriesFrom). name labels the @@ -51,6 +127,13 @@ const ( // the canonical data it came from, are not changed. Series are of different // instruments; an empty series contributes nothing. func RunSeries(ctx context.Context, name string, tradeFrom time.Time, series ...marketdata.Series) (*Result, error) { + return RunSeriesWith(ctx, RunOptions{}, name, tradeFrom, series...) +} + +// RunSeriesWith is RunSeries with options. A run with the zero RunOptions is a +// RunSeries run, and what it trades does not depend on the options: a journal only +// records. +func RunSeriesWith(ctx context.Context, opts RunOptions, name string, tradeFrom time.Time, series ...marketdata.Series) (*Result, error) { if len(series) == 0 { return nil, errors.New("run: no series") } @@ -92,14 +175,20 @@ func RunSeries(ctx context.Context, name string, tradeFrom time.Time, series ... return nil, err } br := newBroker("sim", markets) + if opts.Journal != nil { + br.journal = newJournal(opts.Journal) + } tr := &trader{ broker: br, - account: br.addAccount("sim123", num.MustParseMoney(runBalance, usdCurrency)), + account: br.addAccount(runAccount, num.MustParseMoney(runBalance, usdCurrency)), strategy: strat, } if err := (&backtest{}).runSeriesFrom(ctx, tr, tradeFrom, run...); err != nil { return nil, err } + if err := br.journal.err(); err != nil { // the last frame's transitions too + return nil, fmt.Errorf("journal: %w", err) + } res, err := newResult(name, bars, tr) if err != nil { return nil, err diff --git a/internal/results/run.go b/internal/results/run.go new file mode 100644 index 00000000..37049e0b --- /dev/null +++ b/internal/results/run.go @@ -0,0 +1,182 @@ +// Package results keeps what a run leaves behind: a directory per run under a +// results root, named by its RunID, holding run.json (what the run was and how it +// ended) and events.jsonl (the engine's journal of every order transition). It is +// the one place that opens those files; the engine only writes to an io.Writer it is +// handed, and a client such as Investor only reads the two files. +package results + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/rustyeddy/trader/internal/clock" + "github.com/rustyeddy/trader/internal/id" +) + +// DefaultRoot is where the results of runs are kept when nothing else is +// configured. It is a default, not a place the code knows: the transport resolves +// the root (flag, environment, config file, this) and passes it in. +const DefaultRoot = "/srv/trading/backtest/results" + +// Run states, as run.json says them. +const ( + Running = "running" + Completed = "completed" + Failed = "failed" + Cancelled = "cancelled" +) + +// File names inside a run directory. +const ( + RunFile = "run.json" // the Record + EventsFile = "events.jsonl" // one JSON line per order transition +) + +// Record is run.json: what a run was and how it ended. It holds no result of its +// own: Result is the engine's summary, copied as it came, and the orders and fills +// are events.jsonl, the engine's own record. A reader needs only these two files. +type Record struct { + RunID id.RunID `json:"run_id"` + Kind string `json:"kind"` + Strategy struct { + Name string `json:"name"` + Transport string `json:"transport"` + } `json:"strategy"` + Source string `json:"source"` + Interval string `json:"interval"` + Instruments []string `json:"instruments"` + From time.Time `json:"from"` + To time.Time `json:"to"` + + // StartedAt and EndedAt are wall-clock time. Nothing else in a run's files is: the + // events carry the simulated time only. + StartedAt time.Time `json:"started_at"` + EndedAt time.Time `json:"ended_at,omitzero"` + + // Status is Running from the start, so a crash leaves a run that says so, and + // then Completed, Failed or Cancelled. Error says why for the last two. + Status string `json:"status"` + Error string `json:"error,omitempty"` + + // Result is the engine's summary of a completed run, set by Finish. + Result any `json:"result,omitempty"` +} + +// Run is a run in progress: its directory, its journal file and its record. +type Run struct { + dir string + rec Record + events *os.File + buf *bufio.Writer +} + +// Start makes the run's directory under root, named by the RunID, and opens its +// journal. The directory is created, never reused: a RunID that already has one is +// an error. run.json is written at once with status "running". +func Start(root string, rec Record) (*Run, error) { + if root == "" { + return nil, errors.New("no results root") + } + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("results root: %w", err) + } + dir := filepath.Join(root, rec.RunID.String()) + if err := os.Mkdir(dir, 0o755); err != nil { + return nil, fmt.Errorf("run directory: %w", err) + } + f, err := os.OpenFile(filepath.Join(dir, EventsFile), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, fmt.Errorf("run journal: %w", err) + } + r := &Run{dir: dir, rec: rec, events: f, buf: bufio.NewWriter(f)} + r.rec.Status = Running + if err := writeRecord(dir, r.rec); err != nil { + _ = f.Close() + return nil, err + } + return r, nil +} + +// Dir is the run's directory. +func (r *Run) Dir() string { return r.dir } + +// Journal is where the engine writes the run's events. +func (r *Run) Journal() io.Writer { return r.buf } + +// Finish ends the run: it flushes and closes the journal and rewrites run.json +// with the outcome. runErr is how the run ended (nil for completed), result the +// engine's summary when it completed. +// +// The status is the run's own: completed, cancelled (runErr is context.Canceled) or +// failed. The journal is always flushed and closed, and if that fails too the error +// is added to the run's, never dropped: a completed run becomes failed, since what it +// recorded is not all there, and a failed or cancelled one keeps its status and says +// in its error that events.jsonl may be incomplete. The returned error carries both +// (errors.Is still finds the run's own). +func (r *Run) Finish(runErr error, result any) error { + var jerr error + if err := r.buf.Flush(); err != nil { + jerr = fmt.Errorf("journal: %w", err) + } + if err := r.events.Close(); err != nil { + jerr = errors.Join(jerr, fmt.Errorf("journal: %w", err)) + } + + status := Completed + switch { + case runErr == nil && jerr != nil: + status, runErr, jerr = Failed, jerr, nil + case runErr == nil: + case errors.Is(runErr, context.Canceled): + status = Cancelled + default: + status = Failed + } + if jerr != nil { + runErr = fmt.Errorf("%w (and the journal is incomplete: %w)", runErr, jerr) + } + + r.rec.EndedAt = clock.Clock() + r.rec.Status = status + if runErr != nil { + r.rec.Error = runErr.Error() + } else { + r.rec.Result = result + } + if err := writeRecord(r.dir, r.rec); err != nil { + return errors.Join(runErr, err) + } + return runErr +} + +// writeRecord replaces run.json with rec: written to a temporary file, synced and +// renamed over it, so a reader sees the old record or the new one and never half. +func writeRecord(dir string, rec Record) error { + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return fmt.Errorf("run record: %w", err) + } + tmp := filepath.Join(dir, RunFile+".tmp") + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("run record: %w", err) + } + _, werr := f.Write(append(data, '\n')) + serr := f.Sync() + cerr := f.Close() + if err := errors.Join(werr, serr, cerr); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("run record: %w", err) + } + if err := os.Rename(tmp, filepath.Join(dir, RunFile)); err != nil { + return fmt.Errorf("run record: %w", err) + } + return nil +} diff --git a/internal/results/run_test.go b/internal/results/run_test.go new file mode 100644 index 00000000..910ab08e --- /dev/null +++ b/internal/results/run_test.go @@ -0,0 +1,195 @@ +package results + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/rustyeddy/trader/internal/id" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newRecord(t *testing.T) Record { + t.Helper() + runID, err := id.NewGenerator(func() time.Time { return time.Date(2026, 11, 21, 18, 0, 0, 0, time.UTC) }, id.Random{}).RunID() + require.NoError(t, err) + rec := Record{ + RunID: runID, Kind: "backtest", Source: "oanda", Interval: "h1", Instruments: []string{"EURUSD"}, + From: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC), + StartedAt: time.Date(2026, 11, 21, 18, 0, 0, 0, time.UTC), + } + rec.Strategy.Name, rec.Strategy.Transport = "emax", "in-process" + return rec +} + +func readRecord(t *testing.T, dir string) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, RunFile)) + require.NoError(t, err) + var m map[string]any + require.NoError(t, json.Unmarshal(data, &m)) + return m +} + +func TestStart_MakesTheRunDirectoryAndWritesARunningRecord(t *testing.T) { + root := filepath.Join(t.TempDir(), "results", "backtest") // the root need not exist + rec := newRecord(t) + + run, err := Start(root, rec) + require.NoError(t, err) + defer func() { _ = run.Finish(nil, nil) }() + + assert.Equal(t, filepath.Join(root, rec.RunID.String()), run.Dir()) + _, err = id.ParseRunID(filepath.Base(run.Dir())) + require.NoError(t, err, "the directory is named by the RunID") + + m := readRecord(t, run.Dir()) + assert.Equal(t, "running", m["status"], "a run that is cut off says it was running") + assert.Equal(t, rec.RunID.String(), m["run_id"]) + assert.Equal(t, "backtest", m["kind"]) + assert.Equal(t, map[string]any{"name": "emax", "transport": "in-process"}, m["strategy"]) + assert.Equal(t, "oanda", m["source"]) + assert.Equal(t, []any{"EURUSD"}, m["instruments"]) + assert.Equal(t, "2024-01-01T00:00:00Z", m["from"]) + assert.NotContains(t, m, "ended_at") + assert.NotContains(t, m, "result") + info, err := os.Stat(filepath.Join(run.Dir(), EventsFile)) + require.NoError(t, err) + assert.Zero(t, info.Size(), "the journal is open and empty") +} + +func TestStart_Refuses(t *testing.T) { + t.Run("no root", func(t *testing.T) { + _, err := Start("", newRecord(t)) + assert.ErrorContains(t, err, "no results root") + }) + t.Run("a root that is a file", func(t *testing.T) { + file := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + _, err := Start(file, newRecord(t)) + assert.ErrorContains(t, err, "results root") + }) + t.Run("a run id that already has a directory, which is left alone", func(t *testing.T) { + root := t.TempDir() + rec := newRecord(t) + require.NoError(t, os.Mkdir(filepath.Join(root, rec.RunID.String()), 0o755)) + keep := filepath.Join(root, rec.RunID.String(), "keep.txt") + require.NoError(t, os.WriteFile(keep, []byte("mine"), 0o644)) + + _, err := Start(root, rec) + assert.ErrorIs(t, err, os.ErrExist) + data, rerr := os.ReadFile(keep) + require.NoError(t, rerr) + assert.Equal(t, "mine", string(data)) + _, serr := os.Stat(filepath.Join(root, rec.RunID.String(), RunFile)) + assert.ErrorIs(t, serr, os.ErrNotExist, "nothing was written into it") + }) +} + +func TestFinish(t *testing.T) { + start := func(t *testing.T) *Run { + run, err := Start(t.TempDir(), newRecord(t)) + require.NoError(t, err) + return run + } + events := func(t *testing.T, run *Run) string { + data, err := os.ReadFile(filepath.Join(run.Dir(), EventsFile)) + require.NoError(t, err) + return string(data) + } + + t.Run("a completed run keeps its events and its result", func(t *testing.T) { + run := start(t) + _, err := run.Journal().Write([]byte(`{"seq":1}` + "\n")) + require.NoError(t, err) + + require.NoError(t, run.Finish(nil, map[string]int{"trades": 3})) + + m := readRecord(t, run.Dir()) + assert.Equal(t, "completed", m["status"]) + assert.Equal(t, map[string]any{"trades": float64(3)}, m["result"]) + assert.NotContains(t, m, "error") + ended, err := time.Parse(time.RFC3339, m["ended_at"].(string)) + require.NoError(t, err) + assert.False(t, ended.Before(newRecord(t).StartedAt)) + assert.Equal(t, `{"seq":1}`+"\n", events(t, run), "what was journaled is on disk") + }) + t.Run("a failed run keeps the events it had and says why", func(t *testing.T) { + run := start(t) + _, err := run.Journal().Write([]byte(`{"seq":1}` + "\n")) + require.NoError(t, err) + + got := run.Finish(errors.New("worker disconnected"), nil) + + assert.EqualError(t, got, "worker disconnected", "the run's own error comes back") + m := readRecord(t, run.Dir()) + assert.Equal(t, "failed", m["status"]) + assert.Equal(t, "worker disconnected", m["error"]) + assert.NotContains(t, m, "result") + assert.Equal(t, `{"seq":1}`+"\n", events(t, run), "the partial journal is kept") + }) + t.Run("a cancelled run is cancelled, not failed", func(t *testing.T) { + run := start(t) + require.ErrorIs(t, run.Finish(context.Canceled, nil), context.Canceled) + assert.Equal(t, "cancelled", readRecord(t, run.Dir())["status"]) + }) + t.Run("a journal that cannot be flushed fails a run that would have completed", func(t *testing.T) { + run := start(t) + _, err := run.Journal().Write([]byte("not on disk yet\n")) + require.NoError(t, err) + require.NoError(t, run.events.Close()) // the disk goes away under the buffer + + got := run.Finish(nil, map[string]int{"trades": 1}) + + require.ErrorContains(t, got, "journal") + m := readRecord(t, run.Dir()) + assert.Equal(t, "failed", m["status"]) + assert.Contains(t, m["error"], "journal") + assert.NotContains(t, m, "result", "a run whose record is incomplete reports no result") + }) + t.Run("a failed run whose journal also fails says both and keeps its own status", func(t *testing.T) { + run := start(t) + _, err := run.Journal().Write([]byte("not on disk yet\n")) + require.NoError(t, err) + require.NoError(t, run.events.Close()) + disconnect := errors.New("worker disconnected") + + got := run.Finish(disconnect, nil) + + require.ErrorIs(t, got, disconnect) + assert.ErrorContains(t, got, "journal") + m := readRecord(t, run.Dir()) + assert.Equal(t, "failed", m["status"]) + assert.Contains(t, m["error"], "worker disconnected") + assert.Contains(t, m["error"], "the journal is incomplete", "the disconnect is not the only thing wrong with events.jsonl") + }) + t.Run("a cancelled run whose journal also fails is still cancelled and says so", func(t *testing.T) { + run := start(t) + _, err := run.Journal().Write([]byte("not on disk yet\n")) + require.NoError(t, err) + require.NoError(t, run.events.Close()) + + got := run.Finish(context.Canceled, nil) + + require.ErrorIs(t, got, context.Canceled, "the status is classified from the run's own error") + m := readRecord(t, run.Dir()) + assert.Equal(t, "cancelled", m["status"]) + assert.Contains(t, m["error"], "the journal is incomplete") + }) + t.Run("run.json is replaced whole and no temporary file is left", func(t *testing.T) { + run := start(t) + require.NoError(t, run.Finish(nil, nil)) + entries, err := os.ReadDir(run.Dir()) + require.NoError(t, err) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + assert.ElementsMatch(t, []string{RunFile, EventsFile}, names) + }) +} diff --git a/journal.go b/journal.go new file mode 100644 index 00000000..7d635711 --- /dev/null +++ b/journal.go @@ -0,0 +1,137 @@ +package trader + +import ( + "encoding/json" + "io" + "time" + + "github.com/rustyeddy/trader/num" +) + +// journal records every transition of every order of a run, one JSON line each, in +// the order the transitions happen. It is not a second record of the run: order +// history is still the authority, and the journal only copies what order.record +// writes into it, at the moment it writes it, so the final frame's transitions are +// in it too, whether or not any strategy was ever told of them. +// +// A line carries simulated time only, never the wall clock, so two runs that trade +// the same way write the same lines. Prices and sizes are decimal strings. +// +// The engine opens no file: the journal writes to an io.Writer its caller gives it. +// order.record has no way to return an error and must not change what the engine +// does, so a journal that cannot write keeps the first error (err), stops writing +// and lets the run go on; the run asks for the error at safe points and ends with +// it. +type journal struct { + w io.Writer + seq int + ordinals map[*order]int + failed error +} + +func newJournal(w io.Writer) *journal { + return &journal{w: w, ordinals: make(map[*order]int)} +} + +// journalLine is one line of the journal. Everything but seq, time, order and +// state is left out when it has no value, so a line says only what is new: the +// first line of an order (requested) says what was asked, the others what the +// transition changed, and the fill also says what the order is. +type journalLine struct { + Seq int `json:"seq"` + Time time.Time `json:"time"` // the simulated step the transition happened on + Order int `json:"order"` // 1, 2, ... in the order the journal first saw each order + ID string `json:"id,omitempty"` // the broker's id; empty until the order is queued, and for ever for a refusal + State string `json:"state"` + + // What was asked, on the requested line. + Instrument string `json:"instrument,omitempty"` + Action string `json:"action,omitempty"` + Direction string `json:"direction,omitempty"` + Type string `json:"type,omitempty"` + Cause string `json:"cause,omitempty"` // strategy, stop loss, take profit, margin closeout + Price num.Price `json:"price,omitzero"` + StopLoss num.Price `json:"stop_loss,omitzero"` + TakeProfit num.Price `json:"take_profit,omitzero"` + Trail num.Price `json:"trail,omitzero"` + Expires time.Time `json:"expires,omitzero"` + + // What happened. + Units num.Quantity `json:"units,omitzero"` + FillPrice num.Price `json:"fill_price,omitzero"` + Reason string `json:"reason,omitempty"` // why, for a rejection, a cancellation or an amendment +} + +// record writes the transition e, which order.record has just applied to o. +func (j *journal) record(o *order, e orderEvent) { + if j.failed != nil { + return + } + n, seen := j.ordinals[o] + if !seen { + n = len(j.ordinals) + 1 + j.ordinals[o] = n + } + j.seq++ + l := journalLine{Seq: j.seq, Time: e.time.UTC(), Order: n, ID: o.id, State: stateName(e.state), Reason: e.reason} + if !seen || e.state == stateFilled { + // What the order is, on its first line and again on the fill, so a fill line + // can be read alone: which instrument, what it did and why (a protective + // stop's fill says "stop loss"). + l.Instrument, l.Action, l.Cause = o.instrument, o.action.String(), o.reason.String() + if o.direction != noDirection { + l.Direction = o.direction.String() + } + } + if !seen { + l.Type = o.orderType.String() + l.Price, l.StopLoss, l.TakeProfit, l.Trail, l.Expires = o.price, o.stopLoss, o.takeProfit, o.trail, o.expires + } + switch e.state { + case stateSized, stateSubmitted, stateFilled, stateRejected, stateCancelled: + l.Units = o.units + } + if e.state == stateFilled { + l.FillPrice = o.fillPrice + } + line, err := json.Marshal(l) + if err == nil { + line = append(line, '\n') + var n int + n, err = j.w.Write(line) + if err == nil && n < len(line) { + err = io.ErrShortWrite // a writer may write less and say nothing: a cut line is a failure + } + } + if err != nil { + j.failed = err + } +} + +// err is the first error the journal met, or nil. +func (j *journal) err() error { + if j == nil { + return nil + } + return j.failed +} + +// stateName is the word for a state in the journal. They are the journal's +// vocabulary, fixed whatever the engine calls its states. +func stateName(s orderState) string { + switch s { + case stateRequested: + return "requested" + case stateSized: + return "sized" + case stateSubmitted: + return "submitted" + case stateFilled: + return "filled" + case stateRejected: + return "rejected" + case stateCancelled: + return "cancelled" + } + return "unknown" +} diff --git a/journal_test.go b/journal_test.go new file mode 100644 index 00000000..83866832 --- /dev/null +++ b/journal_test.go @@ -0,0 +1,249 @@ +package trader + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// line is a journal line as a file reader sees it: nothing but its text. +type line map[string]any + +func (l line) str(k string) string { s, _ := l[k].(string); return s } + +func readJournal(t *testing.T, b *bytes.Buffer) []line { + t.Helper() + var out []line + for _, text := range strings.Split(strings.TrimSpace(b.String()), "\n") { + if text == "" { + continue + } + var l line + require.NoError(t, json.Unmarshal([]byte(text), &l), text) + out = append(out, l) + } + return out +} + +// byOrder groups the journal's states by order ordinal. +func byOrder(lines []line) map[int][]line { + out := map[int][]line{} + for _, l := range lines { + n := int(l["order"].(float64)) + out[n] = append(out[n], l) + } + return out +} + +func journaled(t *testing.T, scenario string) ([]line, *Result) { + t.Helper() + captureLogs(t) + var buf bytes.Buffer + res, err := RunSeriesWith(context.Background(), RunOptions{Journal: &buf}, scenario, time.Time{}, scenarioPrices(t, scenario)) + require.NoError(t, err) + return readJournal(t, &buf), res +} + +func TestRunSeriesWith_JournalsEveryTransitionInOrder(t *testing.T) { + lines, _ := journaled(t, "a-reversal-exit") + require.NotEmpty(t, lines) + + for i, l := range lines { + assert.EqualValues(t, i+1, l["seq"], "seq counts the lines") + } + for n, ls := range byOrder(lines) { + assert.Equal(t, "requested", ls[0].str("state"), "order %d: the first line of an order is what was asked, so none was lost", n) + assert.NotEmpty(t, ls[0].str("instrument")) + assert.NotEmpty(t, ls[0].str("action")) + assert.NotEmpty(t, ls[0].str("cause")) + } + + // The strategy's first order is order 1: an open EMAX sized and the broker queued, then filled. + var states []string + for _, l := range byOrder(lines)[1] { + states = append(states, l.str("state")) + } + assert.Equal(t, []string{"requested", "sized", "submitted", "filled"}, states) + first := byOrder(lines)[1] + assert.Equal(t, "open", first[0].str("action")) + assert.Equal(t, "long", first[0].str("direction")) + assert.Equal(t, "market", first[0].str("type")) + assert.Equal(t, "strategy", first[0].str("cause")) + assert.Empty(t, first[0].str("id"), "no id before the broker queues it") + assert.NotEmpty(t, first[1].str("units"), "the account's size, on the sized line") + assert.NotEmpty(t, first[2].str("id"), "the broker's id once queued") + assert.NotEmpty(t, first[3].str("fill_price")) + assert.Equal(t, "open", first[3].str("action"), "a fill line says what it is") +} + +func TestRunSeriesWith_JournalsAProtectiveStopExit(t *testing.T) { + lines, _ := journaled(t, "b2-stop-cross") + + var stopFills []line + for _, l := range lines { + if l.str("state") == "filled" && l.str("cause") == "stop loss" { + stopFills = append(stopFills, l) + } + } + require.Len(t, stopFills, 1, "the broker's stop filled once") + f := stopFills[0] + assert.Equal(t, "close", f.str("action")) + assert.Equal(t, "long", f.str("direction")) + assert.NotEmpty(t, f.str("id")) + assert.NotEmpty(t, f.str("fill_price")) + assert.NotEmpty(t, f.str("units")) + + // The stop was made by the broker when the open filled, and its requested line is in the journal. + stop := byOrder(lines)[int(f["order"].(float64))] + assert.Equal(t, "requested", stop[0].str("state")) + assert.Equal(t, "stop", stop[0].str("type")) + assert.NotEmpty(t, stop[0].str("price"), "where the stop was") +} + +func TestRunSeriesWith_JournalingChangesNothingTheRunTrades(t *testing.T) { + for _, scenario := range scenarios { + t.Run(scenario, func(t *testing.T) { + captureLogs(t) + prices := scenarioPrices(t, scenario) + plain, err := RunSeries(context.Background(), scenario, time.Time{}, prices) + require.NoError(t, err) + + var buf bytes.Buffer + with, err := RunSeriesWith(context.Background(), RunOptions{Journal: &buf}, scenario, time.Time{}, prices) + require.NoError(t, err) + + var a, b bytes.Buffer + require.NoError(t, plain.Write(&a)) + require.NoError(t, with.Write(&b)) + assert.Equal(t, a.String(), b.String()) + assert.Equal(t, plain.Summary(), with.Summary()) + assert.NotZero(t, buf.Len()) + }) + } +} + +func TestRunSeriesWith_TheSameRunWritesTheSameJournal(t *testing.T) { + a, _ := journaled(t, "c-reversal") + b, _ := journaled(t, "c-reversal") + assert.Equal(t, a, b, "simulated time only, no wall clock, no run-varying value") +} + +// failAfterWriter accepts n writes and then fails. +type failAfterWriter struct { + n int + writes int +} + +func (w *failAfterWriter) Write(p []byte) (int, error) { + w.writes++ + if w.writes > w.n { + return 0, errors.New("disk full") + } + return len(p), nil +} + +func TestRunSeriesWith_AJournalThatCannotWriteEndsTheRunWithItsError(t *testing.T) { + captureLogs(t) + prices := scenarioPrices(t, "a-reversal-exit") + + w := &failAfterWriter{n: 3} + res, err := RunSeriesWith(context.Background(), RunOptions{Journal: w}, "x", time.Time{}, prices) + require.ErrorContains(t, err, "journal: disk full") + assert.Nil(t, res) + assert.Equal(t, 4, w.writes, "after the first failure it stops writing") +} + +// shortWriter writes one byte fewer than it is given and reports no error, which +// io.Writer forbids but nothing stops a writer doing. +type shortWriter struct{ buf bytes.Buffer } + +func (w *shortWriter) Write(p []byte) (int, error) { + n, _ := w.buf.Write(p[:len(p)-1]) + return n, nil +} + +func TestRunSeriesWith_AShortWriteIsAFailedJournalNotASuccess(t *testing.T) { + captureLogs(t) + w := &shortWriter{} + + res, err := RunSeriesWith(context.Background(), RunOptions{Journal: w}, "x", time.Time{}, scenarioPrices(t, "a-reversal-exit")) + + require.ErrorIs(t, err, io.ErrShortWrite) + require.ErrorContains(t, err, "journal") + assert.Nil(t, res) + assert.NotContains(t, w.buf.String(), "\n", "no line was reported written, and none is complete") +} + +func TestJournal_ErrOfNilIsNil(t *testing.T) { + var j *journal + assert.NoError(t, j.err()) +} + +func TestJournal_FinalFrameOrdersAreKeptEvenThoughNothingFollows(t *testing.T) { + captureLogs(t) + prices := scenarioPrices(t, remoteScenario) + last := len(prices.Bars) // one group per bar + + s := &feedbackScript{orders: map[int]func(marketdata.Bar) *order{last: entryOrder(108700, 0)}} + tr := newRemoteTraderWith(t, s, usd(1000)) + var buf bytes.Buffer + tr.broker.journal = newJournal(&buf) + require.NoError(t, (&backtest{}).runSeries(context.Background(), tr, prices)) + + lines := readJournal(t, &buf) + require.Equal(t, 3, len(lines), "requested, sized and submitted on the last frame: the broker never saw another bar") + for i, want := range []string{"requested", "sized", "submitted"} { + assert.Equal(t, want, lines[i].str("state")) + } + assert.Empty(t, s.told[last-1], "and no OnBars could have told the strategy") +} + +func TestJournal_ARefusalIsRequestedThenRejectedAndNeverHasAnID(t *testing.T) { + captureLogs(t) + s := &feedbackScript{orders: map[int]func(marketdata.Bar) *order{3: entryOrder(108700, 0)}} + tr := newRemoteTraderWith(t, s, usd(0)) + var buf bytes.Buffer + tr.broker.journal = newJournal(&buf) + require.NoError(t, (&backtest{}).runSeries(context.Background(), tr, scenarioPrices(t, remoteScenario))) + + lines := readJournal(t, &buf) + require.Len(t, lines, 2) + assert.Equal(t, "requested", lines[0].str("state")) + assert.Equal(t, "rejected", lines[1].str("state")) + assert.NotEmpty(t, lines[1].str("reason")) + assert.Empty(t, lines[1].str("id")) +} + +func TestJournal_ATargetAddedByAnAmendIsJournaledFromItsFirstLine(t *testing.T) { + captureLogs(t) + amendTarget := func(b marketdata.Bar) *order { + return &order{instrument: b.Instrument, action: amend, orderType: marketOrder, takeProfit: px(115000), Time: b.Time} + } + s := &feedbackScript{orders: map[int]func(marketdata.Bar) *order{ + 3: entryOrder(108700, 0), // opens with a stop and no target + 6: amendTarget, + }} + tr := newRemoteTraderWith(t, s, usd(1000)) + var buf bytes.Buffer + tr.broker.journal = newJournal(&buf) + require.NoError(t, (&backtest{}).runSeries(context.Background(), tr, scenarioPrices(t, remoteScenario))) + + var target line + for _, ls := range byOrder(readJournal(t, &buf)) { + if ls[0].str("cause") == "take profit" { + target = ls[0] + } + } + require.NotNil(t, target, "the broker's new target order has a journal line") + assert.Equal(t, "requested", target.str("state"), "its first line, not a later one") + assert.Equal(t, "limit", target.str("type")) +} diff --git a/order.go b/order.go index b682a15c..a36c13e5 100644 --- a/order.go +++ b/order.go @@ -130,6 +130,10 @@ type order struct { state orderState history []orderEvent + + // journal, when not nil, is told of every transition record writes. It is set + // before the order's first record, so the requested line is never lost. + journal *journal } // record moves the order to state s at time at, appends the event to its @@ -139,6 +143,9 @@ func (o *order) record(s orderState, at time.Time, reason string) { first := len(o.history) == 0 o.state, o.Time = s, at o.history = append(o.history, orderEvent{state: s, time: at, reason: reason}) + if o.journal != nil { + o.journal.record(o, o.history[len(o.history)-1]) + } attrs := []any{ "order", o.id, diff --git a/service/backtest.go b/service/backtest.go index 85a3d501..5e8df996 100644 --- a/service/backtest.go +++ b/service/backtest.go @@ -4,14 +4,24 @@ import ( "context" "errors" "fmt" + "io" "log/slog" "strings" "time" "github.com/rustyeddy/trader" + "github.com/rustyeddy/trader/internal/clock" + "github.com/rustyeddy/trader/internal/id" + "github.com/rustyeddy/trader/internal/results" "github.com/rustyeddy/trader/marketdata" ) +// DefaultResultsRoot is where a run's results go when nothing else is configured. +const DefaultResultsRoot = results.DefaultRoot + +// runIDs makes the RunID of every run. +var runIDs = id.NewGenerator(clock.Clock, id.Random{}) + // BacktestRequest is what one historical backtest asks for: the canonical bars of // every instrument from one source at one interval whose opening time is in // [From, To), replayed together on one shared account. The values come from the @@ -27,22 +37,45 @@ type BacktestRequest struct { From, To time.Time AllowBad bool AllowMissing bool + + // ResultsRoot is the directory the run's results go under (results.DefaultRoot + // unless the transport was told otherwise). It is required: every run is + // journaled, so there is no way to run without one. + ResultsRoot string +} + +// BacktestRun is a finished backtest: the engine's summary and where the run's +// results are. +type BacktestRun struct { + *trader.Result + RunID id.RunID + // Dir is the run's directory: run.json and events.jsonl. + Dir string } // RunBacktest loads the canonical series for req (LoadSeries) and replays them on -// the engine (trader.RunSeries), returning the engine's summary. It coordinates -// the two and holds no rule of its own: what loads is the market-data service's -// policy, and what runs, and what it validates first, is the engine's. +// the engine (trader.RunSeriesWith), returning the engine's summary and where the +// run's results are. It coordinates the two and holds no rule of its own: what +// loads is the market-data service's policy, and what runs, and what it validates +// first, is the engine's. +// +// Every run is journaled. A run exists from the moment the request is well formed +// enough to describe (the source is supported and there is a results root): it gets +// a RunID and a directory under req.ResultsRoot with run.json and events.jsonl, and +// it ends there with a status, completed, failed or cancelled, whatever happened +// after: bars that cannot be loaded, a request the engine refuses, a cancellation. +// Only an unsupported source, a missing results root or a directory that cannot be +// made stop it before a run exists, since there is then nowhere to record it. // // A request that cannot be run, or an instrument that cannot be loaded as asked, // is an error that names the source, instrument, interval and range (and, for a -// load, the months and their states); a load failure is also ErrNotLoadable, and -// nothing is replayed unless every instrument loaded. A range with no bars at all -// is an error: there is nothing to backtest. The bars of the month before From, when -// they can be loaded, prefill the strategy's indicators and are not traded -// (warmupBars). Cancellation stops the run and -// returns the context's error. Nothing is written. log may be nil. -func RunBacktest(ctx context.Context, log *slog.Logger, req BacktestRequest) (*trader.Result, error) { +// load, the months and their states) and the run's directory; a load failure is +// also ErrNotLoadable, and nothing is replayed unless every instrument loaded. A +// range with no bars at all is an error: there is nothing to backtest. The bars of +// the month before From, when they can be loaded, prefill the strategy's indicators +// and are not traded (warmupBars). Cancellation stops the run and returns the +// context's error. log may be nil. +func RunBacktest(ctx context.Context, log *slog.Logger, req BacktestRequest) (*BacktestRun, error) { source := req.Source if source == "" { source = "oanda" @@ -50,6 +83,33 @@ func RunBacktest(ctx context.Context, log *slog.Logger, req BacktestRequest) (*t if source != "oanda" { return nil, fmt.Errorf("backtest: source %q is not supported (only oanda)", req.Source) } + runID, err := runIDs.RunID() + if err != nil { + return nil, fmt.Errorf("backtest: run id: %w", err) + } + rec := results.Record{ + RunID: runID, Kind: "backtest", Source: source, Interval: req.Interval, + Instruments: req.Instruments, From: req.From.UTC(), To: req.To.UTC(), StartedAt: clock.Clock(), + } + rec.Strategy.Name, rec.Strategy.Transport = trader.StrategyEMAX, "in-process" + run, err := results.Start(req.ResultsRoot, rec) + if err != nil { + return nil, fmt.Errorf("backtest: %w", err) + } + + res, err := runBacktest(ctx, log, req, source, run.Journal()) + var summary any + if res != nil { + summary = res.Summary() + } + if err = run.Finish(err, summary); err != nil { + return nil, fmt.Errorf("%w (run %s, journal in %s)", err, runID, run.Dir()) + } + return &BacktestRun{Result: res, RunID: runID, Dir: run.Dir()}, nil +} + +// runBacktest loads and replays, writing the engine's journal to journal. +func runBacktest(ctx context.Context, log *slog.Logger, req BacktestRequest, source string, journal io.Writer) (*trader.Result, error) { what := fmt.Sprintf("%s %s %s [%s, %s)", source, strings.Join(req.Instruments, ","), req.Interval, req.From.UTC().Format(time.RFC3339), req.To.UTC().Format(time.RFC3339)) @@ -87,7 +147,7 @@ func RunBacktest(ctx context.Context, log *slog.Logger, req BacktestRequest) (*t } series = append(series, s) } - res, err := trader.RunSeries(ctx, what, req.From, series...) + res, err := trader.RunSeriesWith(ctx, trader.RunOptions{Journal: journal}, what, req.From, series...) if err != nil { return nil, fmt.Errorf("backtest %s: %w", what, err) } diff --git a/service/backtest_test.go b/service/backtest_test.go index bf76767c..9ecd8d77 100644 --- a/service/backtest_test.go +++ b/service/backtest_test.go @@ -1,12 +1,18 @@ package service import ( + "bufio" "bytes" "context" + "encoding/json" "fmt" + "os" + "path/filepath" "testing" "time" + "github.com/rustyeddy/trader/internal/id" + "github.com/rustyeddy/trader/internal/results" "github.com/rustyeddy/trader/internal/store/storetest" "github.com/rustyeddy/trader/num" "github.com/rustyeddy/trader/provider/oanda" @@ -58,8 +64,10 @@ func monthSpread(t *testing.T, instrument string, m time.Month, base, spreadTick var wave = []int64{0, 0, 0, 0, 500, 1000, 1500, 1000, 500, 0, -500, -1000} -func backtestReq(instruments ...string) BacktestRequest { - return BacktestRequest{Instruments: instruments, Interval: "h1", From: utc(2024, 1, 1, 0), To: utc(2024, 2, 1, 0)} +// backtestReq is a request for January 2024, journaled under a temporary results root. +func backtestReq(t *testing.T, instruments ...string) BacktestRequest { + t.Helper() + return BacktestRequest{Instruments: instruments, Interval: "h1", From: utc(2024, 1, 1, 0), To: utc(2024, 2, 1, 0), ResultsRoot: t.TempDir()} } func buildCanonical(t *testing.T, req BuildRequest) { @@ -74,7 +82,7 @@ func TestRunBacktest_LoadsTwoInstrumentsFromCanonicalStorageAndRunsThem(t *testi month(t, "GBPUSD", time.January, 125000, wave...) buildCanonical(t, janToMar("EURUSD", "GBPUSD")) - res, err := RunBacktest(context.Background(), nil, backtestReq("EURUSD", "GBPUSD")) + res, err := RunBacktest(context.Background(), nil, backtestReq(t, "EURUSD", "GBPUSD")) require.NoError(t, err) var out bytes.Buffer @@ -88,7 +96,7 @@ func TestRunBacktest_ARangeIsCutByOpeningTime(t *testing.T) { storetest.Root(t) month(t, "EURUSD", time.January, 110000, wave...) buildCanonical(t, janToMar("EURUSD")) - req := backtestReq("EURUSD") + req := backtestReq(t, "EURUSD") req.From, req.To = utc(2024, 1, 2, 2), utc(2024, 1, 2, 8) // opening times 02:00 to 07:00 res, err := RunBacktest(context.Background(), nil, req) @@ -103,7 +111,7 @@ func TestRunBacktest_AMissingMonthFailsUnlessAllowed(t *testing.T) { storetest.Root(t) month(t, "EURUSD", time.January, 110000, wave...) buildCanonical(t, janToMar("EURUSD")) - req := backtestReq("EURUSD") + req := backtestReq(t, "EURUSD") req.To = utc(2024, 3, 1, 0) // February has no data res, err := RunBacktest(context.Background(), nil, req) @@ -128,7 +136,7 @@ func TestRunBacktest_ABadMonthFailsUnlessAllowedAndNothingRunsWhenOneInstrumentF putRaw(t, "GBPUSD", 2024, time.January, rows...) buildCanonical(t, janToMar("EURUSD", "GBPUSD")) - res, err := RunBacktest(context.Background(), nil, backtestReq("EURUSD", "GBPUSD")) + res, err := RunBacktest(context.Background(), nil, backtestReq(t, "EURUSD", "GBPUSD")) require.ErrorIs(t, err, ErrNotLoadable) assert.Nil(t, res) @@ -136,7 +144,7 @@ func TestRunBacktest_ABadMonthFailsUnlessAllowedAndNothingRunsWhenOneInstrumentF assert.Contains(t, err.Error(), "2024-01 bad") assert.NotContains(t, err.Error(), "EURUSD: ", "the instrument that loaded is not blamed") - req := backtestReq("EURUSD", "GBPUSD") + req := backtestReq(t, "EURUSD", "GBPUSD") req.AllowBad = true res, err = RunBacktest(context.Background(), nil, req) require.NoError(t, err) @@ -147,14 +155,14 @@ func TestRunBacktest_RefusesWhatCannotBeRun(t *testing.T) { storetest.Root(t) month(t, "EURUSD", time.January, 110000, wave...) buildCanonical(t, janToMar("EURUSD")) - reversed := backtestReq("EURUSD") + reversed := backtestReq(t, "EURUSD") reversed.From, reversed.To = reversed.To, reversed.From - otherSource := backtestReq("EURUSD") + otherSource := backtestReq(t, "EURUSD") otherSource.Source = "alpaca" - noInstrument := backtestReq() - badInterval := backtestReq("EURUSD") + noInstrument := backtestReq(t) + badInterval := backtestReq(t, "EURUSD") badInterval.Interval = "w1" - nothing := backtestReq("EURUSD") + nothing := backtestReq(t, "EURUSD") nothing.From, nothing.To = utc(2024, 1, 20, 0), utc(2024, 1, 21, 0) // the month loads, no bar opens in it for name, tc := range map[string]struct { req BacktestRequest @@ -181,7 +189,7 @@ func TestRunBacktest_AnInstrumentTheEngineHasNoMarketForIsRefusedByTheEngine(t * month(t, "USDJPY", time.January, 14000000, wave...) buildCanonical(t, janToMar("USDJPY")) - res, err := RunBacktest(context.Background(), nil, backtestReq("USDJPY")) + res, err := RunBacktest(context.Background(), nil, backtestReq(t, "USDJPY")) require.Error(t, err) assert.Nil(t, res) @@ -196,7 +204,7 @@ func TestRunBacktest_ACancelledContextStopsTheRun(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := RunBacktest(ctx, nil, backtestReq("EURUSD")) + _, err := RunBacktest(ctx, nil, backtestReq(t, "EURUSD")) require.ErrorIs(t, err, context.Canceled) } @@ -216,7 +224,7 @@ func TestRunBacktest_ARealisticOneTickSpreadRunsAndKeepsTheHalfTickMid(t *testin require.NoError(t, err) require.False(t, onTick, "the canonical mid sits between two ticks: %s", loaded[0].Series.Bars[0].Mid.Close) - res, err := RunBacktest(context.Background(), nil, backtestReq("EURUSD", "GBPUSD")) + res, err := RunBacktest(context.Background(), nil, backtestReq(t, "EURUSD", "GBPUSD")) require.NoError(t, err) var out bytes.Buffer @@ -234,7 +242,7 @@ func TestRunBacktest_ARealisticOneTickSpreadRunsAndKeepsTheHalfTickMid(t *testin // January, the month before, prefill it when they can be loaded. func TestRunBacktest_TheMonthBeforeTheRangePrefillsTheIndicators(t *testing.T) { febReq := func() BacktestRequest { - r := backtestReq("EURUSD") + r := backtestReq(t, "EURUSD") r.From, r.To = utc(2024, 2, 1, 0), utc(2024, 3, 1, 0) return r } @@ -293,14 +301,220 @@ func TestRunBacktest_AnInteriorGapTheCalendarDoesNotExplainFailsUnlessAllowed(t putRaw(t, "EURUSD", 2024, time.January, rows...) buildCanonical(t, janToMar("EURUSD")) - res, err := RunBacktest(context.Background(), nil, backtestReq("EURUSD")) + res, err := RunBacktest(context.Background(), nil, backtestReq(t, "EURUSD")) require.ErrorIs(t, err, ErrNotLoadable) assert.Nil(t, res, "nothing is replayed over a hole") assert.Contains(t, err.Error(), "gap unexplained 2024-01-09T04:00Z 2024-01-09T08:00Z, 4 bars") - req := backtestReq("EURUSD") + req := backtestReq(t, "EURUSD") req.AllowMissing = true res, err = RunBacktest(context.Background(), nil, req) require.NoError(t, err) assert.NotNil(t, res) } + +// A run, as a client such as Investor reads it: two files, nothing else. +type runFiles struct { + rec map[string]any + events []map[string]any +} + +func readRunDir(t *testing.T, dir string) runFiles { + t.Helper() + var r runFiles + data, err := os.ReadFile(filepath.Join(dir, results.RunFile)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &r.rec)) + f, err := os.Open(filepath.Join(dir, results.EventsFile)) + require.NoError(t, err) + defer func() { _ = f.Close() }() + sc := bufio.NewScanner(f) + for sc.Scan() { + var e map[string]any + require.NoError(t, json.Unmarshal(sc.Bytes(), &e), sc.Text()) + r.events = append(r.events, e) + } + require.NoError(t, sc.Err()) + return r +} + +// onlyRun is the one run directory under root. +func onlyRun(t *testing.T, root string) string { + t.Helper() + entries, err := os.ReadDir(root) + require.NoError(t, err) + require.Len(t, entries, 1) + _, err = id.ParseRunID(entries[0].Name()) + require.NoError(t, err, "a run directory is named by its RunID") + return filepath.Join(root, entries[0].Name()) +} + +func TestRunBacktest_JournalsTheRunInADirectoryNamedByItsRunID(t *testing.T) { + storetest.Root(t) + month(t, "EURUSD", time.January, 110000, wave...) + buildCanonical(t, janToMar("EURUSD")) + req := backtestReq(t, "EURUSD") + + res, err := RunBacktest(context.Background(), nil, req) + require.NoError(t, err) + + assert.Equal(t, onlyRun(t, req.ResultsRoot), res.Dir) + assert.Equal(t, filepath.Base(res.Dir), res.RunID.String()) + + // From here on only the files are read: no trader object. + run := readRunDir(t, res.Dir) + assert.Equal(t, res.RunID.String(), run.rec["run_id"]) + assert.Equal(t, "backtest", run.rec["kind"]) + assert.Equal(t, map[string]any{"name": "emax", "transport": "in-process"}, run.rec["strategy"]) + assert.Equal(t, "oanda", run.rec["source"]) + assert.Equal(t, "h1", run.rec["interval"]) + assert.Equal(t, []any{"EURUSD"}, run.rec["instruments"]) + assert.Equal(t, "2024-01-01T00:00:00Z", run.rec["from"]) + assert.Equal(t, "2024-02-01T00:00:00Z", run.rec["to"]) + assert.Equal(t, "completed", run.rec["status"]) + assert.NotContains(t, run.rec, "error") + started, err := time.Parse(time.RFC3339Nano, run.rec["started_at"].(string)) + require.NoError(t, err) + ended, err := time.Parse(time.RFC3339Nano, run.rec["ended_at"].(string)) + require.NoError(t, err) + assert.False(t, ended.Before(started)) + runStarted, err := res.RunID.Time() + require.NoError(t, err) + assert.WithinDuration(t, started, runStarted, time.Second, "the RunID carries the time the run began") + + // An opening request, its admission, its fill, and a closing fill. + byOrder := map[float64][]map[string]any{} + for i, e := range run.events { + assert.EqualValues(t, i+1, e["seq"]) + byOrder[e["order"].(float64)] = append(byOrder[e["order"].(float64)], e) + } + open := byOrder[1] + require.GreaterOrEqual(t, len(open), 4) + for i, want := range []string{"requested", "sized", "submitted", "filled"} { + assert.Equal(t, want, open[i]["state"]) + } + assert.Equal(t, "open", open[0]["action"]) + assert.Equal(t, "EURUSD", open[0]["instrument"]) + assert.Equal(t, "strategy", open[0]["cause"]) + assert.NotEmpty(t, open[0]["stop_loss"], "the stop the strategy asked for") + assert.NotEmpty(t, open[1]["units"], "the size the account chose") + assert.NotEmpty(t, open[3]["fill_price"]) + + var closed bool + for _, e := range run.events { + if e["state"] == "filled" && e["action"] == "close" { + closed = true + } + } + assert.True(t, closed, "a closing fill, by the strategy or the broker's stop") + + // The record's result is the engine's summary, and agrees with the journal: an + // order is submitted once, and no run here amends a stop or trails one. + result := run.rec["result"].(map[string]any) + assert.Equal(t, "emax", result["strategy"]) + var submitted float64 + for _, e := range run.events { + if e["state"] == "submitted" { + submitted++ + } + } + assert.Equal(t, result["orders"].(map[string]any)["submitted"], submitted, + "every order the engine counts as submitted has one submitted line") +} + +func TestRunBacktest_ARunThatFailsIsStillJournaledAsFailed(t *testing.T) { + storetest.Root(t) + month(t, "EURUSD", time.January, 110000, wave...) + buildCanonical(t, janToMar("EURUSD")) + req := backtestReq(t, "EURUSD") + req.To = utc(2024, 3, 1, 0) // February was never built + + res, err := RunBacktest(context.Background(), nil, req) + + require.ErrorIs(t, err, ErrNotLoadable) + assert.Nil(t, res) + dir := onlyRun(t, req.ResultsRoot) + assert.ErrorContains(t, err, dir, "the error says where the run is") + run := readRunDir(t, dir) + assert.Equal(t, "failed", run.rec["status"]) + assert.Contains(t, run.rec["error"], "EURUSD") + assert.NotContains(t, run.rec, "result") + assert.Equal(t, "backtest", run.rec["kind"]) + assert.NotEmpty(t, run.rec["ended_at"]) + assert.Empty(t, run.events, "it never got as far as an order") +} + +func TestRunBacktest_ARequestTheEngineRefusesIsAFailedRunToo(t *testing.T) { + storetest.Root(t) + req := backtestReq(t) + req.Interval = "m5" + + _, err := RunBacktest(context.Background(), nil, req) + + require.Error(t, err) + run := readRunDir(t, onlyRun(t, req.ResultsRoot)) + assert.Equal(t, "failed", run.rec["status"]) + assert.NotEmpty(t, run.rec["error"]) +} + +func TestRunBacktest_ACancelledRunIsJournaledAsCancelled(t *testing.T) { + storetest.Root(t) + month(t, "EURUSD", time.January, 110000, wave...) + buildCanonical(t, janToMar("EURUSD")) + req := backtestReq(t, "EURUSD") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := RunBacktest(ctx, nil, req) + + require.ErrorIs(t, err, context.Canceled) + run := readRunDir(t, onlyRun(t, req.ResultsRoot)) + assert.Equal(t, "cancelled", run.rec["status"]) +} + +func TestRunBacktest_NothingRunsWithoutSomewhereToRecordIt(t *testing.T) { + storetest.Root(t) + month(t, "EURUSD", time.January, 110000, wave...) + buildCanonical(t, janToMar("EURUSD")) + + t.Run("no results root", func(t *testing.T) { + req := backtestReq(t, "EURUSD") + req.ResultsRoot = "" + _, err := RunBacktest(context.Background(), nil, req) + assert.ErrorContains(t, err, "no results root") + }) + t.Run("a root that cannot be made", func(t *testing.T) { + file := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + req := backtestReq(t, "EURUSD") + req.ResultsRoot = file + _, err := RunBacktest(context.Background(), nil, req) + assert.ErrorContains(t, err, "results root") + }) + t.Run("an unsupported source is refused before a run exists", func(t *testing.T) { + req := backtestReq(t, "EURUSD") + req.Source = "stooq" + _, err := RunBacktest(context.Background(), nil, req) + assert.ErrorContains(t, err, "not supported") + entries, rerr := os.ReadDir(req.ResultsRoot) + require.NoError(t, rerr) + assert.Empty(t, entries, "no run, so no directory") + }) +} + +func TestRunBacktest_EveryRunGetsItsOwnDirectory(t *testing.T) { + storetest.Root(t) + month(t, "EURUSD", time.January, 110000, wave...) + buildCanonical(t, janToMar("EURUSD")) + req := backtestReq(t, "EURUSD") + + first, err := RunBacktest(context.Background(), nil, req) + require.NoError(t, err) + second, err := RunBacktest(context.Background(), nil, req) + require.NoError(t, err) + + assert.NotEqual(t, first.Dir, second.Dir) + assert.Less(t, first.RunID.String(), second.RunID.String(), "later runs sort later") + a, b := readRunDir(t, first.Dir), readRunDir(t, second.Dir) + assert.Equal(t, a.events, b.events, "the same run writes the same journal; only run.json carries a run's own identity") +} diff --git a/service/marketdata_test.go b/service/marketdata_test.go index f6457830..464368b1 100644 --- a/service/marketdata_test.go +++ b/service/marketdata_test.go @@ -224,7 +224,8 @@ func TestBuildCanonical_BuiltAtDefaultsToNow(t *testing.T) { // other file may. func TestService_ImportsOnlyTheDomainServiceAndProviders(t *testing.T) { const mod = "github.com/rustyeddy/trader" - allowed := map[string]bool{mod: true, mod + "/marketdata": true, mod + "/marketdata/service": true, mod + "/provider/oanda": true} + allowed := map[string]bool{mod: true, mod + "/marketdata": true, mod + "/marketdata/service": true, mod + "/provider/oanda": true, + mod + "/internal/clock": true, mod + "/internal/id": true, mod + "/internal/results": true} // foundational: the run's id, its clock and where its results are kept foundational := map[string]bool{"migrate.go": true} files, err := filepath.Glob("*.go") require.NoError(t, err) diff --git a/trader.go b/trader.go index 8d092c39..021cfcaf 100644 --- a/trader.go +++ b/trader.go @@ -129,6 +129,11 @@ func (tr *trader) onFrame(ctx context.Context, f frame) error { return err } } + // The journal could not write: stop at this safe point, after the frame is + // whole, rather than trade on without the record the run is meant to keep. + if err := tr.broker.journal.err(); err != nil { + return fmt.Errorf("journal: %w", err) + } return nil } @@ -153,6 +158,7 @@ func (tr *trader) handleOrder(ctx context.Context, o *order) error { } tr.watch(o) + o.journal = tr.broker.journal // before the first record, so the requested line is kept o.record(stateRequested, o.Time, "") if err := tr.broker.submit(ctx, o); err != nil { if !isRejection(err) {