diff --git a/backtest_test.go b/backtest_test.go index 375481e..ffa9c58 100644 --- a/backtest_test.go +++ b/backtest_test.go @@ -39,7 +39,7 @@ func (s *scriptStrategy) onBar(_ context.Context, b marketdata.Bar) *order { return nil } -func (s *scriptStrategy) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *scriptStrategy) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } @@ -204,7 +204,7 @@ func (s *cancelStrategy) onBar(ctx context.Context, b marketdata.Bar) *order { return s.scriptStrategy.onBar(ctx, b) } -func (s *cancelStrategy) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *cancelStrategy) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } @@ -323,7 +323,7 @@ func (s *rejectFirstOpen) onBar(ctx context.Context, b marketdata.Bar) *order { return o } -func (s *rejectFirstOpen) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *rejectFirstOpen) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } @@ -397,7 +397,7 @@ func (s *seenStrategy) onBar(_ context.Context, b marketdata.Bar) *order { return nil } -func (s *seenStrategy) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *seenStrategy) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } @@ -519,7 +519,12 @@ func TestBacktestRunSeries_SingleInstrumentMatchesThePerBarRun(t *testing.T) { } // first is the first of orders, or nil. -func first(orders []*order) *order { +// first is the first order a strategy's onBars returned, or nil. The strategies it +// wraps cannot fail, so an error is a bug in the test. +func first(orders []*order, err error) *order { + if err != nil { + panic(err) + } if len(orders) == 0 { return nil } diff --git a/emax_backtest_test.go b/emax_backtest_test.go index 9ade617..b0113c9 100644 --- a/emax_backtest_test.go +++ b/emax_backtest_test.go @@ -645,7 +645,7 @@ func (w *positionWatcher) onBar(ctx context.Context, b marketdata.Bar) *order { return first(w.strategy.onBars(ctx, []marketdata.Bar{b})) } -func (w *positionWatcher) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (w *positionWatcher) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, w.onBar) } diff --git a/examples/remote-strategy/main.go b/examples/remote-strategy/main.go new file mode 100644 index 0000000..9aa4399 --- /dev/null +++ b/examples/remote-strategy/main.go @@ -0,0 +1,42 @@ +// Command remote-strategy is a minimal external strategy for Trader: it listens +// on a loopback port, takes each group of bars Trader sends and always answers +// with no orders. +// +// Run it, then point a Trader backtest at the address it prints: +// +// go run ./examples/remote-strategy -addr 127.0.0.1:9100 +// +// It serves one connection and exits when Trader closes it. The wire protocol is +// described in package strategy. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + + "github.com/rustyeddy/trader/marketdata" + "github.com/rustyeddy/trader/strategy" +) + +func main() { + addr := flag.String("addr", "127.0.0.1:0", "loopback address to listen on") + flag.Parse() + + groups := 0 + err := strategy.ListenAndServe(*addr, + func(a net.Addr) { + fmt.Println("listening on", a) + }, // for whoever started us + func([]marketdata.Bar) error { + // A real strategy decides here. This one only counts. + groups++ + return nil + }) + if err != nil { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "done: %d groups\n", groups) +} diff --git a/historical_test.go b/historical_test.go index 0ade6d5..bf505b7 100644 --- a/historical_test.go +++ b/historical_test.go @@ -406,7 +406,7 @@ type recordingStrategy struct{ calls [][]marketdata.Bar } func (*recordingStrategy) name() string { return "recording" } func (*recordingStrategy) init() error { return nil } -func (r *recordingStrategy) onBars(_ context.Context, bars []marketdata.Bar) []*order { +func (r *recordingStrategy) onBars(_ context.Context, bars []marketdata.Bar) ([]*order, error) { r.calls = append(r.calls, slices.Clone(bars)) - return nil + return nil, nil } diff --git a/strategy.go b/strategy.go index c115591..7322c11 100644 --- a/strategy.go +++ b/strategy.go @@ -11,13 +11,14 @@ import ( // strategy decides what to trade. It is called once per availability instant with // every bar that became available then (the bars are fresh and in instrument // order; an instrument with no bar at that instant is absent) and returns the -// orders it wants, in the order they should be submitted, or none. It sees the +// orders it wants, in the order they should be submitted, or none, or an error, +// which stops the run before any order is used. It sees the // whole group before any of its orders is submitted. A single-instrument run is // the same call with a one-bar group. bars is only valid for the call. type strategy interface { name() string init() error - onBars(ctx context.Context, bars []marketdata.Bar) []*order + onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) } // warmer is implemented by a strategy that can prefill its indicators. The bars @@ -30,15 +31,16 @@ type warmer interface { } // eachBar adapts a decision made one bar at a time: it asks decide about every bar -// of the group in order and returns the orders it gave, in that order. -func eachBar(ctx context.Context, bars []marketdata.Bar, decide func(context.Context, marketdata.Bar) *order) []*order { +// of the group in order and returns the orders it gave, in that order. A decision +// that is only a bar-by-bar rule cannot fail, so the error is always nil. +func eachBar(ctx context.Context, bars []marketdata.Bar, decide func(context.Context, marketdata.Bar) *order) ([]*order, error) { var orders []*order for _, b := range bars { if o := decide(ctx, b); o != nil { orders = append(orders, o) } } - return orders + return orders, nil } // emax is the EMA crossover strategy: long while the short EMA is above the long @@ -77,7 +79,7 @@ func (e *emax) warm(_ context.Context, bars []marketdata.Bar) { } // onBars is onBar for each bar of the group. -func (e *emax) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (e *emax) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, e.onBar) } @@ -257,7 +259,7 @@ func (s *emaxEach) warm(ctx context.Context, bars []marketdata.Bar) { // onBars hands each bar to its instrument's emax; the group's orders come back in // the group's instrument order. -func (s *emaxEach) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *emaxEach) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } diff --git a/strategy/client.go b/strategy/client.go new file mode 100644 index 0000000..ad1c639 --- /dev/null +++ b/strategy/client.go @@ -0,0 +1,89 @@ +package strategy + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "time" + + "github.com/rustyeddy/trader/marketdata" +) + +// DefaultTimeout is how long Client waits for the worker to answer one group. +const DefaultTimeout = 30 * time.Second + +// Client is the trader's end of a connection to a worker. +type Client struct { + // Timeout is how long OnBars waits for an answer; zero means DefaultTimeout. + Timeout time.Duration + + conn net.Conn +} + +// Dial connects to a worker listening at addr, which must be a loopback address: +// nothing here is authenticated or encrypted. +func Dial(addr string) (*Client, error) { + if err := requireLoopback(addr); err != nil { + return nil, err + } + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + return nil, err + } + return &Client{conn: conn}, nil +} + +// Close ends the session; the worker sees the connection close and stops. +func (r *Client) Close() error { return r.conn.Close() } + +// OnBars sends one group of bars and waits for the answer. It returns an error if +// the worker does not answer within the timeout or ctx ends first, if the +// connection fails, or if the answer holds any orders. After an error the +// connection should be closed. +func (r *Client) OnBars(ctx context.Context, bars []marketdata.Bar) error { + timeout := r.Timeout + if timeout == 0 { + timeout = DefaultTimeout + } + deadline := time.Now().Add(timeout) + if d, ok := ctx.Deadline(); ok && d.Before(deadline) { + deadline = d + } + if err := r.conn.SetDeadline(deadline); err != nil { + return err + } + // Cancelling ctx expires the deadline at once, which wakes a blocked read. + stop := context.AfterFunc(ctx, func() { _ = r.conn.SetDeadline(time.Unix(1, 0)) }) + defer stop() + + err := r.exchange(bars) + switch { + case err == nil: + return nil + case ctx.Err() != nil: + return ctx.Err() + case errors.Is(err, os.ErrDeadlineExceeded): + // The socket's timer can fire a moment before ctx notices its own deadline. + if d, ok := ctx.Deadline(); ok && !time.Now().Before(d) { + return context.DeadlineExceeded + } + return fmt.Errorf("worker did not answer within %s: %w", timeout, err) + } + return err +} + +func (r *Client) exchange(bars []marketdata.Bar) error { + if err := writeMessage(r.conn, barsMessage{Bars: bars}); err != nil { + return fmt.Errorf("sending bars: %w", err) + } + var reply ordersMessage + if err := readMessage(r.conn, &reply); err != nil { + return fmt.Errorf("reading orders: %w", err) + } + if len(reply.Orders) > 0 { + return fmt.Errorf("worker returned %d orders; orders over the wire are not supported yet", len(reply.Orders)) + } + return nil +} diff --git a/strategy/client_test.go b/strategy/client_test.go new file mode 100644 index 0000000..126f2ab --- /dev/null +++ b/strategy/client_test.go @@ -0,0 +1,144 @@ +package strategy + +import ( + "context" + "encoding/json" + "net" + "testing" + "time" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// rawWorker listens on loopback and hands the one connection it accepts to +// handle, which speaks the protocol by hand. +func rawWorker(t *testing.T, handle func(net.Conn)) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + handle(conn) + }() + return ln.Addr().String() +} + +var oneBar = []marketdata.Bar{{Instrument: "EURUSD", Volume: 1}} + +func TestDial_LoopbackOnly(t *testing.T) { + for _, addr := range []string{"8.8.8.8:9100", "example.com:9100", "0.0.0.0:9100", "no-port"} { + _, err := Dial(addr) + assert.Error(t, err, addr) + } +} + +func TestDial_NothingListening(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + + _, err = Dial(addr) + assert.Error(t, err) +} + +func TestClient_OnBars(t *testing.T) { + ctx := context.Background() + + t.Run("an empty answer is success", func(t *testing.T) { + addr := rawWorker(t, func(c net.Conn) { + for { + var in barsMessage + if readMessage(c, &in) != nil { + return + } + _ = writeMessage(c, ordersMessage{}) + } + }) + r, err := Dial(addr) + require.NoError(t, err) + defer func() { _ = r.Close() }() + assert.NoError(t, r.OnBars(ctx, oneBar)) + assert.NoError(t, r.OnBars(ctx, oneBar)) + }) + t.Run("orders in the answer are an error", func(t *testing.T) { + addr := rawWorker(t, func(c net.Conn) { + var in barsMessage + _ = readMessage(c, &in) + _ = writeMessage(c, ordersMessage{Orders: []json.RawMessage{json.RawMessage(`{}`)}}) + }) + r, err := Dial(addr) + require.NoError(t, err) + defer func() { _ = r.Close() }() + assert.ErrorContains(t, r.OnBars(ctx, oneBar), "not supported") + }) + t.Run("a worker that hangs up is an error", func(t *testing.T) { + addr := rawWorker(t, func(c net.Conn) { + var in barsMessage + _ = readMessage(c, &in) + }) + r, err := Dial(addr) + require.NoError(t, err) + defer func() { _ = r.Close() }() + assert.ErrorContains(t, r.OnBars(ctx, oneBar), "reading orders") + }) + t.Run("a silent worker times out", func(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + addr := rawWorker(t, func(c net.Conn) { + var in barsMessage + _ = readMessage(c, &in) + <-release // connected, took the bars, never answers + }) + r, err := Dial(addr) + require.NoError(t, err) + defer func() { _ = r.Close() }() + r.Timeout = 50 * time.Millisecond + + start := time.Now() + err = r.OnBars(ctx, oneBar) + assert.ErrorContains(t, err, "did not answer within") + assert.Less(t, time.Since(start), 2*time.Second) + }) + t.Run("cancelling the context stops a silent worker's wait", func(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + addr := rawWorker(t, func(c net.Conn) { + var in barsMessage + _ = readMessage(c, &in) + <-release + }) + r, err := Dial(addr) // default 30 s timeout: only the cancel can end this in time + require.NoError(t, err) + defer func() { _ = r.Close() }() + + cctx, cancel := context.WithCancel(ctx) + time.AfterFunc(50*time.Millisecond, cancel) + start := time.Now() + assert.ErrorIs(t, r.OnBars(cctx, oneBar), context.Canceled) + assert.Less(t, time.Since(start), 2*time.Second) + }) + t.Run("a context deadline shorter than the timeout wins", func(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + addr := rawWorker(t, func(c net.Conn) { + var in barsMessage + _ = readMessage(c, &in) + <-release + }) + r, err := Dial(addr) + require.NoError(t, err) + defer func() { _ = r.Close() }() + + cctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + assert.ErrorIs(t, r.OnBars(cctx, oneBar), context.DeadlineExceeded) + }) +} diff --git a/strategy/frame.go b/strategy/frame.go new file mode 100644 index 0000000..e584da8 --- /dev/null +++ b/strategy/frame.go @@ -0,0 +1,53 @@ +package strategy + +import ( + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" +) + +// maxMessage caps one message body, so a garbled length prefix cannot ask for +// gigabytes. Both sides read through readMessage, so both are bounded. +const maxMessage = 16 << 20 + +// writeMessage sends v as one length-prefixed JSON message, in a single write. +func writeMessage(w io.Writer, v any) error { + body, err := json.Marshal(v) + if err != nil { + return err + } + if len(body) > maxMessage { + return fmt.Errorf("message of %d bytes exceeds %d", len(body), maxMessage) + } + buf := binary.BigEndian.AppendUint32(make([]byte, 0, 4+len(body)), uint32(len(body))) + buf = append(buf, body...) + n, err := w.Write(buf) + if err == nil && n < len(buf) { + err = io.ErrShortWrite + } + return err +} + +// readMessage reads one length-prefixed JSON message into v. It returns io.EOF +// when the peer closed before a message began, and io.ErrUnexpectedEOF when it +// closed in the middle of one. +func readMessage(r io.Reader, v any) error { + var prefix [4]byte + if _, err := io.ReadFull(r, prefix[:]); err != nil { + return err + } + n := binary.BigEndian.Uint32(prefix[:]) + if n > maxMessage { + return fmt.Errorf("message of %d bytes exceeds %d", n, maxMessage) + } + body := make([]byte, n) + if _, err := io.ReadFull(r, body); err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return err + } + return json.Unmarshal(body, v) +} diff --git a/strategy/frame_test.go b/strategy/frame_test.go new file mode 100644 index 0000000..ee8eef0 --- /dev/null +++ b/strategy/frame_test.go @@ -0,0 +1,63 @@ +package strategy + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "testing" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// shortWriter accepts only n bytes of each write and reports no error. +type shortWriter struct{ n int } + +func (s shortWriter) Write(p []byte) (int, error) { return min(s.n, len(p)), nil } + +func TestMessage_RoundTrip(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeMessage(&buf, barsMessage{Bars: []marketdata.Bar{{Instrument: "EURUSD", Volume: 7}}})) + assert.Equal(t, uint32(buf.Len()-4), binary.BigEndian.Uint32(buf.Bytes()), "the prefix is the body length") + + var got barsMessage + require.NoError(t, readMessage(&buf, &got)) + require.Len(t, got.Bars, 1) + assert.Equal(t, "EURUSD", got.Bars[0].Instrument) + assert.Equal(t, int64(7), got.Bars[0].Volume) +} + +func TestWriteMessage_ShortWrite(t *testing.T) { + err := writeMessage(shortWriter{n: 3}, ordersMessage{}) + assert.ErrorIs(t, err, io.ErrShortWrite, "a write that sends part of a frame is an error, not success") +} + +func TestWriteMessage_TooLarge(t *testing.T) { + big := struct{ Data string }{Data: strings.Repeat("a", maxMessage)} + assert.ErrorContains(t, writeMessage(io.Discard, big), "exceeds") +} + +func TestReadMessage_Errors(t *testing.T) { + var v ordersMessage + frameOf := func(body string) []byte { + return append(binary.BigEndian.AppendUint32(nil, uint32(len(body))), body...) + } + t.Run("closed before a message is a clean EOF", func(t *testing.T) { + assert.ErrorIs(t, readMessage(bytes.NewReader(nil), &v), io.EOF) + }) + t.Run("closed inside the prefix", func(t *testing.T) { + assert.ErrorIs(t, readMessage(bytes.NewReader([]byte{0, 0}), &v), io.ErrUnexpectedEOF) + }) + t.Run("closed inside the body", func(t *testing.T) { + assert.ErrorIs(t, readMessage(bytes.NewReader(frameOf(`{"orders":[]}`)[:8]), &v), io.ErrUnexpectedEOF) + }) + t.Run("a body that is not JSON", func(t *testing.T) { + assert.Error(t, readMessage(bytes.NewReader(frameOf(`nope`)), &v)) + }) + t.Run("a length over the cap is refused before it is read", func(t *testing.T) { + huge := binary.BigEndian.AppendUint32(nil, maxMessage+1) + assert.ErrorContains(t, readMessage(bytes.NewReader(huge), &v), "exceeds") + }) +} diff --git a/strategy/server.go b/strategy/server.go new file mode 100644 index 0000000..94e3c55 --- /dev/null +++ b/strategy/server.go @@ -0,0 +1,55 @@ +package strategy + +import ( + "encoding/json" + "io" + "net" + + "github.com/rustyeddy/trader/marketdata" +) + +// ListenAndServe listens on the loopback address addr, accepts one connection +// from the trader and serves it (see Serve) until the trader closes it. ready, if +// not nil, is called once with the address actually bound, which is how a caller +// learns the port of "127.0.0.1:0". +func ListenAndServe(addr string, ready func(net.Addr), onBars func(bars []marketdata.Bar) error) error { + if err := requireLoopback(addr); err != nil { + return err + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + defer func() { _ = ln.Close() }() + if ready != nil { + ready(ln.Addr()) + } + conn, err := ln.Accept() + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + return Serve(conn, onBars) +} + +// Serve is the worker's end of a connection. It reads each group of bars, calls +// onBars with it and answers with no orders, until the trader closes the +// connection (Serve then returns nil). If onBars returns an error Serve returns +// it without answering. Serve does not close conn. +func Serve(conn net.Conn, onBars func(bars []marketdata.Bar) error) error { + for { + var in barsMessage + if err := readMessage(conn, &in); err != nil { + if err == io.EOF { + return nil + } + return err + } + if err := onBars(in.Bars); err != nil { + return err + } + if err := writeMessage(conn, ordersMessage{Orders: []json.RawMessage{}}); err != nil { + return err + } + } +} diff --git a/strategy/server_test.go b/strategy/server_test.go new file mode 100644 index 0000000..21262b6 --- /dev/null +++ b/strategy/server_test.go @@ -0,0 +1,102 @@ +package strategy + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServe(t *testing.T) { + t.Run("answers each group with no orders until the peer closes", func(t *testing.T) { + a, b := net.Pipe() + var got [][]marketdata.Bar + done := make(chan error, 1) + go func() { + done <- Serve(b, func(bars []marketdata.Bar) error { got = append(got, bars); return nil }) + }() + + for i := 0; i < 3; i++ { + require.NoError(t, writeMessage(a, barsMessage{Bars: oneBar})) + var out ordersMessage + require.NoError(t, readMessage(a, &out)) + assert.NotNil(t, out.Orders, "an empty list on the wire, not null") + assert.Empty(t, out.Orders) + } + require.NoError(t, a.Close()) + require.NoError(t, <-done, "a closed connection is a clean end") + assert.Len(t, got, 3) + }) + t.Run("an error from onBars ends it without an answer", func(t *testing.T) { + a, b := net.Pipe() + defer func() { _ = a.Close() }() + boom := errors.New("boom") + done := make(chan error, 1) + go func() { done <- Serve(b, func([]marketdata.Bar) error { return boom }) }() + + require.NoError(t, writeMessage(a, barsMessage{Bars: oneBar})) + assert.ErrorIs(t, <-done, boom) + }) + t.Run("a garbled message is an error", func(t *testing.T) { + a, b := net.Pipe() + done := make(chan error, 1) + go func() { done <- Serve(b, func([]marketdata.Bar) error { return nil }) }() + + _, err := a.Write([]byte{0, 0, 0, 4, 'n', 'o', 'p', 'e'}) + require.NoError(t, err) + assert.Error(t, <-done) + }) +} + +func TestListenAndServe(t *testing.T) { + t.Run("serves one connection on the bound address", func(t *testing.T) { + addrs := make(chan net.Addr, 1) + groups := 0 + done := make(chan error, 1) + go func() { + done <- ListenAndServe("127.0.0.1:0", func(a net.Addr) { addrs <- a }, + func([]marketdata.Bar) error { groups++; return nil }) + }() + + c, err := Dial((<-addrs).String()) + require.NoError(t, err) + require.NoError(t, c.OnBars(context.Background(), oneBar)) + require.NoError(t, c.OnBars(context.Background(), oneBar)) + require.NoError(t, c.Close()) + require.NoError(t, <-done) + assert.Equal(t, 2, groups) + }) + t.Run("a nil ready is allowed", func(t *testing.T) { + // Without ready the port is unknown, so pick a free one and give it back. + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := probe.Addr().String() + require.NoError(t, probe.Close()) + + done := make(chan error, 1) + go func() { done <- ListenAndServe(addr, nil, func([]marketdata.Bar) error { return nil }) }() + + var c *Client + require.Eventually(t, func() bool { c, err = Dial(addr); return err == nil }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, c.OnBars(context.Background(), oneBar)) + require.NoError(t, c.Close()) + require.NoError(t, <-done) + }) + t.Run("a non-loopback address is refused", func(t *testing.T) { + for _, addr := range []string{"0.0.0.0:0", ":0", "8.8.8.8:9100", "no-port"} { + err := ListenAndServe(addr, nil, func([]marketdata.Bar) error { return nil }) + assert.Error(t, err, addr) + } + }) + t.Run("an address in use is an error", func(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + assert.Error(t, ListenAndServe(ln.Addr().String(), nil, func([]marketdata.Bar) error { return nil })) + }) +} diff --git a/strategy/strategy.go b/strategy/strategy.go new file mode 100644 index 0000000..f4bb13c --- /dev/null +++ b/strategy/strategy.go @@ -0,0 +1,49 @@ +// Package strategy carries bars to a strategy running in another process on this +// machine and brings its answer back. It is the transport only: it knows nothing +// of orders, accounts or the engine, and it must not import the trader package. +// +// The protocol is two JSON messages, each preceded by its length as four +// big-endian bytes. For every group of bars the trader sends +// +// {"bars":[...]} +// +// (marketdata.Bar as encoding/json writes it) and the worker answers +// +// {"orders":[]} +// +// Orders are not carried yet: the answer must be empty. +// +// The trader side is Dial and Client; the worker side is ListenAndServe, or Serve +// for a connection already in hand. Both sides are loopback-only: nothing here is +// authenticated or encrypted. +package strategy + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/rustyeddy/trader/marketdata" +) + +type barsMessage struct { + Bars []marketdata.Bar `json:"bars"` +} + +type ordersMessage struct { + Orders []json.RawMessage `json:"orders"` +} + +// requireLoopback refuses an address that is not on this machine's loopback +// interface. A host name other than localhost is refused too, since it may +// resolve anywhere. +func requireLoopback(addr string) error { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("address %q: %w", addr, err) + } + if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return fmt.Errorf("address %q is not loopback", addr) + } + return nil +} diff --git a/strategy/strategy_test.go b/strategy/strategy_test.go new file mode 100644 index 0000000..af43c4c --- /dev/null +++ b/strategy/strategy_test.go @@ -0,0 +1,60 @@ +package strategy + +import ( + "context" + "go/parser" + "go/token" + "net" + "path/filepath" + "strconv" + "testing" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The transport must not import the engine (package trader, the module root), or +// the engine could not use it. +func TestPackage_DoesNotImportTheEngine(t *testing.T) { + files, err := filepath.Glob("*.go") + require.NoError(t, err) + require.NotEmpty(t, files) + for _, name := range files { + f, err := parser.ParseFile(token.NewFileSet(), name, nil, parser.ImportsOnly) + require.NoError(t, err) + for _, imp := range f.Imports { + path, err := strconv.Unquote(imp.Path.Value) + require.NoError(t, err) + assert.NotEqual(t, "github.com/rustyeddy/trader", path, name) + } + } +} + +// A real Client against Serve over loopback TCP. +func TestClientAndServe(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + + groups := 0 + done := make(chan error, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + done <- err + return + } + defer func() { _ = conn.Close() }() + done <- Serve(conn, func([]marketdata.Bar) error { groups++; return nil }) + }() + + r, err := Dial(ln.Addr().String()) + require.NoError(t, err) + for i := 0; i < 5; i++ { + require.NoError(t, r.OnBars(context.Background(), oneBar)) + } + require.NoError(t, r.Close()) + require.NoError(t, <-done) + assert.Equal(t, 5, groups) +} diff --git a/strategy_remote.go b/strategy_remote.go new file mode 100644 index 0000000..120115a --- /dev/null +++ b/strategy_remote.go @@ -0,0 +1,42 @@ +package trader + +import ( + "context" + "fmt" + + "github.com/rustyeddy/trader/marketdata" + remotestrat "github.com/rustyeddy/trader/strategy" +) + +// remoteStrategy is a strategy that runs in another process on this machine. It +// adapts the transport in package strategy to the engine's strategy interface: +// each onBars sends the frame to the worker and waits for its answer, so the +// engine runs exactly as it does for an in-process strategy. +type remoteStrategy struct { + id string + client *remotestrat.Client +} + +var _ strategy = (*remoteStrategy)(nil) + +// dialRemote connects to the worker listening at addr (loopback only). +func dialRemote(id, addr string) (*remoteStrategy, error) { + c, err := remotestrat.Dial(addr) + if err != nil { + return nil, fmt.Errorf("remote strategy %s: %w", id, err) + } + return &remoteStrategy{id: id, client: c}, nil +} + +func (r *remoteStrategy) name() string { return r.id } + +func (r *remoteStrategy) init() error { return nil } + +// close ends the session; the worker sees the connection close and exits. +func (r *remoteStrategy) close() error { return r.client.Close() } + +// onBars hands the frame to the worker, which answers with no orders. Any failure +// is returned and stops the run. +func (r *remoteStrategy) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { + return nil, r.client.OnBars(ctx, bars) +} diff --git a/strategy_remote_test.go b/strategy_remote_test.go new file mode 100644 index 0000000..4e8bacc --- /dev/null +++ b/strategy_remote_test.go @@ -0,0 +1,56 @@ +package trader + +import ( + "context" + "net" + "testing" + + "github.com/rustyeddy/trader/marketdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDialRemote_Errors(t *testing.T) { + _, err := dialRemote("x", "8.8.8.8:9100") + assert.ErrorContains(t, err, "remote strategy x") + + ln, lerr := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, lerr) + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + _, err = dialRemote("x", addr) + assert.ErrorContains(t, err, "remote strategy x") +} + +func TestRemoteStrategy_OnBars(t *testing.T) { + ctx := context.Background() + bars := []marketdata.Bar{{Instrument: "EURUSD", Volume: 1}} + + t.Run("an empty answer is no orders and no error", func(t *testing.T) { + w := startWorker(t, workerOpts{}) + r, err := dialRemote("remote", w.addr) + require.NoError(t, err) + + assert.Equal(t, "remote", r.name()) + require.NoError(t, r.init()) + for range 2 { + orders, err := r.onBars(ctx, bars) + require.NoError(t, err) + assert.Empty(t, orders) + } + require.NoError(t, r.close()) + assert.Len(t, w.finish(), 2, "the worker saw both groups") + }) + t.Run("a dead worker is an error", func(t *testing.T) { + w := startWorker(t, workerOpts{dropOn: 2}) + r, err := dialRemote("remote", w.addr) + require.NoError(t, err) + defer func() { _ = r.close() }() + + _, err = r.onBars(ctx, bars) + require.NoError(t, err, "the first group was answered") + orders, err := r.onBars(ctx, bars) + assert.Error(t, err) + assert.Empty(t, orders) + }) +} diff --git a/trader.go b/trader.go index d55e175..a0700bb 100644 --- a/trader.go +++ b/trader.go @@ -110,7 +110,11 @@ func (tr *trader) onFrame(ctx context.Context, f frame) error { return fmt.Errorf("broker on frame at %s: %w", f.availableAt.Format(time.RFC3339), err) } - for _, o := range tr.strategy.onBars(ctx, f.bars) { + orders, err := tr.strategy.onBars(ctx, f.bars) + if err != nil { + return fmt.Errorf("strategy %s: %w", tr.strategy.name(), err) + } + for _, o := range orders { if o == nil { continue } diff --git a/trader_test.go b/trader_test.go index 85bcecc..7e7fc45 100644 --- a/trader_test.go +++ b/trader_test.go @@ -1,9 +1,21 @@ package trader import ( + "bufio" + "bytes" "context" + "encoding/json" + "errors" + "net" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync" "testing" + "github.com/rustyeddy/trader/marketdata" + remotestrat "github.com/rustyeddy/trader/strategy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -215,3 +227,170 @@ func TestTrader_HandleOrder_Amend(t *testing.T) { assert.Equal(t, "a", o.accountID) }) } + +// workerOpts shapes testWorker. The zero value answers every group with no orders +// until the trader hangs up. +type workerOpts struct { + dropOn int // hang up on receiving this group without answering it; 0 never +} + +// testWorker is the example worker in process: it listens on loopback, serves one +// connection with strategy.Serve and keeps every group of bars it was sent. +type testWorker struct { + addr string + groups [][]marketdata.Bar + done sync.WaitGroup +} + +func startWorker(t *testing.T, o workerOpts) *testWorker { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + w := &testWorker{addr: ln.Addr().String()} + w.done.Add(1) + t.Cleanup(func() { _ = ln.Close() }) + + go func() { + defer w.done.Done() + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _ = remotestrat.Serve(conn, func(bars []marketdata.Bar) error { + w.groups = append(w.groups, bars) + if o.dropOn > 0 && len(w.groups) == o.dropOn { + return errors.New("hang up") + } + return nil + }) + }() + return w +} + +// finish waits for the worker to stop (the trader closed or it hung up) and returns +// what it was sent. +func (w *testWorker) finish() [][]marketdata.Bar { + w.done.Wait() + return w.groups +} + +// jsonOf is a value as the wire carries it, to compare bars without caring how +// time.Time or num values are held in memory. +func jsonOf(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return string(b) +} + +// recorder is an in-process strategy that records the groups it is shown and +// trades nothing: the reference the remote path is compared with. +type recorder struct{ groups [][]marketdata.Bar } + +func (r *recorder) name() string { return "recorder" } +func (r *recorder) init() error { return nil } +func (r *recorder) onBars(_ context.Context, bars []marketdata.Bar) ([]*order, error) { + r.groups = append(r.groups, slices.Clone(bars)) + return nil, nil +} + +func newRemoteTrader(t *testing.T, s strategy) *trader { + t.Helper() + tr := newTestTrader() + tr.strategy = s + return tr +} + +// failingStrategy returns an order together with an error: the order must not be +// used. +type failingStrategy struct{} + +func (failingStrategy) name() string { return "failing" } +func (failingStrategy) init() error { return nil } +func (failingStrategy) onBars(context.Context, []marketdata.Bar) ([]*order, error) { + return []*order{{instrument: "EURUSD", action: open, direction: long, orderType: marketOrder}}, errors.New("worker is gone") +} + +func TestTrader_OnFrame_StopsWhenTheStrategyHasFailed(t *testing.T) { + captureLogs(t) + tr := newRemoteTrader(t, failingStrategy{}) + f := frame{availableAt: testStart, bars: []marketdata.Bar{{Instrument: "EURUSD", Time: testStart, AvailableAt: testStart}}} + + assert.ErrorContains(t, tr.onFrame(context.Background(), f), "worker is gone") + assert.Empty(t, tr.broker.orders, "the order it returned was not submitted") + assert.Empty(t, tr.rejected) +} + +const remoteScenario = "a-reversal-exit" + +func TestRemoteStrategy_BacktestMatchesInProcess(t *testing.T) { + ctx := context.Background() + captureLogs(t) + prices := scenarioPrices(t, remoteScenario) + + ref := &recorder{} + require.NoError(t, (&backtest{}).runSeries(ctx, newRemoteTrader(t, ref), prices)) + require.Greater(t, len(ref.groups), 3, "the fixture gives several deliveries") + + w := startWorker(t, workerOpts{}) + remote, err := dialRemote("remote", w.addr) + require.NoError(t, err) + tr := newRemoteTrader(t, remote) + require.NoError(t, (&backtest{}).runSeries(ctx, tr, prices)) + require.NoError(t, remote.close()) + + assert.Equal(t, jsonOf(t, ref.groups), jsonOf(t, w.finish()), "the worker saw the same groups, in the same order") + assert.Empty(t, tr.broker.completed, "no orders, so nothing happened") + assert.Empty(t, tr.account.trades, "no trades") + assert.True(t, tr.account.balance.Equal(usd(1000)), "balance untouched") +} + +func TestRemoteStrategy_BacktestFailsWhenTheWorkerDisconnects(t *testing.T) { + ctx := context.Background() + captureLogs(t) + prices := scenarioPrices(t, remoteScenario) + + w := startWorker(t, workerOpts{dropOn: 3}) + remote, err := dialRemote("remote", w.addr) + require.NoError(t, err) + defer func() { _ = remote.close() }() + + err = (&backtest{}).runSeries(ctx, newRemoteTrader(t, remote), prices) + require.Error(t, err, "a dead worker ends the run instead of hanging it") + assert.ErrorContains(t, err, "strategy remote") + assert.Len(t, w.finish(), 3, "the worker hung up on its third group, which the run never got an answer to") +} + +// TestRemoteStrategy_ExampleWorker runs the real example as a separate process. +func TestRemoteStrategy_ExampleWorker(t *testing.T) { + ctx := context.Background() + captureLogs(t) + prices := scenarioPrices(t, remoteScenario) + + bin := filepath.Join(t.TempDir(), "remote-strategy") + out, err := exec.Command("go", "build", "-o", bin, "./examples/remote-strategy").CombinedOutput() + require.NoError(t, err, string(out)) + + var stderr bytes.Buffer + cmd := exec.Command(bin, "-addr", "127.0.0.1:0") + cmd.Stderr = &stderr + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill() }) + + line, err := bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + addr := strings.TrimSpace(strings.TrimPrefix(line, "listening on ")) + + remote, err := dialRemote("example", addr) + require.NoError(t, err) + tr := newRemoteTrader(t, remote) + require.NoError(t, (&backtest{}).runSeries(ctx, tr, prices)) + require.NoError(t, remote.close()) + require.NoError(t, cmd.Wait(), "the worker exits when the trader closes") + + assert.Empty(t, tr.account.trades) + assert.Contains(t, stderr.String(), "done: ") +} diff --git a/twosymbol_test.go b/twosymbol_test.go index 292db55..67b070c 100644 --- a/twosymbol_test.go +++ b/twosymbol_test.go @@ -57,7 +57,7 @@ func (s *reactStrategy) onBar(_ context.Context, b marketdata.Bar) *order { return o } -func (s *reactStrategy) onBars(ctx context.Context, bars []marketdata.Bar) []*order { +func (s *reactStrategy) onBars(ctx context.Context, bars []marketdata.Bar) ([]*order, error) { return eachBar(ctx, bars, s.onBar) } @@ -543,7 +543,7 @@ type groupSpy struct { func (s *groupSpy) name() string { return "groupspy" } func (s *groupSpy) init() error { return nil } -func (s *groupSpy) onBars(_ context.Context, bars []marketdata.Bar) []*order { +func (s *groupSpy) onBars(_ context.Context, bars []marketdata.Bar) ([]*order, error) { var labels []string var orders []*order for _, b := range bars { @@ -558,7 +558,7 @@ func (s *groupSpy) onBars(_ context.Context, bars []marketdata.Bar) []*order { } s.groups = append(s.groups, labels) s.atCall = append(s.atCall, s.held()) - return orders + return orders, nil } // The strategy is called exactly once per distinct availability instant, with the