From bd28a1e58f1854b673abc11ec8483da277891dd4 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:57:25 +0200 Subject: [PATCH 1/2] docker-container: give readiness wait a full startup timeout A recently started builder should use its StartedAt timestamp to decide whether BuildKit may still be coming up, but not as the absolute deadline for the client readiness wait. When the first client request arrived near the end of that startup window, the wait could inherit only a few milliseconds and fail with DeadlineExceeded even though the builder was still legitimately starting. Give those recent containers a fresh bounded wait from the current time while continuing to skip the wait for established builders. Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/docker-container/driver.go | 5 ++--- driver/docker-container/driver_test.go | 14 +++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go index b9f95c8a9103..1276f76f4fda 100644 --- a/driver/docker-container/driver.go +++ b/driver/docker-container/driver.go @@ -573,11 +573,10 @@ func clientWaitDeadline(state *container.State, now time.Time) (time.Time, error if err != nil { return time.Time{}, nil } - deadline := startedAt.Add(buildkitdStartupTimeout) - if !now.Before(deadline) { + if !now.Before(startedAt.Add(buildkitdStartupTimeout)) { return time.Time{}, nil } - return deadline, nil + return now.Add(buildkitdStartupTimeout), nil } func (d *Driver) Factory() driver.Factory { diff --git a/driver/docker-container/driver_test.go b/driver/docker-container/driver_test.go index 9a47579c426a..77ab5e16076a 100644 --- a/driver/docker-container/driver_test.go +++ b/driver/docker-container/driver_test.go @@ -28,13 +28,21 @@ func TestClientWaitDeadline(t *testing.T) { }) t.Run("recent-start-builder-waits", func(t *testing.T) { - startedAt := now.Add(-time.Second) deadline, err := clientWaitDeadline(&container.State{ Running: true, - StartedAt: startedAt.Format(time.RFC3339Nano), + StartedAt: now.Add(-time.Second).Format(time.RFC3339Nano), }, now) require.NoError(t, err) - require.True(t, deadline.Equal(startedAt.Add(buildkitdStartupTimeout))) + require.True(t, deadline.Equal(now.Add(buildkitdStartupTimeout))) + }) + + t.Run("nearly-expired-startup-window-gets-full-wait", func(t *testing.T) { + deadline, err := clientWaitDeadline(&container.State{ + Running: true, + StartedAt: now.Add(-buildkitdStartupTimeout + time.Nanosecond).Format(time.RFC3339Nano), + }, now) + require.NoError(t, err) + require.True(t, deadline.Equal(now.Add(buildkitdStartupTimeout))) }) t.Run("invalid-start-time-skips-wait", func(t *testing.T) { From 310bba93927005191204871498802a10252a4e9a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 10 Sep 2026 09:48:30 -0700 Subject: [PATCH 2/2] docker-container: separate readiness timeout Use the container start time only to decide whether readiness waiting is needed. Start a full timeout immediately before waiting so Docker dial time does not consume the readiness budget. Name the startup window and readiness timeout independently, and cover the exact startup-window boundary. Signed-off-by: Tonis Tiigi --- driver/docker-container/driver.go | 28 +++++----- driver/docker-container/driver_test.go | 73 +++++++++++--------------- 2 files changed, 43 insertions(+), 58 deletions(-) diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go index 1276f76f4fda..5800311f40d5 100644 --- a/driver/docker-container/driver.go +++ b/driver/docker-container/driver.go @@ -35,9 +35,10 @@ import ( ) const ( - volumeStateSuffix = "_state" - buildkitdConfigFile = "buildkitd.toml" - buildkitdStartupTimeout = 20 * time.Second + volumeStateSuffix = "_state" + buildkitdConfigFile = "buildkitd.toml" + buildkitdStartupWindow = 20 * time.Second + buildkitdReadyTimeout = 20 * time.Second ) type Driver struct { @@ -525,7 +526,7 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. } return nil, errors.WithStack(err) } - waitDeadline, err := clientWaitDeadline(res.Container.State, time.Now()) + waitReady, err := clientWaitReady(res.Container.State, time.Now()) if err != nil { return nil, err } @@ -549,11 +550,11 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. _ = conn.Close() return nil, err } - if waitDeadline.IsZero() { + if !waitReady { return c, nil } - waitCtx, cancel := context.WithDeadlineCause(ctx, waitDeadline, errors.WithStack(context.DeadlineExceeded)) + waitCtx, cancel := context.WithTimeoutCause(ctx, buildkitdReadyTimeout, errors.WithStack(context.DeadlineExceeded)) defer cancel() if err := c.Wait(waitCtx); err != nil { _ = c.Close() @@ -562,21 +563,18 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. return c, nil } -func clientWaitDeadline(state *container.State, now time.Time) (time.Time, error) { +func clientWaitReady(state *container.State, now time.Time) (bool, error) { if state == nil || !state.Running { - return time.Time{}, driver.ErrNotRunning{} + return false, driver.ErrNotRunning{} } // Docker reports a container as running before buildkitd has bound its - // socket. Wait only during that startup window so an established but broken - // builder still returns its connection error promptly. + // socket. Use the startup window only to decide whether to wait so an + // established but broken builder still returns its connection error promptly. startedAt, err := time.Parse(time.RFC3339Nano, state.StartedAt) if err != nil { - return time.Time{}, nil + return false, nil } - if !now.Before(startedAt.Add(buildkitdStartupTimeout)) { - return time.Time{}, nil - } - return now.Add(buildkitdStartupTimeout), nil + return now.Before(startedAt.Add(buildkitdStartupWindow)), nil } func (d *Driver) Factory() driver.Factory { diff --git a/driver/docker-container/driver_test.go b/driver/docker-container/driver_test.go index 77ab5e16076a..61d60ac6e9b0 100644 --- a/driver/docker-container/driver_test.go +++ b/driver/docker-container/driver_test.go @@ -9,48 +9,35 @@ import ( "github.com/stretchr/testify/require" ) -func TestClientWaitDeadline(t *testing.T) { +func TestClientWaitReady(t *testing.T) { now := time.Now() - - t.Run("stopped-builder-fails-fast", func(t *testing.T) { - deadline, err := clientWaitDeadline(&container.State{}, now) - require.ErrorIs(t, err, driver.ErrNotRunning{}) - require.True(t, deadline.IsZero()) - }) - - t.Run("established-builder-skips-wait", func(t *testing.T) { - deadline, err := clientWaitDeadline(&container.State{ - Running: true, - StartedAt: now.Add(-2 * buildkitdStartupTimeout).Format(time.RFC3339Nano), - }, now) - require.NoError(t, err) - require.True(t, deadline.IsZero()) - }) - - t.Run("recent-start-builder-waits", func(t *testing.T) { - deadline, err := clientWaitDeadline(&container.State{ - Running: true, - StartedAt: now.Add(-time.Second).Format(time.RFC3339Nano), - }, now) - require.NoError(t, err) - require.True(t, deadline.Equal(now.Add(buildkitdStartupTimeout))) - }) - - t.Run("nearly-expired-startup-window-gets-full-wait", func(t *testing.T) { - deadline, err := clientWaitDeadline(&container.State{ - Running: true, - StartedAt: now.Add(-buildkitdStartupTimeout + time.Nanosecond).Format(time.RFC3339Nano), - }, now) - require.NoError(t, err) - require.True(t, deadline.Equal(now.Add(buildkitdStartupTimeout))) - }) - - t.Run("invalid-start-time-skips-wait", func(t *testing.T) { - deadline, err := clientWaitDeadline(&container.State{ - Running: true, - StartedAt: "invalid", - }, now) - require.NoError(t, err) - require.True(t, deadline.IsZero()) - }) + running := func(startedAt string) *container.State { + return &container.State{Running: true, StartedAt: startedAt} + } + tests := []struct { + name string + state *container.State + wantWait bool + wantErr error + }{ + {name: "missing-state-fails-fast", wantErr: driver.ErrNotRunning{}}, + {name: "stopped-builder-fails-fast", state: &container.State{}, wantErr: driver.ErrNotRunning{}}, + {name: "established-builder-skips-wait", state: running(now.Add(-2 * buildkitdStartupWindow).Format(time.RFC3339Nano))}, + {name: "recent-start-builder-waits", state: running(now.Add(-time.Second).Format(time.RFC3339Nano)), wantWait: true}, + {name: "nearly-expired-startup-window-waits", state: running(now.Add(-buildkitdStartupWindow + time.Nanosecond).Format(time.RFC3339Nano)), wantWait: true}, + {name: "expired-startup-window-skips-wait", state: running(now.Add(-buildkitdStartupWindow).Format(time.RFC3339Nano))}, + {name: "invalid-start-time-skips-wait", state: running("invalid")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wait, err := clientWaitReady(tt.state, now) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.wantWait, wait) + }) + } }