From d9b6176ec5a58592a29b5305e7e6f7f90b553369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 6 Nov 2026 22:25:25 +0100 Subject: [PATCH 01/28] feat(objectstore): persist neutral object bytes atomically Store immutable SHA-256 blobs behind logical bucket/key metadata. Verify bytes on reads and reopen; retain blobs on deletion. Exclude concurrent root owners and unsafe parents. No S3 routes, snapshot changes or garbage collection. Refs: #850 Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/architecture.md | 39 + internal/core/objectstore/files.go | 331 +++++++ internal/core/objectstore/manifest.go | 122 +++ internal/core/objectstore/store.go | 399 ++++++++ internal/core/objectstore/store_test.go | 858 ++++++++++++++++++ tools/falsify/specs/neutral-object-bytes.json | 117 +++ tools/testplan/rules.go | 6 + 7 files changed, 1872 insertions(+) create mode 100644 internal/core/objectstore/files.go create mode 100644 internal/core/objectstore/manifest.go create mode 100644 internal/core/objectstore/store.go create mode 100644 internal/core/objectstore/store_test.go create mode 100644 tools/falsify/specs/neutral-object-bytes.json diff --git a/docs/architecture.md b/docs/architecture.md index 83c6d392..0b5b00c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,6 +13,7 @@ internal/cli/ the commands: lifecycle, serve, env, doctor, snapsho coverage, probe, docs, catalog, clean, version internal/core/resource/ the neutral resource: ID, Kind, Tenant, State, Attrs, Runtime internal/core/store/ in-memory storage plus JSON snapshot, with no provider knowledge +internal/core/objectstore/ neutral persisted object bytes and metadata; no S3 route yet internal/core/emulator/ Env, Pack, Route, the single-port mount, /_feint/* internal/core/machine/ the machine runtime behind --vm: Incus containers, VMs, OVN internal/core/network/ address plans, subnets, firewall bindings @@ -73,6 +74,44 @@ the pack is what only that provider knows — its image catalogue, the login its machines use, the field its API publishes an address under, or the fact that it publishes none. +## Neutral object bytes, not yet an S3 service + +[objectstore](../internal/core/objectstore/store.go) is an independent library +for #850. It is not wired into the emulator or CLI, and does not change the +Object Storage support limit. Its caller explicitly selects a private root +whose parent already exists. New roots are mode 0700; files are mode 0600. +The pinned parent must belong to this user or root; group/other write permission +requires its sticky bit so another UID cannot replace the newly created root +before it is opened. A sticky directory owned by a foreign UID is not trusted. +Existing unmarked roots, symlinks, non-regular files, foreign owners and public +permissions are refused, not repaired or adopted. + +The store holds a directory lock across its lifetime on Linux/macOS local +filesystems. Logical bucket names and keys are JSON data, never paths. A +streaming PUT declares its exact decoded size, writes and syncs a temporary +blob, links it under its SHA-256 without overwriting an existing blob, and +syncs that directory before atomically publishing and syncing metadata. +Uploads to distinct keys run independently. A generation check prevents an +unfinished upload from repopulating a deleted or recreated bucket. + +GET and HEAD verify the complete blob; listing is a metadata inventory only. +The digest is not an S3 ETag. Deletion removes references, not blobs. The +private `feint-objectstore` v1 index is not `feint-snapshot` v1: no snapshot, +checkpoint/reset or cleanup command reads it yet. Unreferenced blobs and +temporary files from a process crash are retained; there is no collector or +recursive root deletion. +An interrupted initial creation may leave an unmarked directory; inspect and +remove that explicitly rather than asking Open to adopt it. + +A successful mutation follows file and directory syncs. If metadata was renamed +but its directory sync fails, the call reports failure and the instance requires +close/reopen; failure may therefore have published metadata. Reopen, corruption +and concurrency are tested, not actual power loss or network filesystem locks. +The root is not a sandbox against a malicious process sharing its owner's UID. +The caller must impose storage quotas and unblock its input reader on +cancellation; waiting for an existing same-key transaction may also delay +cancellation. No structured format passes through a text template. + ## What varies is a field, not a convention Three packs do the same things with different words, and confusing the two levels diff --git a/internal/core/objectstore/files.go b/internal/core/objectstore/files.go new file mode 100644 index 00000000..1dc4148e --- /dev/null +++ b/internal/core/objectstore/files.go @@ -0,0 +1,331 @@ +package objectstore + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "syscall" +) + +// Open creates a new private store at path, or reopens a marked, valid store. +// The parent must exist and prevent replacement by other UIDs. Existing +// unmarked directories are never adopted. Existing roots, metadata and blobs +// must be private and owned by this user. +// A root symlink is refused. A directory lock excludes other Store instances +// and processes on the supported Linux and macOS local filesystems. +func Open(ctx context.Context, path string) (result *Store, err error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if path == "" { + return nil, fmt.Errorf("%w: storage root must be explicit", ErrInvalid) + } + path, err = filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve storage root: %w", err) + } + parent, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return nil, fmt.Errorf("open storage parent: %w", err) + } + var s *Store + defer func() { + err = errors.Join(err, parent.Close()) + if err != nil && s != nil { + if s.blobs != nil { + err = errors.Join(err, s.blobs.Close()) + } + err = errors.Join(err, s.root.Close()) + if s.dir != nil { + err = errors.Join(err, s.dir.Close()) + } + result = nil + } + }() + parentInfo, err := parent.Stat(".") + if err != nil { + return nil, fmt.Errorf("inspect storage parent: %w", err) + } + if err := checkParent(parentInfo); err != nil { + return nil, err + } + name := filepath.Base(path) + created := false + if err := parent.Mkdir(name, 0o700); err == nil { + created = true + } else if !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("create storage root: %w", err) + } + info, err := parent.Lstat(name) + if err != nil { + return nil, fmt.Errorf("inspect storage root: %w", err) + } + if err := private(info, true); err != nil { + return nil, fmt.Errorf("storage root: %w", errors.Join(ErrUnowned, err)) + } + root, err := parent.OpenRoot(name) + if err != nil { + return nil, fmt.Errorf("open storage root: %w", err) + } + s = &Store{root: root, domain: rand.Text()} + s.dir, err = root.OpenFile(".", os.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0) + if err != nil { + return nil, fmt.Errorf("open storage lock directory: %w", err) + } + opened, err := s.dir.Stat() + if err != nil { + return nil, fmt.Errorf("inspect opened storage root: %w", err) + } + if !os.SameFile(info, opened) { + return nil, fmt.Errorf("%w: storage root changed while opening", ErrUnowned) + } + if err := private(opened, true); err != nil { + return nil, fmt.Errorf("opened storage root: %w", errors.Join(ErrUnowned, err)) + } + if err := syscall.Flock(int(s.dir.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, ErrLocked + } + return nil, fmt.Errorf("lock storage root: %w", err) + } + if created { + if err := root.Mkdir("blobs", 0o700); err != nil { + return nil, fmt.Errorf("create blob directory: %w", err) + } + s.state = manifest{Format: storeFormat, Version: 1, Buckets: make(map[string]*bucket)} + if err := s.commit(ctx, s.state); err != nil { + return nil, err + } + parentDir, err := parent.Open(".") + if err != nil { + return nil, fmt.Errorf("open storage parent for sync: %w", err) + } + if err := errors.Join(parentDir.Sync(), parentDir.Close()); err != nil { + return nil, fmt.Errorf("sync new storage root: %w", err) + } + } else { + s.state, err = readManifest(ctx, root) + if err != nil { + return nil, err + } + } + info, err = root.Lstat("blobs") + if err != nil { + return nil, fmt.Errorf("inspect blob directory: %w", errors.Join(ErrCorrupt, err)) + } + if err := private(info, true); err != nil { + return nil, fmt.Errorf("blob directory: %w", errors.Join(ErrCorrupt, err)) + } + s.blobs, err = root.OpenRoot("blobs") + if err != nil { + return nil, fmt.Errorf("open blob directory: %w", err) + } + opened, err = s.blobs.Stat(".") + if err != nil { + return nil, fmt.Errorf("inspect opened blob directory: %w", err) + } + if !os.SameFile(info, opened) { + return nil, fmt.Errorf("%w: blob directory changed while opening", ErrCorrupt) + } + if err := private(opened, true); err != nil { + return nil, fmt.Errorf("opened blob directory: %w", errors.Join(ErrCorrupt, err)) + } + for _, b := range s.state.Buckets { + for _, obj := range b.Objects { + file, err := s.openBlob(ctx, obj) + if err != nil { + return nil, err + } + if err := file.Close(); err != nil { + return nil, fmt.Errorf("close startup blob verification: %w", err) + } + } + } + return s, nil +} + +// TestUnsafeParentIsRefusedBeforeCreation holds the Mkdir-to-open substitution +// boundary: a trusted parent must prevent another UID from replacing entries. +func checkParent(info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || (stat.Uid != 0 && int64(stat.Uid) != int64(os.Geteuid())) { + return fmt.Errorf("%w: storage parent must belong to this user or root", ErrUnowned) + } + if info.Mode().Perm()&0o022 != 0 && info.Mode()&os.ModeSticky == 0 { + return fmt.Errorf("%w: writable storage parent requires sticky protection", ErrUnowned) + } + return nil +} + +func private(info os.FileInfo, directory bool) error { + if (directory && !info.IsDir()) || (!directory && !info.Mode().IsRegular()) { + return errors.New("expected a real directory or regular file, not a symlink or special file") + } + if info.Mode().Perm()&0o077 != 0 { + return errors.New("group or other permissions are not private") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int64(stat.Uid) != int64(os.Geteuid()) { + return errors.New("filesystem entry is not owned by the current user") + } + return nil +} + +func openPrivate(root *os.Root, name string) (*os.File, error) { + // Root resolves internal symlinks before opening: O_NOFOLLOW alone does + // not hold TestRootOwnershipPermissionsAndSymlinksAreRefused. + info, err := root.Lstat(name) + if err != nil { + return nil, err + } + if err := private(info, false); err != nil { + return nil, err + } + file, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + opened, err := file.Stat() + if err == nil { + err = private(opened, false) + } + if err == nil && !os.SameFile(info, opened) { + err = errors.New("filesystem entry changed while opening") + } + if err != nil { + return nil, errors.Join(err, file.Close()) + } + return file, nil +} + +func tempFile(root *os.Root) (*os.File, string, error) { + name := ".pending-" + rand.Text() + file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return nil, "", fmt.Errorf("create private temporary file: %w", err) + } + return file, name, nil +} + +func (s *Store) writeBlob(ctx context.Context, body io.Reader, size int64) (_ string, err error) { + file, temp, err := tempFile(s.blobs) + if err != nil { + return "", err + } + defer func() { err = errors.Join(err, s.blobs.Remove(temp)) }() + hash := sha256.New() + n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(&contextReader{ctx: ctx, reader: body}, size+1)) + if copyErr == nil && n != size { + copyErr = fmt.Errorf("%w: expected %d bytes, received %d", ErrInvalid, size, n) + } + if copyErr == nil { + copyErr = ctx.Err() + } + if copyErr == nil { + copyErr = file.Sync() + } + if err := errors.Join(copyErr, file.Close()); err != nil { + return "", fmt.Errorf("write blob: %w", err) + } + digest := hex.EncodeToString(hash.Sum(nil)) + if err := s.blobs.Link(temp, digest); err != nil { + if !errors.Is(err, os.ErrExist) { + return "", fmt.Errorf("publish blob: %w", err) + } + existing, err := s.openBlob(ctx, Object{Digest: digest, Size: size}) + if err != nil { + return "", err + } + if err := existing.Close(); err != nil { + return "", fmt.Errorf("close existing blob: %w", err) + } + } + dir, err := s.blobs.Open(".") + if err != nil { + return "", fmt.Errorf("open blob directory for sync: %w", err) + } + if err := errors.Join(dir.Sync(), dir.Close()); err != nil { + return "", fmt.Errorf("sync published blob: %w", err) + } + return digest, nil +} + +func (s *Store) openBlob(ctx context.Context, obj Object) (_ *os.File, err error) { + if !validDigest(obj.Digest) || obj.Size < 0 || obj.Size == math.MaxInt64 { + return nil, fmt.Errorf("%w: invalid blob reference", ErrCorrupt) + } + file, err := openPrivate(s.blobs, obj.Digest) + if err != nil { + return nil, fmt.Errorf("open blob %s: %w", obj.Digest, errors.Join(ErrCorrupt, err)) + } + defer func() { + if err != nil { + err = errors.Join(err, file.Close()) + } + }() + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat blob: %w", err) + } + if info.Size() != obj.Size { + return nil, fmt.Errorf("%w: blob %s has size %d, expected %d", ErrCorrupt, obj.Digest, info.Size(), obj.Size) + } + hash := sha256.New() + n, err := io.Copy(hash, io.LimitReader(&contextReader{ctx: ctx, reader: file}, obj.Size+1)) + if err != nil { + return nil, fmt.Errorf("verify blob: %w", err) + } + if n != obj.Size || hex.EncodeToString(hash.Sum(nil)) != obj.Digest { + return nil, fmt.Errorf("%w: blob %s digest does not match bytes", ErrCorrupt, obj.Digest) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("rewind verified blob: %w", err) + } + return file, nil +} + +func validDigest(digest string) bool { + if len(digest) != sha256.Size*2 { + return false + } + for _, c := range digest { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r *contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +// TestReturnedReaderHonorsCancellationThroughCopy forbids embedding File: +// its promoted WriteTo would bypass the context-aware Read. +type blobReader struct { + ctx context.Context + file *os.File +} + +func (r *blobReader) Read(p []byte) (int, error) { + return (&contextReader{ctx: r.ctx, reader: r.file}).Read(p) +} + +func (r *blobReader) Close() error { + return r.file.Close() +} diff --git a/internal/core/objectstore/manifest.go b/internal/core/objectstore/manifest.go new file mode 100644 index 00000000..011b3b3f --- /dev/null +++ b/internal/core/objectstore/manifest.go @@ -0,0 +1,122 @@ +package objectstore + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "unicode/utf8" +) + +const ( + storeFormat = "feint-objectstore" + maxManifestBytes = 64 << 20 +) + +func readManifest(ctx context.Context, root *os.Root) (_ manifest, err error) { + file, err := openPrivate(root, "index.json") + if err != nil { + return manifest{}, fmt.Errorf("read storage ownership metadata: %w", errors.Join(ErrUnowned, err)) + } + defer func() { err = errors.Join(err, file.Close()) }() + body, err := io.ReadAll(io.LimitReader(&contextReader{ctx: ctx, reader: file}, maxManifestBytes+1)) + if err != nil { + return manifest{}, fmt.Errorf("read storage metadata: %w", err) + } + if len(body) > maxManifestBytes { + return manifest{}, fmt.Errorf("%w: metadata exceeds %d bytes", ErrCorrupt, maxManifestBytes) + } + return decodeManifest(body) +} + +func decodeManifest(body []byte) (manifest, error) { + var state manifest + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&state); err != nil { + return manifest{}, fmt.Errorf("decode storage metadata: %w", errors.Join(ErrCorrupt, err)) + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return manifest{}, fmt.Errorf("%w: trailing storage metadata", ErrCorrupt) + } + if state.Format != storeFormat { + return manifest{}, ErrUnowned + } + if state.Version != 1 || state.Buckets == nil { + return manifest{}, fmt.Errorf("%w: unsupported version or missing buckets", ErrCorrupt) + } + for name, b := range state.Buckets { + if err := validName(name); err != nil { + return manifest{}, errors.Join(ErrCorrupt, err) + } + if b == nil || b.Objects == nil || b.Generation == "" || len(b.Generation) > 128 || !utf8.ValidString(b.Generation) { + return manifest{}, fmt.Errorf("%w: invalid bucket metadata", ErrCorrupt) + } + for key, obj := range b.Objects { + if err := validKey(key); err != nil { + return manifest{}, errors.Join(ErrCorrupt, err) + } + if !validDigest(obj.Digest) || obj.Size < 0 || obj.Size == math.MaxInt64 { + return manifest{}, fmt.Errorf("%w: invalid object reference", ErrCorrupt) + } + if err := validMetadata(obj); err != nil { + return manifest{}, errors.Join(ErrCorrupt, err) + } + } + } + return state, nil +} + +// commit runs under mu. Upload I/O is outside it; the only serialized I/O is +// the small metadata transaction. After a rename, a directory-sync failure is +// indeterminate durability, not success: stop serving mutations until reopen. +func (s *Store) commit(ctx context.Context, next manifest) (err error) { + if s.fault != nil { + return s.fault + } + if err := ctx.Err(); err != nil { + return err + } + body, err := json.Marshal(next) + if err != nil { + return fmt.Errorf("encode storage metadata: %w", err) + } + if len(body) > maxManifestBytes { + return fmt.Errorf("%w: metadata exceeds %d bytes", ErrInvalid, maxManifestBytes) + } + file, temp, err := tempFile(s.root) + if err != nil { + return err + } + renamed := false + defer func() { + if !renamed { + err = errors.Join(err, s.root.Remove(temp)) + } + }() + _, writeErr := file.Write(body) + if writeErr == nil { + writeErr = file.Sync() + } + if err := errors.Join(writeErr, file.Close()); err != nil { + return fmt.Errorf("write storage metadata: %w", err) + } + if err := ctx.Err(); err != nil { + return err + } + if err := s.root.Rename(temp, "index.json"); err != nil { + return fmt.Errorf("publish storage metadata: %w", err) + } + renamed = true + s.state = next + if err := s.dir.Sync(); err != nil { + s.fault = fmt.Errorf("metadata published but directory sync failed; close and reopen required: %w", err) + return s.fault + } + return nil +} diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go new file mode 100644 index 00000000..6fb144ef --- /dev/null +++ b/internal/core/objectstore/store.go @@ -0,0 +1,399 @@ +// Package objectstore persists neutral bucket metadata and real object bytes. +// +// Blobs are immutable and addressed by SHA-256, never by a bucket or object key. +// Mutations publish metadata only after the blob is durable. One Store owns a +// private directory exclusively; Close releases it so Open can load it again. +// This package provides no S3 protocol, resource snapshots, checkpoints or GC. +package objectstore + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "maps" + "math" + "os" + "sort" + "sync" + "unicode/utf8" + + "github.com/stephrobert/feint/internal/core/serialise" +) + +// Errors distinguish neutral store outcomes from provider protocol errors. +var ( + ErrBucketExists = errors.New("bucket already exists") + ErrBucketNotFound = errors.New("bucket not found") + ErrBucketNotEmpty = errors.New("bucket is not empty") + ErrObjectNotFound = errors.New("object not found") + ErrInvalid = errors.New("invalid object store input") + ErrCorrupt = errors.New("object store data is corrupt") + ErrUnowned = errors.New("directory is not an owned object store") + ErrLocked = errors.New("object store is already open") + ErrClosed = errors.New("object store is closed") +) + +// Object describes stored bytes. Digest is lowercase SHA-256, not an S3 ETag. +// Metadata is copied on intake and on every return. +type Object struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` + Metadata map[string]string `json:"metadata"` +} + +// PutOptions declares the exact decoded byte count and application metadata. +// Size must be nonnegative; callers must impose their own quota before upload. +type PutOptions struct { + Size int64 + ContentType string + Metadata map[string]string +} + +type bucket struct { + Generation string `json:"generation"` + Objects map[string]Object `json:"objects"` +} + +type manifest struct { + Format string `json:"format"` + Version int `json:"version"` + Buckets map[string]*bucket `json:"buckets"` +} + +// Store is safe for concurrent use. Uploads to different keys run independently; +// writes and deletes to the same key are serialized. Metadata commits are short +// exclusive transactions. Close waits for in-flight operations, not for the +// independent readers returned by GetObject. +type Store struct { + life sync.RWMutex + mu sync.RWMutex + root *os.Root + blobs *os.Root + dir *os.File + state manifest + closed bool + fault error + domain string +} + +// Close releases directory handles and the exclusive process lock. +// Returned object readers remain valid until the caller closes them. +func (s *Store) Close() error { + s.life.Lock() + defer s.life.Unlock() + if s.closed { + return nil + } + s.closed = true + return errors.Join(s.blobs.Close(), s.root.Close(), s.dir.Close()) +} + +func (s *Store) begin(ctx context.Context) (func(), error) { + s.life.RLock() + if err := ctx.Err(); err != nil { + s.life.RUnlock() + return nil, err + } + if s.closed { + s.life.RUnlock() + return nil, ErrClosed + } + s.mu.RLock() + err := s.fault + s.mu.RUnlock() + if err != nil { + s.life.RUnlock() + return nil, err + } + return s.life.RUnlock, nil +} + +// CreateBucket durably creates a logical bucket. Its name never becomes a path. +func (s *Store) CreateBucket(ctx context.Context, name string) error { + if err := validName(name); err != nil { + return err + } + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.state.Buckets[name]; ok { + return ErrBucketExists + } + next := s.clone() + next.Buckets[name] = &bucket{Generation: rand.Text(), Objects: make(map[string]Object)} + return s.commit(ctx, next) +} + +// HeadBucket reports whether a logical bucket exists. +func (s *Store) HeadBucket(ctx context.Context, name string) error { + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + _, err = s.lookup(name) + return err +} + +// ListBuckets returns logical names in lexical order. +func (s *Store) ListBuckets(ctx context.Context) ([]string, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + names := make([]string, 0, len(s.state.Buckets)) + for name := range s.state.Buckets { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// DeleteBucket durably removes an empty bucket. It never removes blob files. +// An unfinished new upload cannot resurrect a deleted or recreated bucket. +func (s *Store) DeleteBucket(ctx context.Context, name string) error { + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(name) + if err != nil { + return err + } + if len(b.Objects) != 0 { + return ErrBucketNotEmpty + } + next := s.clone() + delete(next.Buckets, name) + return s.commit(ctx, next) +} + +// PutObject streams exactly options.Size bytes and publishes their metadata. +// Cancellation is checked between reads and before publication. A caller must +// arrange to unblock a reader that can block inside Read on cancellation. +func (s *Store) PutObject(ctx context.Context, name, key string, body io.Reader, options PutOptions) (Object, error) { + if err := validName(name); err != nil { + return Object{}, err + } + if err := validKey(key); err != nil { + return Object{}, err + } + if body == nil || options.Size < 0 || options.Size == math.MaxInt64 { + return Object{}, fmt.Errorf("%w: a reader and exact representable size are required", ErrInvalid) + } + obj := Object{Size: options.Size, ContentType: options.ContentType, Metadata: options.Metadata} + if err := validMetadata(obj); err != nil { + return Object{}, err + } + obj.Metadata = maps.Clone(obj.Metadata) + end, err := s.begin(ctx) + if err != nil { + return Object{}, err + } + defer end() + release := serialise.Lock(s.domain + "/" + name + "\x00" + key) + defer release() + s.mu.RLock() + b, err := s.lookup(name) + var generation string + if err == nil { + generation = b.Generation + } + s.mu.RUnlock() + if err != nil { + return Object{}, err + } + obj.Digest, err = s.writeBlob(ctx, body, obj.Size) + if err != nil { + return Object{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + b, err = s.lookup(name) + if err != nil { + return Object{}, err + } + if b.Generation != generation { + return Object{}, fmt.Errorf("%w: bucket was recreated during upload", ErrBucketNotFound) + } + next := s.clone() + next.Buckets[name].Objects[key] = obj + if err := s.commit(ctx, next); err != nil { + return Object{}, err + } + return copyObject(obj), nil +} + +// GetObject verifies the complete blob's size and digest before returning a +// reader positioned at byte zero. It never returns missing or corrupt bytes +// as a successful empty object. The caller must close the reader. +func (s *Store) GetObject(ctx context.Context, name, key string) (Object, io.ReadCloser, error) { + if err := validKey(key); err != nil { + return Object{}, nil, err + } + end, err := s.begin(ctx) + if err != nil { + return Object{}, nil, err + } + defer end() + s.mu.RLock() + b, err := s.lookup(name) + var obj Object + if err == nil { + var ok bool + obj, ok = b.Objects[key] + if !ok { + err = ErrObjectNotFound + } + } + s.mu.RUnlock() + if err != nil { + return Object{}, nil, err + } + file, err := s.openBlob(ctx, obj) + if err != nil { + return Object{}, nil, err + } + return copyObject(obj), &blobReader{ctx: ctx, file: file}, nil +} + +// HeadObject returns metadata only after verifying the corresponding blob. +func (s *Store) HeadObject(ctx context.Context, name, key string) (Object, error) { + obj, reader, err := s.GetObject(ctx, name, key) + if err != nil { + return Object{}, err + } + if err := reader.Close(); err != nil { + return Object{}, fmt.Errorf("close verified blob: %w", err) + } + return obj, nil +} + +// ListObjects returns an independent metadata inventory, not a verification of +// every blob. GetObject or HeadObject performs byte verification. +func (s *Store) ListObjects(ctx context.Context, name string) (map[string]Object, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + b, err := s.lookup(name) + if err != nil { + return nil, err + } + result := make(map[string]Object, len(b.Objects)) + for key, obj := range b.Objects { + result[key] = copyObject(obj) + } + return result, nil +} + +// DeleteObject removes a metadata reference, retaining its immutable blob. +// Protocol adapters may choose idempotent deletion; this layer reports absence. +func (s *Store) DeleteObject(ctx context.Context, name, key string) error { + if err := validKey(key); err != nil { + return err + } + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + release := serialise.Lock(s.domain + "/" + name + "\x00" + key) + defer release() + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(name) + if err != nil { + return err + } + if _, ok := b.Objects[key]; !ok { + return ErrObjectNotFound + } + next := s.clone() + delete(next.Buckets[name].Objects, key) + return s.commit(ctx, next) +} + +func (s *Store) lookup(name string) (*bucket, error) { + if err := validName(name); err != nil { + return nil, err + } + if s.fault != nil { + return nil, s.fault + } + b, ok := s.state.Buckets[name] + if !ok { + return nil, ErrBucketNotFound + } + return b, nil +} + +func (s *Store) clone() manifest { + next := manifest{Format: s.state.Format, Version: s.state.Version, Buckets: make(map[string]*bucket)} + for name, b := range s.state.Buckets { + next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects)} + } + return next +} + +func copyObject(obj Object) Object { + obj.Metadata = maps.Clone(obj.Metadata) + return obj +} + +func validName(name string) error { + if name == "" || len(name) > 255 || !utf8.ValidString(name) { + return fmt.Errorf("%w: bucket name must be 1-255 UTF-8 bytes", ErrInvalid) + } + for _, r := range name { + if r == 0 { + return fmt.Errorf("%w: bucket name contains NUL", ErrInvalid) + } + } + return nil +} + +func validKey(key string) error { + if key == "" || len(key) > 1024 || !utf8.ValidString(key) { + return fmt.Errorf("%w: object key must be 1-1024 UTF-8 bytes", ErrInvalid) + } + return nil +} + +func validMetadata(obj Object) error { + size := len(obj.ContentType) + if !utf8.ValidString(obj.ContentType) { + return fmt.Errorf("%w: content type is not UTF-8", ErrInvalid) + } + for key, value := range obj.Metadata { + if !utf8.ValidString(key) || !utf8.ValidString(value) { + return fmt.Errorf("%w: metadata is not UTF-8", ErrInvalid) + } + size += len(key) + len(value) + if size > 64<<10 { + return fmt.Errorf("%w: metadata exceeds 65536 bytes", ErrInvalid) + } + } + if size > 64<<10 { + return fmt.Errorf("%w: metadata exceeds 65536 bytes", ErrInvalid) + } + return nil +} diff --git a/internal/core/objectstore/store_test.go b/internal/core/objectstore/store_test.go new file mode 100644 index 00000000..e113eac5 --- /dev/null +++ b/internal/core/objectstore/store_test.go @@ -0,0 +1,858 @@ +package objectstore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +func newStore(t *testing.T) (*Store, string) { + t.Helper() + path := filepath.Join(privateDir(t), "objects") + s, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }) + return s, path +} + +func privateDir(t *testing.T) string { + t.Helper() + path := t.TempDir() + if err := os.Chmod(path, 0o700); err != nil { + t.Fatal(err) + } + return path +} + +func mustBucket(t *testing.T, s *Store, name string) { + t.Helper() + if err := s.CreateBucket(t.Context(), name); err != nil { + t.Fatal(err) + } +} + +func mustPut(t *testing.T, s *Store, name, key, data string) Object { + t.Helper() + obj, err := s.PutObject(t.Context(), name, key, strings.NewReader(data), PutOptions{Size: int64(len(data))}) + if err != nil { + t.Fatal(err) + } + return obj +} + +func readObject(t *testing.T, s *Store, name, key string) (Object, []byte) { + t.Helper() + obj, reader, err := s.GetObject(t.Context(), name, key) + if err != nil { + t.Fatal(err) + } + data, readErr := io.ReadAll(reader) + if err := errors.Join(readErr, reader.Close()); err != nil { + t.Fatal(err) + } + return obj, data +} + +func TestBytesAndMetadataSurviveReopen(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + data := bytes.Repeat([]byte{0, 1, 255, '\n'}, 100000) + hash := sha256.Sum256(data) + options := PutOptions{Size: int64(len(data)), ContentType: "application/octet-stream", Metadata: map[string]string{"purpose": "roundtrip"}} + obj, err := s.PutObject(t.Context(), "b", "nested/key", bytes.NewReader(data), options) + if err != nil { + t.Fatal(err) + } + if obj.Digest != hex.EncodeToString(hash[:]) || obj.Size != int64(len(data)) { + t.Fatalf("wrong stored digest/size: %+v", obj) + } + options.Metadata["purpose"] = "caller mutation" + obj.Metadata["purpose"] = "result mutation" + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + got, body := readObject(t, reopened, "b", "nested/key") + if !bytes.Equal(body, data) || got.Metadata["purpose"] != "roundtrip" || got.ContentType != options.ContentType { + t.Fatalf("bytes or metadata changed after reopen: %+v", got) + } + list, err := reopened.ListObjects(t.Context(), "b") + if err != nil { + t.Fatal(err) + } + list["nested/key"].Metadata["purpose"] = "inventory mutation" + delete(list, "nested/key") + head, err := reopened.HeadObject(t.Context(), "b", "nested/key") + if err != nil || head.Metadata["purpose"] != "roundtrip" { + t.Fatalf("returned inventory aliased the store: %+v, %v", head, err) + } +} + +func TestLogicalNamesNeverBecomeFilesystemPaths(t *testing.T) { + s, path := newStore(t) + outside := filepath.Join(filepath.Dir(path), "outside") + if err := os.WriteFile(outside, []byte("untouched"), 0o600); err != nil { + t.Fatal(err) + } + name := "../outside" + mustBucket(t, s, name) + for _, key := range []string{"../../outside", "/absolute/key", `C:\outside`, "nested//../key", "key\x00suffix", "cle-\u00e9"} { + obj := mustPut(t, s, name, key, "real bytes") + _, data := readObject(t, s, name, key) + if string(data) != "real bytes" { + t.Fatalf("logical key %q did not roundtrip", key) + } + if !validDigest(obj.Digest) { + t.Fatal("physical blob name is not a digest") + } + } + data, err := os.ReadFile(outside) + if err != nil || string(data) != "untouched" { + t.Fatalf("logical names modified outside bytes: %q, %v", data, err) + } + files, err := os.ReadDir(filepath.Join(path, "blobs")) + if err != nil || len(files) != 1 { + t.Fatalf("identical bytes were not deduplicated: %v, %v", files, err) + } + if !validDigest(files[0].Name()) { + t.Fatalf("unexpected physical filename %q", files[0].Name()) + } +} + +func TestEmptyObjectsAndBucketLifecycle(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "z") + mustBucket(t, s, "a") + if err := s.CreateBucket(t.Context(), "a"); !errors.Is(err, ErrBucketExists) { + t.Fatalf("duplicate create: %v", err) + } + names, err := s.ListBuckets(t.Context()) + if err != nil || !reflect.DeepEqual(names, []string{"a", "z"}) { + t.Fatalf("bucket list: %v, %v", names, err) + } + mustPut(t, s, "a", "empty", "") + obj, data := readObject(t, s, "a", "empty") + if len(data) != 0 || obj.Size != 0 || obj.Digest != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Fatalf("invalid empty object: %+v, %q", obj, data) + } + if err := s.DeleteBucket(t.Context(), "a"); !errors.Is(err, ErrBucketNotEmpty) { + t.Fatalf("nonempty bucket delete: %v", err) + } + if err := s.DeleteObject(t.Context(), "a", "empty"); err != nil { + t.Fatal(err) + } + if err := s.DeleteObject(t.Context(), "a", "empty"); !errors.Is(err, ErrObjectNotFound) { + t.Fatalf("missing object delete: %v", err) + } + if err := s.DeleteBucket(t.Context(), "a"); err != nil { + t.Fatal(err) + } + if err := s.HeadBucket(t.Context(), "a"); !errors.Is(err, ErrBucketNotFound) { + t.Fatalf("deleted bucket head: %v", err) + } +} + +type failedReader struct{} + +func (failedReader) Read(p []byte) (int, error) { + copy(p, "partial") + return min(len(p), len("partial")), io.ErrUnexpectedEOF +} + +func TestUnfinishedUploadsNeverReplaceCommittedBytes(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + original := mustPut(t, s, "b", "k", "old") + for _, tc := range []struct { + name string + body io.Reader + size int64 + want error + }{ + {"short", strings.NewReader("x"), 2, ErrInvalid}, + {"long", strings.NewReader("xxx"), 2, ErrInvalid}, + {"interrupted", failedReader{}, 100, io.ErrUnexpectedEOF}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := s.PutObject(t.Context(), "b", "k", tc.body, PutOptions{Size: tc.size}) + if !errors.Is(err, tc.want) { + t.Fatalf("upload refusal: %v", err) + } + obj, body := readObject(t, s, "b", "k") + if string(body) != "old" || obj.Digest != original.Digest { + t.Fatalf("failed upload published bytes: %+v %q", obj, body) + } + }) + } + entries, err := os.ReadDir(filepath.Join(path, "blobs")) + if err != nil || len(entries) != 1 { + t.Fatalf("unfinished upload leaked temporary bytes: %v, %v", entries, err) + } +} + +type gatedReader struct { + reader io.Reader + entered chan struct{} + release chan struct{} + once sync.Once +} + +func gate(t *testing.T, data string) (*gatedReader, func()) { + t.Helper() + r := &gatedReader{reader: strings.NewReader(data), entered: make(chan struct{}), release: make(chan struct{})} + var once sync.Once + unblock := func() { once.Do(func() { close(r.release) }) } + t.Cleanup(unblock) + return r, unblock +} + +func (r *gatedReader) Read(p []byte) (int, error) { + r.once.Do(func() { + close(r.entered) + <-r.release + }) + return r.reader.Read(p) +} + +func waitFor(t *testing.T, ch <-chan error) error { + t.Helper() + select { + case result := <-ch: + return result + case <-time.After(5 * time.Second): + t.Fatal("bounded operation did not finish") + return errors.New("operation did not finish") + } +} + +func waitReady(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("upload did not reach its controlled read") + } +} + +func TestDeletedAndRecreatedBucketCannotBeResurrectedByUpload(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + reader, unblock := gate(t, "new") + result := make(chan error, 1) + go func() { + _, err := s.PutObject(t.Context(), "b", "k", reader, PutOptions{Size: 3}) + result <- err + }() + waitReady(t, reader.entered) + if err := s.DeleteBucket(t.Context(), "b"); err != nil { + t.Fatal(err) + } + mustBucket(t, s, "b") + unblock() + if err := waitFor(t, result); !errors.Is(err, ErrBucketNotFound) { + t.Fatalf("stale upload committed into recreated bucket: %v", err) + } + objects, err := s.ListObjects(t.Context(), "b") + if err != nil || len(objects) != 0 { + t.Fatalf("new bucket inherited stale upload: %v, %v", objects, err) + } + mustPut(t, s, "b", "k", "accepted") +} + +func TestUnrelatedKeysProgressWhileAnUploadIsBlocked(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + reader, unblock := gate(t, "blocked") + first := make(chan error, 1) + go func() { + _, err := s.PutObject(t.Context(), "b", "one", reader, PutOptions{Size: 7}) + first <- err + }() + waitReady(t, reader.entered) + second := make(chan error, 1) + go func() { + _, err := s.PutObject(t.Context(), "b", "two", strings.NewReader("independent"), PutOptions{Size: 11}) + second <- err + }() + if err := waitFor(t, second); err != nil { + t.Fatal(err) + } + unblock() + if err := waitFor(t, first); err != nil { + t.Fatal(err) + } +} + +func TestCancellationNeverPublishesPartialOrCompletedUpload(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + reader, unblock := gate(t, "new") + ctx, cancel := context.WithCancel(t.Context()) + result := make(chan error, 1) + go func() { + _, err := s.PutObject(ctx, "b", "k", reader, PutOptions{Size: 3}) + result <- err + }() + waitReady(t, reader.entered) + cancel() + unblock() + if err := waitFor(t, result); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled upload: %v", err) + } + if _, _, err := s.GetObject(t.Context(), "b", "k"); !errors.Is(err, ErrObjectNotFound) { + t.Fatalf("canceled upload published metadata: %v", err) + } + mustPut(t, s, "b", "k", "accepted") +} + +func TestReaderSeesCompleteOldBytesDuringReplacementAndAfterDelete(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + mustPut(t, s, "b", "k", "old") + reader, unblock := gate(t, "replacement") + result := make(chan error, 1) + go func() { + _, err := s.PutObject(t.Context(), "b", "k", reader, PutOptions{Size: 11}) + result <- err + }() + waitReady(t, reader.entered) + _, old := readObject(t, s, "b", "k") + if string(old) != "old" { + t.Fatalf("uncommitted replacement leaked: %q", old) + } + unblock() + if err := waitFor(t, result); err != nil { + t.Fatal(err) + } + _, held, err := s.GetObject(t.Context(), "b", "k") + if err != nil { + t.Fatal(err) + } + if err := s.DeleteObject(t.Context(), "b", "k"); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + data, readErr := io.ReadAll(held) + if err := errors.Join(readErr, held.Close()); err != nil { + t.Fatal(err) + } + if string(data) != "replacement" { + t.Fatalf("held reader lost immutable bytes: %q", data) + } +} + +func TestReturnedReaderHonorsCancellationThroughCopy(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + mustPut(t, s, "b", "k", "good") + ctx, cancel := context.WithCancel(t.Context()) + _, reader, err := s.GetObject(ctx, "b", "k") + if err != nil { + t.Fatal(err) + } + cancel() + n, err := io.Copy(io.Discard, reader) + if !errors.Is(err, context.Canceled) || n != 0 { + t.Fatalf("io.Copy bypassed cancellation: %d bytes, %v", n, err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } +} + +func TestConcurrentPutAndDeleteNeverTearBytesOrLoseOtherKeys(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + var wg sync.WaitGroup + results := make(chan error, 64) + for i := range 32 { + wg.Go(func() { + data := strings.Repeat(fmt.Sprintf("%02d", i), 4096) + _, err := s.PutObject(t.Context(), "b", "shared", strings.NewReader(data), PutOptions{Size: int64(len(data))}) + if err != nil { + results <- err + return + } + err = s.DeleteObject(t.Context(), "b", "shared") + if err != nil && !errors.Is(err, ErrObjectNotFound) { + results <- err + } + _, err = s.PutObject(t.Context(), "b", fmt.Sprintf("key-%02d", i), strings.NewReader(data), PutOptions{Size: int64(len(data))}) + results <- err + }) + } + wg.Wait() + close(results) + for err := range results { + if err != nil { + t.Fatal(err) + } + } + for i := range 32 { + obj, data := readObject(t, s, "b", fmt.Sprintf("key-%02d", i)) + expected := strings.Repeat(fmt.Sprintf("%02d", i), 4096) + hash := sha256.Sum256(data) + if string(data) != expected || obj.Digest != hex.EncodeToString(hash[:]) { + t.Fatal("concurrent operation tore bytes or lost metadata") + } + } +} + +func TestMissingAndCorruptBlobsFailGetHeadAndReopen(t *testing.T) { + for _, mode := range []string{"missing", "wrong-size", "wrong-digest", "symlink", "permissions"} { + t.Run(mode, func(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + obj := mustPut(t, s, "b", "k", "good") + blob := filepath.Join(path, "blobs", obj.Digest) + var err error + switch mode { + case "missing": + err = os.Remove(blob) + case "wrong-size": + err = os.WriteFile(blob, []byte("shorter"), 0o600) + case "wrong-digest": + err = os.WriteFile(blob, []byte("evil"), 0o600) + case "symlink": + target := filepath.Join(filepath.Dir(path), "outside") + if err := os.WriteFile(target, []byte("good"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Remove(blob); err != nil { + t.Fatal(err) + } + err = os.Symlink(target, blob) + case "permissions": + err = os.Chmod(blob, 0o644) + } + if err != nil { + t.Fatal(err) + } + if _, reader, err := s.GetObject(t.Context(), "b", "k"); !errors.Is(err, ErrCorrupt) || reader != nil { + t.Fatalf("GET served invalid bytes: reader=%v err=%v", reader, err) + } + if _, err := s.HeadObject(t.Context(), "b", "k"); !errors.Is(err, ErrCorrupt) { + t.Fatalf("HEAD reported corrupt blob held: %v", err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if reopened, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || reopened != nil { + t.Fatalf("reopen admitted corrupt blob: store=%v err=%v", reopened, err) + } + }) + } +} + +func TestRootOwnershipPermissionsAndSymlinksAreRefused(t *testing.T) { + t.Run("unmarked", func(t *testing.T) { + path := filepath.Join(privateDir(t), "operator") + if err := os.Mkdir(path, 0o700); err != nil { + t.Fatal(err) + } + if _, err := Open(t.Context(), path); !errors.Is(err, ErrUnowned) { + t.Fatalf("adopted unmarked directory: %v", err) + } + entries, err := os.ReadDir(path) + if err != nil || len(entries) != 0 { + t.Fatalf("unmarked directory modified: %v, %v", entries, err) + } + }) + for _, mode := range []string{"root-symlink", "root-permissions", "index-symlink", "index-permissions", "blobs-symlink"} { + t.Run(mode, func(t *testing.T) { + s, path := newStore(t) + if err := s.Close(); err != nil { + t.Fatal(err) + } + switch mode { + case "root-symlink": + alias := path + "-alias" + if err := os.Symlink(filepath.Base(path), alias); err != nil { + t.Fatal(err) + } + path = alias + case "root-permissions": + if err := os.Chmod(path, 0o755); err != nil { + t.Fatal(err) + } + case "index-symlink": + if err := os.Rename(filepath.Join(path, "index.json"), filepath.Join(path, "saved-index")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("saved-index", filepath.Join(path, "index.json")); err != nil { + t.Fatal(err) + } + case "index-permissions": + if err := os.Chmod(filepath.Join(path, "index.json"), 0o644); err != nil { + t.Fatal(err) + } + case "blobs-symlink": + if err := os.Rename(filepath.Join(path, "blobs"), filepath.Join(path, "saved-blobs")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("saved-blobs", filepath.Join(path, "blobs")); err != nil { + t.Fatal(err) + } + } + reopened, err := Open(t.Context(), path) + if err == nil || reopened != nil { + t.Fatalf("unsafe store root admitted: %v, %v", reopened, err) + } + }) + } +} + +func TestUnsafeParentIsRefusedBeforeCreation(t *testing.T) { + for _, mode := range []os.FileMode{0o777, 0o770, 0o707} { + t.Run(mode.String(), func(t *testing.T) { + parent := t.TempDir() + if err := os.Chmod(parent, mode); err != nil { + t.Fatal(err) + } + operator := filepath.Join(parent, "operator") + if err := os.Mkdir(operator, 0o700); err != nil { + t.Fatal(err) + } + index := filepath.Join(operator, "index.json") + if err := os.WriteFile(index, []byte("unrelated operator data"), 0o600); err != nil { + t.Fatal(err) + } + path := filepath.Join(parent, "new") + store, err := Open(t.Context(), path) + if store != nil { + if err := store.Close(); err != nil { + t.Error(err) + } + } + if !errors.Is(err, ErrUnowned) || store != nil { + t.Fatalf("unsafe parent admitted: %v, %v", store, err) + } + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unsafe parent was modified before refusal: %v", err) + } + data, err := os.ReadFile(index) + if err != nil || string(data) != "unrelated operator data" { + t.Fatalf("operator index modified: %q, %v", data, err) + } + }) + } + for _, mode := range []os.FileMode{0o700, 0o755, os.ModeSticky | 0o777} { + t.Run("accept-"+mode.String(), func(t *testing.T) { + parent := t.TempDir() + if err := os.Chmod(parent, mode); err != nil { + t.Fatal(err) + } + s, err := Open(t.Context(), filepath.Join(parent, "new")) + if err != nil { + t.Fatalf("protected parent refused: %v", err) + } + mustBucket(t, s, "b") + mustPut(t, s, "b", "k", "safe") + if err := s.Close(); err != nil { + t.Fatal(err) + } + }) + } +} + +type parentOwnerInfo struct { + os.FileInfo + uid uint32 +} + +func (info parentOwnerInfo) Sys() any { + stat := *info.FileInfo.Sys().(*syscall.Stat_t) + stat.Uid = info.uid + return &stat +} + +func TestStickyProtectionDoesNotTrustAForeignParentOwner(t *testing.T) { + path := t.TempDir() + if err := os.Chmod(path, os.ModeSticky|0o777); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + foreign := uint32(1) + if int64(foreign) == int64(os.Geteuid()) { + foreign = 2 + } + if err := checkParent(parentOwnerInfo{FileInfo: info, uid: foreign}); !errors.Is(err, ErrUnowned) { + t.Fatalf("sticky parent owner can still replace entries: %v", err) + } + if err := checkParent(parentOwnerInfo{FileInfo: info, uid: 0}); err != nil { + t.Fatalf("root-owned sticky parent refused: %v", err) + } + if err := checkParent(info); err != nil { + t.Fatalf("this user's sticky parent refused: %v", err) + } +} + +func TestRootIsExclusiveAcrossInstancesAndProcesses(t *testing.T) { + const envName = "FEINT_OBJECTSTORE_TEST_LOCK_ROOT" + const readAfterClose = "FEINT_OBJECTSTORE_TEST_READ_AFTER_CLOSE" + if path := os.Getenv(envName); path != "" { + if os.Getenv(readAfterClose) == "1" { + s, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + _, body := readObject(t, s, "b", "k") + if string(body) != "durable bytes" { + t.Fatalf("fresh process lost persisted bytes: %q", body) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + return + } + if other, err := Open(t.Context(), path); !errors.Is(err, ErrLocked) || other != nil { + t.Fatalf("child acquired locked root: %v, %v", other, err) + } + return + } + s, path := newStore(t) + mustBucket(t, s, "b") + mustPut(t, s, "b", "k", "durable bytes") + if other, err := Open(t.Context(), path); !errors.Is(err, ErrLocked) || other != nil { + t.Fatalf("second instance acquired locked root: %v, %v", other, err) + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRootIsExclusiveAcrossInstancesAndProcesses$") + cmd.Env = append(os.Environ(), envName+"="+path, readAfterClose+"=0") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("child root ownership check failed: %v\n%s", err, output) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + cmd = exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRootIsExclusiveAcrossInstancesAndProcesses$") + cmd.Env = append(os.Environ(), envName+"="+path, readAfterClose+"=1") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fresh process could not read persisted bytes: %v\n%s", err, output) + } + other, err := Open(t.Context(), path) + if err != nil { + t.Fatalf("closed store did not release process lock: %v", err) + } + if err := other.Close(); err != nil { + t.Fatal(err) + } +} + +func TestInvalidInputsAndClosedStoreReturnErrors(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + for _, name := range []string{"", "nul\x00name", strings.Repeat("n", 256), string([]byte{255})} { + if err := s.CreateBucket(t.Context(), name); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid bucket name %q accepted: %v", name, err) + } + } + for _, key := range []string{"", strings.Repeat("k", 1025), string([]byte{255})} { + if _, err := s.PutObject(t.Context(), "b", key, strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid key accepted: %v", err) + } + if _, reader, err := s.GetObject(t.Context(), "b", key); !errors.Is(err, ErrInvalid) || reader != nil { + t.Fatalf("invalid read key accepted: %v", err) + } + } + for _, options := range []PutOptions{ + {Size: -1}, {Size: math.MaxInt64}, + {Size: 1, ContentType: strings.Repeat("c", 65537)}, + {Size: 1, Metadata: map[string]string{"m": strings.Repeat("v", 65536)}}, + {Size: 1, Metadata: map[string]string{"m": string([]byte{255})}}, + } { + if _, err := s.PutObject(t.Context(), "b", "k", strings.NewReader("x"), options); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid options accepted: %v", err) + } + } + if _, err := s.PutObject(t.Context(), "b", "k", nil, PutOptions{}); !errors.Is(err, ErrInvalid) { + t.Fatalf("nil body accepted: %v", err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := s.CreateBucket(ctx, "canceled"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled mutation accepted: %v", err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.ListBuckets(t.Context()); !errors.Is(err, ErrClosed) { + t.Fatalf("closed store accepted access: %v", err) + } +} + +func TestPoisonedMetadataCannotOpenAnOutsideBlob(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + mustPut(t, s, "b", "k", "good") + if err := s.Close(); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(filepath.Join(path, "index.json")) + if err != nil { + t.Fatal(err) + } + state, err := decodeManifest(body) + if err != nil { + t.Fatal(err) + } + obj := state.Buckets["b"].Objects["k"] + obj.Digest = "../" + strings.Repeat("a", 61) + state.Buckets["b"].Objects["k"] = obj + poisoned, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "index.json"), poisoned, 0o600); err != nil { + t.Fatal(err) + } + if other, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || other != nil { + t.Fatalf("poisoned persisted path was admitted: %v, %v", other, err) + } +} + +func TestManifestIsClosedVersionedAndValidated(t *testing.T) { + valid := `{"format":"feint-objectstore","version":1,"buckets":{}}` + if _, err := decodeManifest([]byte(valid)); err != nil { + t.Fatalf("valid manifest refused: %v", err) + } + for _, body := range []string{ + `null`, `{}`, `{"format":"operator","version":1,"buckets":{}}`, + `{"format":"feint-objectstore","version":2,"buckets":{}}`, + `{"format":"feint-objectstore","version":1,"buckets":null}`, + `{"format":"feint-objectstore","version":1,"buckets":{"b":null}}`, + `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"","objects":{}}}}`, + `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"test","objects":{"k":{"digest":"../../outside","size":4}}}}}`, + `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"test","objects":{"k":{"digest":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":-1}}}}}`, + `{"format":"feint-objectstore","version":1,"buckets":{},"snapshot":{}}`, + valid + `{}`, valid + `junk`, + } { + if _, err := decodeManifest([]byte(body)); err == nil { + t.Fatalf("invalid manifest admitted: %s", body) + } + } +} + +func TestDirectorySyncFailureIsNotSuccessAndRequiresReopen(t *testing.T) { + s, path := newStore(t) + failedDir, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + if err := failedDir.Close(); err != nil { + t.Fatal(err) + } + lockedDir := s.dir + s.dir = failedDir + defer func() { s.dir = lockedDir }() + if err := s.CreateBucket(t.Context(), "uncertain"); err == nil { + t.Fatal("directory sync failure reported success") + } + if _, err := s.ListBuckets(t.Context()); err == nil { + t.Fatal("indeterminate instance continued serving") + } + if err := s.CreateBucket(t.Context(), "later"); err == nil { + t.Fatal("indeterminate instance continued mutating") + } + s.dir = lockedDir + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + if err := reopened.HeadBucket(t.Context(), "uncertain"); err != nil { + t.Fatalf("reopen did not load published metadata: %v", err) + } + if err := reopened.HeadBucket(t.Context(), "later"); !errors.Is(err, ErrBucketNotFound) { + t.Fatalf("refused later mutation was published: %v", err) + } +} + +func TestFailedOpenReleasesOwnershipLock(t *testing.T) { + s, path := newStore(t) + if err := s.Close(); err != nil { + t.Fatal(err) + } + index := filepath.Join(path, "index.json") + original, err := os.ReadFile(index) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":2,"buckets":{}}`), 0o600); err != nil { + t.Fatal(err) + } + if broken, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || broken != nil { + t.Fatalf("invalid metadata accepted: %v, %v", broken, err) + } + if err := os.WriteFile(index, original, 0o600); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatalf("failed open retained a process lock: %v", err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } +} + +func FuzzManifest(f *testing.F) { + f.Add([]byte(`{"format":"feint-objectstore","version":1,"buckets":{}}`)) + f.Add([]byte(`{"format":"feint-objectstore","version":1,"buckets":{"b":null}}`)) + f.Fuzz(func(t *testing.T, body []byte) { + state, err := decodeManifest(body) + if err != nil { + return + } + encoded, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + roundtrip, err := decodeManifest(encoded) + if err != nil || !reflect.DeepEqual(state, roundtrip) { + t.Fatalf("accepted manifest cannot roundtrip: %v", err) + } + }) +} diff --git a/tools/falsify/specs/neutral-object-bytes.json b/tools/falsify/specs/neutral-object-bytes.json new file mode 100644 index 00000000..d0cca6d4 --- /dev/null +++ b/tools/falsify/specs/neutral-object-bytes.json @@ -0,0 +1,117 @@ +{ + "package": "./internal/core/objectstore/", + "mutations": [ + { + "label": "a returned reader lets io.Copy continue after cancellation", + "file": "internal/core/objectstore/files.go", + "find": "if err := r.ctx.Err(); err != nil {", + "replace": "if err := r.ctx.Err(); err != nil && false {", + "test": "TestReturnedReaderHonorsCancellationThroughCopy" + }, + { + "label": "a writable non-sticky parent admits the root-creation substitution window", + "file": "internal/core/objectstore/files.go", + "find": "if info.Mode().Perm()&0o022 != 0 && info.Mode()&os.ModeSticky == 0 {", + "replace": "if info.Mode().Perm()&0o022 != 0 && info.Mode()&os.ModeSticky == 0 && false {", + "test": "TestUnsafeParentIsRefusedBeforeCreation" + }, + { + "label": "a foreign parent owner is trusted merely because the directory is sticky", + "file": "internal/core/objectstore/files.go", + "find": "if !ok || (stat.Uid != 0 && int64(stat.Uid) != int64(os.Geteuid())) {", + "replace": "if !ok || (stat.Uid != 0 && int64(stat.Uid) != int64(os.Geteuid()) && false) {", + "test": "TestStickyProtectionDoesNotTrustAForeignParentOwner" + }, + { + "label": "the blob loses its written bytes before its metadata is published", + "file": "internal/core/objectstore/files.go", + "find": "\t\tcopyErr = file.Sync()", + "replace": "\t\tcopyErr = file.Truncate(0)\n\t\tif copyErr == nil {\n\t\t\tcopyErr = file.Sync()\n\t\t}", + "test": "TestBytesAndMetadataSurviveReopen" + }, + { + "label": "a stale upload is allowed into a recreated bucket", + "file": "internal/core/objectstore/store.go", + "find": "if b.Generation != generation {", + "replace": "if b.Generation != generation && false {", + "test": "TestDeletedAndRecreatedBucketCannotBeResurrectedByUpload" + }, + { + "label": "a nonempty bucket can be deleted", + "file": "internal/core/objectstore/store.go", + "find": "if len(b.Objects) != 0 {", + "replace": "if len(b.Objects) != 0 && false {", + "test": "TestEmptyObjectsAndBucketLifecycle" + }, + { + "label": "short and long uploads publish metadata", + "file": "internal/core/objectstore/files.go", + "find": "if copyErr == nil && n != size {", + "replace": "if copyErr == nil && n != size && false {", + "test": "TestUnfinishedUploadsNeverReplaceCommittedBytes" + }, + { + "label": "same-length corrupted bytes pass digest verification", + "file": "internal/core/objectstore/files.go", + "find": "if n != obj.Size || hex.EncodeToString(hash.Sum(nil)) != obj.Digest {", + "replace": "if n != obj.Size || (hex.EncodeToString(hash.Sum(nil)) != obj.Digest && false) {", + "test": "TestMissingAndCorruptBlobsFailGetHeadAndReopen" + }, + { + "label": "public permissions are accepted as private ownership", + "file": "internal/core/objectstore/files.go", + "find": "if info.Mode().Perm()&0o077 != 0 {", + "replace": "if info.Mode().Perm()&0o077 != 0 && false {", + "test": "TestRootOwnershipPermissionsAndSymlinksAreRefused" + }, + { + "label": "root inspection follows a symlink instead of refusing it", + "file": "internal/core/objectstore/files.go", + "find": "info, err := parent.Lstat(name)", + "replace": "info, err := parent.Stat(name)", + "test": "TestRootOwnershipPermissionsAndSymlinksAreRefused" + }, + { + "label": "metadata inspection follows an internal symlink", + "file": "internal/core/objectstore/files.go", + "find": "info, err := root.Lstat(name)", + "replace": "info, err := root.Stat(name)", + "test": "TestRootOwnershipPermissionsAndSymlinksAreRefused" + }, + { + "label": "an unmarked private directory can be adopted", + "file": "internal/core/objectstore/files.go", + "find": "\t\ts.state, err = readManifest(ctx, root)", + "replace": "\t\ts.state, err = readManifest(ctx, root)\n\t\tif err != nil {\n\t\t\ts.state = manifest{Format: storeFormat, Version: 1, Buckets: make(map[string]*bucket)}\n\t\t\terr = root.MkdirAll(\"blobs\", 0o700)\n\t\t}", + "test": "TestRootOwnershipPermissionsAndSymlinksAreRefused" + }, + { + "label": "a future metadata version is silently accepted", + "file": "internal/core/objectstore/manifest.go", + "find": "if state.Version != 1 || state.Buckets == nil {", + "replace": "if (state.Version != 1 && false) || state.Buckets == nil {", + "test": "TestManifestIsClosedVersionedAndValidated" + }, + { + "label": "unknown persisted fields are silently ignored", + "file": "internal/core/objectstore/manifest.go", + "find": "\tdecoder.DisallowUnknownFields()", + "replace": "\tif false { decoder.DisallowUnknownFields() }", + "test": "TestManifestIsClosedVersionedAndValidated" + }, + { + "label": "the exclusive directory process lock is disabled", + "file": "internal/core/objectstore/files.go", + "find": "syscall.Flock(int(s.dir.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)", + "replace": "syscall.Flock(int(s.dir.Fd()), syscall.LOCK_UN)", + "test": "TestRootIsExclusiveAcrossInstancesAndProcesses" + }, + { + "label": "directory sync failure reports a successful metadata commit", + "file": "internal/core/objectstore/manifest.go", + "find": "if err := s.dir.Sync(); err != nil {", + "replace": "if err := s.dir.Sync(); err != nil && false {", + "test": "TestDirectorySyncFailureIsNotSuccessAndRequiresReopen" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index b33f6af6..68d23c2a 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -234,6 +234,12 @@ var rules = []rule{ Absent: []string{"--state", "snapshot"}, }}, }, + { + Path: "internal/core/objectstore/", + Why: "neutral persisted bytes, not yet connected to a served protocol", + Unproven: "the package tests prove byte round trips and reopen, not S3 responses, " + + "real-client conformance, checkpoint/reset, power-loss recovery or network filesystem locking", + }, // ------------------------------------------------------ what describes it { From d0e335c0d2bbb30b309ed2b677ecd5ca74f70018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 6 Nov 2026 23:06:34 +0100 Subject: [PATCH 02/28] feat(s3): prove explicit gateway byte transfers with AWS CLI Adapt the neutral ObjectStore to bounded path-style CRUD. Decode aws-chunked framing and verify CRC32/SHA-256 before metadata publication. Prove five lifecycle operations over trusted loopback HTTPS, plus official-client checksum rejection. Keep provider mounting, interception, ETags and checkpoints out of scope. Validation: mise run check and conformance:s3 passed; 15 declared mutations compiled and bit, with restored baseline green. prepush remains red on two untriaged upstream Scaleway LB operations, unrelated to this change. Refs: #850 Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/architecture.md | 59 ++- internal/s3/client_conformance_test.go | 225 +++++++++ internal/s3/server.go | 380 ++++++++++++++++ internal/s3/server_test.go | 428 ++++++++++++++++++ .../s3/testdata/exoscale-no-such-bucket.xml | 1 + .../s3/testdata/scaleway-no-such-bucket.xml | 1 + internal/s3/upload.go | 196 ++++++++ mise.toml | 4 + tools/falsify/specs/s3-gateway.json | 110 +++++ tools/testplan/rules.go | 14 +- 10 files changed, 1414 insertions(+), 4 deletions(-) create mode 100644 internal/s3/client_conformance_test.go create mode 100644 internal/s3/server.go create mode 100644 internal/s3/server_test.go create mode 100644 internal/s3/testdata/exoscale-no-such-bucket.xml create mode 100644 internal/s3/testdata/scaleway-no-such-bucket.xml create mode 100644 internal/s3/upload.go create mode 100644 tools/falsify/specs/s3-gateway.json diff --git a/docs/architecture.md b/docs/architecture.md index 0b5b00c5..5cd5c7d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,7 +13,8 @@ internal/cli/ the commands: lifecycle, serve, env, doctor, snapsho coverage, probe, docs, catalog, clean, version internal/core/resource/ the neutral resource: ID, Kind, Tenant, State, Attrs, Runtime internal/core/store/ in-memory storage plus JSON snapshot, with no provider knowledge -internal/core/objectstore/ neutral persisted object bytes and metadata; no S3 route yet +internal/core/objectstore/ neutral persisted object bytes and metadata +internal/s3/ bounded path-style protocol component; not mounted yet internal/core/emulator/ Env, Pack, Route, the single-port mount, /_feint/* internal/core/machine/ the machine runtime behind --vm: Incus containers, VMs, OVN internal/core/network/ address plans, subnets, firewall bindings @@ -112,6 +113,62 @@ The caller must impose storage quotas and unblock its input reader on cancellation; waiting for an existing same-key transaction may also delay cancellation. No structured format passes through a text template. +### Explicit S3 component, not shipped Object Storage support + +[s3](../internal/s3/server.go) is a direct HTTP handler over that store, with no +listener, CLI option, provider route or snapshot integration. The mono-port +server remains unchanged. Mounting provider doors, trusted intercepted host +handoff and virtual-host routing are separate decisions, not implicit promises. +It accepts loopback peers and literal loopback/localhost endpoint hosts only. +SigV4 credentials are accepted without authenticating their secret; they are +never logged. Local callers still have access to the whole selected store. + +Run `mise run conformance:s3` with **AWS CLI 2.32.17** installed. Absence or a +different version fails, never skips. The isolated HOME uses the repository's +public fake credentials, an ephemeral trusted HTTPS certificate, metadata +discovery disabled and a non-forwarding proxy with a positive CONNECT control. +This is a client proxy boundary, **not a network namespace or a proof that +arbitrary clients cannot bypass proxies**. + +The witness performs CreateBucket, PutObject, GetObject, DeleteObject and +DeleteBucket against the component, retrieves 73,728 identical binary bytes and +checks their independent SHA-256. HTTPS makes this client emit `aws-chunked` +with a CRC32 trailer. CRC32 is an explicit fixture selection, not a claim about +the default algorithm or unmodified Terraform stacks. A sixth GET deliberately +changes only the response checksum: the official client must reject it. No +ETag is emitted; this client succeeds without one. This does **not** establish +Scaleway/SOS ETag behavior or support clients that require ETags. + +Uploads require an exact decoded length, capped by the caller's byte bound. +Framing is decoded and CRC32/SHA-256 checksums verified **before** EOF can let +the core publish metadata. The raw body is bounded to twice that cap plus +64 KiB; the trailer section is capped at 8 KiB. Logical object keys are decoded +once, never cleaned. HTTP metadata is validated at intake and again after +reading the neutral store, including case-insensitive header collisions. +Callers need server read/header deadlines, header limits and **cumulative** +disk quotas; this per-request bound does not prevent many uploads or reclaim +unreferenced blobs. + +Only empty-body bucket creation, bucket HEAD/deletion and full object +PUT/GET/HEAD/deletion are implemented. Queries other than SDK `x-id`, range and +conditional requests, ACLs, policies, versioning, multipart, encryption, +Content-MD5 and other checksum algorithms are explicitly refused. There are no +successful empty bucket-listing or multipart stubs. +The only cloud XML error is NoSuchBucket, with explicitly selected Scaleway or +SOS envelopes from the anonymous HTTPS GET recordings of +2026-11-06T20:33:53Z at `s3.fr-par.scw.cloud` and `sos-ch-gva-2.exo.io`. +[The test](../internal/s3/server_test.go) decodes the recorded producer bodies; +their response identifiers are redacted, not their fields. Applying that error +family to other missing-bucket operations is an adapter choice, not another +cloud measurement. NoSuchKey, conflicting creation and BucketNotEmpty still +get a logged **Feint 501 text refusal**, not fabricated cloud XML. + +The official-client witness proves a protocol component, not an invocation of +`feint serve`, `scw`, `exo`, a surveyed Terraform stack, interception or reset. +Existing Object Storage support limits and the full conformance population are +unchanged. [The declared mutations](../tools/falsify/specs/s3-gateway.json) +hold checksum, framing, size, metadata and unsupported-feature guards. + ## What varies is a field, not a convention Three packs do the same things with different words, and confusing the two levels diff --git a/internal/s3/client_conformance_test.go b/internal/s3/client_conformance_test.go new file mode 100644 index 00000000..c575d957 --- /dev/null +++ b/internal/s3/client_conformance_test.go @@ -0,0 +1,225 @@ +//go:build conformance + +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestOfficialAWSCLIBytes(t *testing.T) { + aws, err := exec.LookPath("aws") + if err != nil { + t.Fatal("install AWS CLI 2.32.17 for this explicit real-client proof:", err) + } + s, store := fixture(t) + var guardHits atomic.Int64 + guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + guardHits.Add(1) + http.Error(w, "feint conformance: external requests are refused, never forwarded", 502) + })) + defer guard.Close() + // A positive control proves the proxy actually records and refuses CONNECT. + if err := probeProxyGuard(t.Context(), guard.URL); err != nil { + t.Fatal(err) + } + if guardHits.Load() != 1 { + t.Fatal("guard did not observe its positive control") + } + before := guardHits.Load() + type exchange struct { + Method, Path, Encoding, DecodedLength, ChecksumAlgorithm string + SigV4 bool + Status int + } + var mu sync.Mutex + var exchanges []exchange + var corruptChecksum atomic.Bool + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + record := exchange{Method: r.Method, Path: r.URL.RequestURI(), + Encoding: r.Header.Get("Content-Encoding"), DecodedLength: r.Header.Get("X-Amz-Decoded-Content-Length"), + ChecksumAlgorithm: r.Header.Get("X-Amz-Sdk-Checksum-Algorithm"), + SigV4: strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 ")} + response := &statusWriter{ResponseWriter: w, corrupt: corruptChecksum.Load()} + s.ServeHTTP(response, r) + record.Status = response.status + mu.Lock() + exchanges = append(exchanges, record) + mu.Unlock() + })) + endpoint.Config.ReadHeaderTimeout = 5 * time.Second + endpoint.Config.ReadTimeout = 30 * time.Second + endpoint.Config.WriteTimeout = 30 * time.Second + endpoint.StartTLS() + defer endpoint.Close() + home := t.TempDir() + ca := filepath.Join(home, "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: endpoint.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") + if err != nil { + t.Fatal(err) + } + fake := make(map[string]string) + for _, line := range strings.Split(string(creds), "\n") { + if key, value, ok := strings.Cut(line, "="); ok && strings.HasPrefix(key, "SCW_") { + fake[key] = value + } + } + if fake["SCW_ACCESS_KEY"] == "" || fake["SCW_SECRET_KEY"] == "" { + t.Fatal("canonical public fake credentials are absent") + } + env := []string{ + "HOME=" + home, "PATH=" + filepath.Dir(aws) + ":/usr/bin:/bin", + "AWS_ACCESS_KEY_ID=" + fake["SCW_ACCESS_KEY"], "AWS_SECRET_ACCESS_KEY=" + fake["SCW_SECRET_KEY"], + "AWS_DEFAULT_REGION=fr-par", "AWS_EC2_METADATA_DISABLED=true", + "AWS_CONFIG_FILE=" + filepath.Join(home, "config"), "AWS_SHARED_CREDENTIALS_FILE=" + filepath.Join(home, "credentials"), + "AWS_PAGER=", "AWS_CLI_AUTO_PROMPT=off", "AWS_MAX_ATTEMPTS=1", + "AWS_CA_BUNDLE=" + ca, + "HTTP_PROXY=" + guard.URL, "HTTPS_PROXY=" + guard.URL, "ALL_PROXY=" + guard.URL, + "NO_PROXY=127.0.0.1,localhost,::1", "LANG=C.UTF-8", + } + run := func(args ...string) []byte { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, aws, args...) + cmd.Env = env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("AWS CLI %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return out + } + version := string(run("--version")) + if !strings.HasPrefix(version, "aws-cli/2.32.17 ") { + t.Fatal("proof requires the measured AWS CLI 2.32.17, got:", version) + } + t.Log(strings.TrimSpace(version)) + call := func(args ...string) []byte { + t.Helper() + return run(append([]string{"--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api"}, args...)...) + } + payload := bytes.Repeat([]byte{0, 255, 'f', 'e', 'i', 'n', 't', '\r', '\n'}, 8192) + input, output := filepath.Join(home, "input"), filepath.Join(home, "output") + if err := os.WriteFile(input, payload, 0600); err != nil { + t.Fatal(err) + } + call("create-bucket", "--bucket", "feint-byte-proof") + put := call("put-object", "--bucket", "feint-byte-proof", "--key", "proof/data", + "--body", input, "--content-type", "application/octet-stream", "--metadata", "purpose=byte-proof", + "--checksum-algorithm", "CRC32") + get := call("get-object", "--bucket", "feint-byte-proof", "--key", "proof/data", "--checksum-mode", "ENABLED", output) + got, err := os.ReadFile(output) + if err != nil || !bytes.Equal(got, payload) { + t.Fatal("official client did not retrieve identical bytes", err) + } + sum := sha256.Sum256(payload) + for _, out := range [][]byte{put, get} { + var result map[string]json.RawMessage + if err := json.Unmarshal(out, &result); err != nil { + t.Fatal("client result is not JSON:", err, string(out)) + } + var checksum string + if err := json.Unmarshal(result["ChecksumSHA256"], &checksum); err != nil || checksum != base64.StdEncoding.EncodeToString(sum[:]) { + t.Fatal("client did not return the real SHA-256 checksum", err, string(out)) + } + if _, exists := result["ETag"]; exists { + t.Fatal("gateway must not fabricate an unmeasured ETag") + } + } + // The accepting half alone cannot prove the client validates this header. + // Alter only the response checksum, retaining the same successful bytes. + corruptChecksum.Store(true) + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + cmd := exec.CommandContext(ctx, aws, "--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api", + "get-object", "--bucket", "feint-byte-proof", "--key", "proof/data", "--checksum-mode", "ENABLED", filepath.Join(home, "corrupt-output")) + cmd.Env = env + control, controlErr := cmd.CombinedOutput() + cancel() + corruptChecksum.Store(false) + if controlErr == nil || !strings.Contains(strings.ToLower(string(control)), "checksum") { + t.Fatal("official client did not reject a wrong response checksum", controlErr, string(control)) + } + t.Log("negative control: identical response bytes with a wrong SHA-256 checksum were rejected by the official client") + call("delete-object", "--bucket", "feint-byte-proof", "--key", "proof/data") + call("delete-bucket", "--bucket", "feint-byte-proof") + names, err := store.ListBuckets(t.Context()) + if err != nil || len(names) != 0 || guardHits.Load() != before { + t.Fatal("cleanup or no-external-proxy-request proof failed", names, err, guardHits.Load()-before) + } + mu.Lock() + defer mu.Unlock() + if len(exchanges) != 6 { + t.Fatal("expected five lifecycle exchanges plus one checksum refusal control", exchanges) + } + for i, code := range []int{200, 200, 200, 200, 204, 204} { + if exchanges[i].Status != code || !exchanges[i].SigV4 { + t.Fatal("unexpected status or unsigned client request", exchanges) + } + } + if exchanges[1].Encoding != "aws-chunked" || exchanges[1].DecodedLength != "73728" { + t.Fatal("HTTPS proof did not exercise the client's real streaming framing", exchanges[1]) + } + wire, err := json.Marshal(exchanges) + if err != nil { + t.Fatal(err) + } + t.Logf("exchanges=%s", wire) + t.Logf("bytes=%d sha256=%s; ETag absent and not required by this client; proxy guard delta=0 (not a network namespace)", len(got), hex.EncodeToString(sum[:])) +} + +type statusWriter struct { + http.ResponseWriter + status int + corrupt bool +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + if w.corrupt { + w.Header().Set("X-Amz-Checksum-Sha256", base64.StdEncoding.EncodeToString(make([]byte, sha256.Size))) + } + w.ResponseWriter.WriteHeader(status) +} + +func probeProxyGuard(ctx context.Context, endpoint string) error { + proxy, err := url.Parse(endpoint) + if err != nil { + return err + } + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxy)}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + r, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://s3.fr-par.scw.cloud/never-forward", nil) + if err != nil { + return err + } + response, err := client.Do(r) + if response != nil { + if closeErr := response.Body.Close(); closeErr != nil { + return closeErr + } + } + if err == nil { + return fmt.Errorf("guard unexpectedly permitted CONNECT") + } + return nil +} diff --git a/internal/s3/server.go b/internal/s3/server.go new file mode 100644 index 00000000..6b2d88ac --- /dev/null +++ b/internal/s3/server.go @@ -0,0 +1,380 @@ +// Package s3 adapts a neutral ObjectStore to a bounded, path-style S3 subset. +// +// It is a component, not a listener or a mounted provider pack. Callers retain +// ownership of the store. Only direct loopback requests are accepted. SigV4 +// requests are accepted without authenticating their secret; this is not IAM. +package s3 + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +// MissingBucketShape selects only the measured NoSuchBucket error envelope. +// It does not imply support for either provider's other S3 operations. +type MissingBucketShape uint8 + +const ( + // ScalewayMissingBucket follows the anonymous fr-par GET recording. + ScalewayMissingBucket MissingBucketShape = iota + 1 + // ExoscaleMissingBucket follows the anonymous ch-gva-2 GET recording. + ExoscaleMissingBucket +) + +// Config requires an explicit region, byte bound, measured error shape and log. +// MaxObjectBytes bounds each upload, not cumulative disk use. Callers also need +// HTTP read deadlines, header limits and a cumulative quota before exposure. +type Config struct { + Region string + MaxObjectBytes int64 + MissingBucket MissingBucketShape + Log *slog.Logger +} + +// Server is a direct HTTP handler. Do not put it behind a path-cleaning mux: +// repeated slashes, dot segments and escaped characters are logical key data. +type Server struct { + store *objectstore.Store + cfg Config +} + +// New validates explicit dependencies; it neither opens storage nor listens. +func New(store *objectstore.Store, cfg Config) (*Server, error) { + if store == nil || cfg.Log == nil || cfg.Region == "" || !headerValue(cfg.Region) { + return nil, errors.New("s3: store, logger and a header-safe region are required") + } + if cfg.MaxObjectBytes <= 0 || cfg.MaxObjectBytes > (math.MaxInt64-(64<<10))/2 { + return nil, errors.New("s3: a positive representable upload byte bound is required") + } + if cfg.MissingBucket != ScalewayMissingBucket && cfg.MissingBucket != ExoscaleMissingBucket { + return nil, errors.New("s3: select a measured missing-bucket response shape") + } + return &Server{store: store, cfg: cfg}, nil +} + +// ServeHTTP implements bucket PUT/HEAD/DELETE and object PUT/GET/HEAD/DELETE. +// Other features get an explicit Feint refusal, never a successful empty stub +// or an invented cloud XML error. Signatures and credentials are never logged. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + peer, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil || !peer.Addr().Unmap().IsLoopback() { + s.refuse(w, r, http.StatusForbidden, "only direct loopback peers are supported", err) + return + } + host := r.Host + if h, _, splitErr := net.SplitHostPort(host); splitErr == nil { + host = h + } + ip, ipErr := netip.ParseAddr(strings.Trim(host, "[]")) + if host != "localhost" && (ipErr != nil || !ip.Unmap().IsLoopback()) { + s.refuse(w, r, http.StatusNotImplemented, "only explicit loopback path-style endpoints are supported", nil) + return + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid query", err) + return + } + for key := range query { + if key != "x-id" { + s.refuse(w, r, http.StatusNotImplemented, "S3 query features are not implemented", nil) + return + } + } + if err := supportedHeaders(r); err != nil { + s.refuse(w, r, http.StatusNotImplemented, "S3 header feature is not implemented", err) + return + } + rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") + bucket, bucketErr := url.PathUnescape(rawBucket) + key, keyErr := url.PathUnescape(rawKey) + if bucketErr != nil || keyErr != nil || !bucketName(bucket) || !utf8.ValidString(key) || len(key) > 1024 { + s.refuse(w, r, http.StatusBadRequest, "invalid bucket or object key", errors.Join(bucketErr, keyErr)) + return + } + if key == "" { + s.bucket(w, r, bucket) + return + } + switch r.Method { + case http.MethodPut: + s.put(w, r, bucket, key) + case http.MethodGet, http.MethodHead: + s.get(w, r, bucket, key) + case http.MethodDelete: + err := s.store.DeleteObject(r.Context(), bucket, key) + if err != nil && !errors.Is(err, objectstore.ErrObjectNotFound) { + s.storeError(w, r, bucket, err) + return + } + w.WriteHeader(http.StatusNoContent) + default: + s.refuse(w, r, http.StatusNotImplemented, "S3 operation is not implemented", nil) + } +} + +func (s *Server) bucket(w http.ResponseWriter, r *http.Request, name string) { + var err error + status := http.StatusOK + switch r.Method { + case http.MethodPut: + // This slice accepts only the empty CreateBucket request. Region XML + // configuration must be measured before enabling that request shape. + body, readErr := io.ReadAll(http.MaxBytesReader(w, r.Body, 1)) + if readErr != nil || len(body) != 0 { + s.refuse(w, r, http.StatusNotImplemented, "CreateBucket configuration is not implemented", readErr) + return + } + err = s.store.CreateBucket(r.Context(), name) + case http.MethodHead: + err = s.store.HeadBucket(r.Context(), name) + case http.MethodDelete: + err = s.store.DeleteBucket(r.Context(), name) + status = http.StatusNoContent + case http.MethodGet: + err = s.store.HeadBucket(r.Context(), name) + if err == nil { + s.refuse(w, r, http.StatusNotImplemented, "bucket listing is not implemented", nil) + return + } + default: + s.refuse(w, r, http.StatusNotImplemented, "S3 operation is not implemented", nil) + return + } + if err != nil { + s.storeError(w, r, name, err) + return + } + w.Header().Set("X-Amz-Bucket-Region", s.cfg.Region) + w.WriteHeader(status) +} + +func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) { + metadata := make(map[string]string) + for name, values := range r.Header { + if suffix, ok := strings.CutPrefix(strings.ToLower(name), "x-amz-meta-"); ok { + if len(values) != 1 || !headerName(suffix) || !headerValue(values[0]) { + s.refuse(w, r, http.StatusBadRequest, "invalid object metadata header", nil) + return + } + metadata[suffix] = values[0] + } + } + if !headerValue(r.Header.Get("Content-Type")) { + s.refuse(w, r, http.StatusBadRequest, "invalid content type", nil) + return + } + upload, size, err := newUpload(w, r, s.cfg.MaxObjectBytes) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid S3 upload", err) + return + } + obj, err := s.store.PutObject(r.Context(), bucket, key, upload, objectstore.PutOptions{ + Size: size, ContentType: r.Header.Get("Content-Type"), Metadata: metadata, + }) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + // The SHA-256 is a verified byte checksum, deliberately not an ETag. + if !s.digestHeader(w, r, obj) { + return + } + w.Header().Set("X-Amz-Checksum-Crc32", upload.crc32()) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) get(w http.ResponseWriter, r *http.Request, bucket, key string) { + obj, body, err := s.store.GetObject(r.Context(), bucket, key) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + defer func() { + if err := body.Close(); err != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 object reader close failed", "error", err) + } + }() + if !headerValue(obj.ContentType) { + s.refuse(w, r, http.StatusInternalServerError, "stored metadata cannot be represented as HTTP headers", nil) + return + } + seen := make(map[string]bool) + for key, value := range obj.Metadata { + canonical := strings.ToLower(key) + if !headerName(key) || !headerValue(value) || seen[canonical] { + s.refuse(w, r, http.StatusInternalServerError, "stored metadata cannot be represented as HTTP headers", nil) + return + } + seen[canonical] = true + } + if !s.digestHeader(w, r, obj) { + return + } + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) + w.Header().Set("Content-Type", obj.ContentType) + for key, value := range obj.Metadata { + w.Header().Set("X-Amz-Meta-"+key, value) + } + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + if _, err := io.Copy(w, body); err != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 download failed after headers", "error", err) + } + } +} + +func (s *Server) digestHeader(w http.ResponseWriter, r *http.Request, obj objectstore.Object) bool { + digest, err := hex.DecodeString(obj.Digest) + if err != nil || len(digest) != 32 { + s.refuse(w, r, http.StatusInternalServerError, "stored byte checksum is invalid", err) + return false + } + w.Header().Set("X-Amz-Checksum-Sha256", base64.StdEncoding.EncodeToString(digest)) + return true +} + +func (s *Server) storeError(w http.ResponseWriter, r *http.Request, bucket string, err error) { + if errors.Is(err, objectstore.ErrBucketNotFound) { + s.missingBucket(w, r, bucket) + return + } + status, message := http.StatusInternalServerError, "object storage operation failed" + var tooLarge *http.MaxBytesError + switch { + case errors.Is(err, objectstore.ErrObjectNotFound), errors.Is(err, objectstore.ErrBucketExists), + errors.Is(err, objectstore.ErrBucketNotEmpty): + status, message = http.StatusNotImplemented, "cloud error shape has not been measured for this store outcome" + case errors.As(err, &tooLarge): + status, message = http.StatusRequestEntityTooLarge, "upload exceeds the encoded byte bound" + case errors.Is(err, errUpload), errors.Is(err, objectstore.ErrInvalid): + status, message = http.StatusBadRequest, "invalid S3 upload or metadata" + } + s.refuse(w, r, status, message, err) +} + +func (s *Server) refuse(w http.ResponseWriter, r *http.Request, status int, message string, err error) { + s.cfg.Log.WarnContext(r.Context(), "S3 request refused", "method", r.Method, "status", status, "reason", message, "error", err) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(status) + if r.Method != http.MethodHead { + if _, writeErr := fmt.Fprintln(w, "feint: "+message); writeErr != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 refusal response write failed", "error", writeErr) + } + } +} + +// These fields and their order follow the two anonymous GET controls from +// phase 0, not a synthetic XML fixture. No other cloud error is inferred. +type missingBucketError struct { + XMLName xml.Name `xml:"Error"` + Code string + Message string + RequestID string `xml:"RequestId,omitempty"` + HostID string `xml:"HostId,omitempty"` + Resource string `xml:",omitempty"` + BucketName string +} + +func (s *Server) missingBucket(w http.ResponseWriter, r *http.Request, bucket string) { + id, hostID := rand.Text(), rand.Text() + wire := missingBucketError{Code: "NoSuchBucket", Message: "The specified bucket does not exist", BucketName: bucket} + if s.cfg.MissingBucket == ScalewayMissingBucket { + hostID = id + wire.RequestID, wire.HostID, wire.Resource = id, hostID, "/"+bucket + } else { + wire.Message += "." + } + body, err := xml.Marshal(wire) + if err != nil { + s.refuse(w, r, http.StatusInternalServerError, "missing-bucket response encoding failed", err) + return + } + if s.cfg.MissingBucket == ExoscaleMissingBucket { + body = append([]byte(strings.TrimSuffix(xml.Header, "\n")), body...) + w.Header().Set("X-Amzn-Request-Id", id) + } + w.Header().Set("X-Amz-Request-Id", id) + w.Header().Set("X-Amz-Id-2", hostID) + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + if r.Method != http.MethodHead { + if _, err := w.Write(body); err != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 missing-bucket response write failed", "error", err) + } + } +} + +func bucketName(name string) bool { + if len(name) < 3 || len(name) > 63 || name[0] == '.' || name[0] == '-' || name[len(name)-1] == '.' || name[len(name)-1] == '-' { + return false + } + for _, c := range name { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '.' && c != '-' { + return false + } + } + return !strings.Contains(name, "..") +} + +func headerValue(value string) bool { + if !utf8.ValidString(value) { + return false + } + for _, c := range value { + if unicode.IsControl(c) { + return false + } + } + return true +} + +func headerName(name string) bool { + if name == "" { + return false + } + for _, c := range name { + if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && !strings.ContainsRune("!#$%&'*+-.^_`|~", c) { + return false + } + } + return true +} + +func supportedHeaders(r *http.Request) error { + for name := range r.Header { + lower := strings.ToLower(name) + switch lower { + case "range", "if-match", "if-none-match", "if-modified-since", "if-unmodified-since", "if-range", "content-md5": + return fmt.Errorf("unsupported header %s", name) + } + if !strings.HasPrefix(lower, "x-amz-") || strings.HasPrefix(lower, "x-amz-meta-") { + continue + } + switch lower { + case "x-amz-date", "x-amz-content-sha256", "x-amz-security-token", "x-amz-user-agent", + "x-amz-checksum-crc32", "x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm", + "x-amz-trailer", "x-amz-decoded-content-length", "x-amz-checksum-mode": + default: + return fmt.Errorf("unsupported header %s", name) + } + } + return nil +} diff --git a/internal/s3/server_test.go b/internal/s3/server_test.go new file mode 100644 index 00000000..71f6236a --- /dev/null +++ b/internal/s3/server_test.go @@ -0,0 +1,428 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "io" + "log/slog" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +func fixture(t *testing.T) (*Server, *objectstore.Store) { + t.Helper() + parent := t.TempDir() + if err := os.Chmod(parent, 0700); err != nil { + t.Fatal(err) + } + store, err := objectstore.Open(t.Context(), filepath.Join(parent, "objects")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }) + server, err := New(store, Config{Region: "fr-par", MaxObjectBytes: 1 << 20, + MissingBucket: ExoscaleMissingBucket, Log: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatal(err) + } + return server, store +} + +func request(t *testing.T, s *Server, method, path string, body []byte, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(method, "http://127.0.0.1:4599"+path, bytes.NewReader(body)) + r.RemoteAddr = "127.0.0.1:1234" + for key, value := range headers { + r.Header.Set(key, value) + } + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + return w +} + +func TestRoundTripPreservesLogicalKeysAndBytes(t *testing.T) { + s, store := fixture(t) + if w := request(t, s, "PUT", "/bucket", nil, nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + payload := []byte{0, 1, 255, 13, 10, 128} + for _, path := range []string{"/bucket/a/../b//c", "/bucket//root", "/bucket/%252F", "/bucket/a%2Fb", "/bucket/empty"} { + body := payload + if path == "/bucket/empty" { + body = nil + } + w := request(t, s, "PUT", path, body, map[string]string{"Content-Type": "application/octet-stream", "X-Amz-Meta-Purpose": "byte-proof"}) + if w.Code != 200 { + t.Fatal(path, w.Code, w.Body.String()) + } + w = request(t, s, "GET", path, nil, nil) + sum := sha256.Sum256(body) + if w.Code != 200 || !bytes.Equal(w.Body.Bytes(), body) || w.Header().Get("X-Amz-Checksum-Sha256") != base64.StdEncoding.EncodeToString(sum[:]) { + t.Fatalf("GET %q: status=%d body=%q headers=%v", path, w.Code, w.Body.Bytes(), w.Header()) + } + if w.Header().Get("ETag") != "" || w.Header().Get("X-Amz-Meta-Purpose") != "byte-proof" { + t.Fatal("invented ETag or lost metadata") + } + } + objects, err := store.ListObjects(t.Context(), "bucket") + if err != nil || len(objects) != 5 { + t.Fatal(objects, err) + } + for _, key := range []string{"a/../b//c", "/root", "%2F", "a/b", "empty"} { + if _, ok := objects[key]; !ok { + t.Fatal("key was cleaned or decoded twice", key) + } + } + if w := request(t, s, "HEAD", "/bucket/a%2Fb", nil, nil); w.Code != 200 || w.Body.Len() != 0 || w.Header().Get("Content-Length") != "6" { + t.Fatal(w.Code, w.Body.String(), w.Header()) + } + for range 2 { + if w := request(t, s, "DELETE", "/bucket/a%2Fb", nil, nil); w.Code != 204 { + t.Fatal(w.Code, w.Body.String()) + } + } +} + +func TestUploadChecksumsAndFramingBeforePublication(t *testing.T) { + // The 53-byte CRC32 trailer request is the actual phase-0 Terraform + // Scaleway wire payload, not a generated fixture matching the decoder. + framed := []byte("c\r\nfeint-proof\n\r\n0\r\nx-amz-checksum-crc32:2ZZ3Xg==\r\n\r\n") + base := map[string]string{ + "Content-Encoding": "aws-chunked", "X-Amz-Decoded-Content-Length": "12", + "X-Amz-Trailer": "x-amz-checksum-crc32", "X-Amz-Content-Sha256": "STREAMING-UNSIGNED-PAYLOAD-TRAILER", + "X-Amz-Sdk-Checksum-Algorithm": "CRC32", + } + tests := []struct { + name string + body []byte + edit func(map[string]string) + ok bool + }{ + {"recorded", framed, nil, true}, + {"checksum-mismatch", bytes.ReplaceAll(framed, []byte("2ZZ3Xg=="), []byte("AAAAAA==")), nil, false}, + {"short-decoded", framed, func(h map[string]string) { h["X-Amz-Decoded-Content-Length"] = "13" }, false}, + {"long-decoded", framed, func(h map[string]string) { h["X-Amz-Decoded-Content-Length"] = "11" }, false}, + {"missing-trailer", []byte("c\r\nfeint-proof\n\r\n0\r\n\r\n"), nil, false}, + {"extra-bytes", append(bytes.Clone(framed), []byte("junk\r\n\r\n")...), nil, false}, + {"truncated", framed[:len(framed)-2], nil, false}, + {"bare-lf-trailer", bytes.Replace(framed, []byte("2ZZ3Xg==\r\n\r\n"), []byte("2ZZ3Xg==\n\n"), 1), nil, false}, + {"undeclared-trailer", framed, func(h map[string]string) { delete(h, "X-Amz-Trailer"); delete(h, "X-Amz-Sdk-Checksum-Algorithm") }, false}, + {"unsupported-algorithm", framed, func(h map[string]string) { h["X-Amz-Sdk-Checksum-Algorithm"] = "CRC64NVME" }, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + headers := make(map[string]string) + for k, v := range base { + headers[k] = v + } + if tc.edit != nil { + tc.edit(headers) + } + w := request(t, s, "PUT", "/bucket/key", tc.body, headers) + obj, err := store.HeadObject(t.Context(), "bucket", "key") + if tc.ok { + if w.Code != 200 || err != nil || obj.Size != 12 || obj.Digest != "e56788086092c82c6af2ab449d9d26f920080381a55070fb1378fc7fedd04872" { + t.Fatal(w.Code, w.Body.String(), obj, err) + } + } else if w.Code < 400 || !errors.Is(err, objectstore.ErrObjectNotFound) { + t.Fatal("failed upload published metadata", w.Code, obj, err) + } + }) + } +} + +func TestPlainUploadChecksumsAndBound(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte("feint-proof\n")) + for _, tc := range []struct { + name, header, value string + body []byte + ok bool + }{ + {"crc-good", "X-Amz-Checksum-Crc32", "2ZZ3Xg==", []byte("feint-proof\n"), true}, + {"crc-bad", "X-Amz-Checksum-Crc32", "AAAAAA==", []byte("feint-proof\n"), false}, + {"sha-good", "X-Amz-Content-Sha256", hex.EncodeToString(sum[:]), []byte("feint-proof\n"), true}, + {"sha-bad", "X-Amz-Content-Sha256", strings.Repeat("0", 64), []byte("feint-proof\n"), false}, + {"oversized", "", "", bytes.Repeat([]byte("x"), (1<<20)+1), false}, + } { + w := request(t, s, "PUT", "/bucket/"+tc.name, tc.body, map[string]string{tc.header: tc.value}) + _, err := store.HeadObject(t.Context(), "bucket", tc.name) + if tc.ok { + if w.Code != 200 || err != nil { + t.Fatal(tc.name, w.Code, w.Body.String(), err) + } + } else if w.Code < 400 || !errors.Is(err, objectstore.ErrObjectNotFound) { + t.Fatal(tc.name, "invalid upload published", w.Code, err) + } + } +} + +func TestRefusalsDoNotMasqueradeAsCloudSupport(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + if w := request(t, s, "PUT", "/bucket/key", []byte("x"), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + for _, tc := range []struct { + method, path string + headers map[string]string + }{ + {"GET", "/bucket?list-type=2", nil}, + {"GET", "/bucket/missing", nil}, + {"PUT", "/bucket", nil}, + {"PUT", "/bucket/key?uploads", nil}, + {"PUT", "/bucket/key", map[string]string{"X-Amz-Acl": "public-read"}}, + {"GET", "/bucket/key", map[string]string{"Range": "bytes=0-1"}}, + {"PUT", "/bucket/key", map[string]string{"If-None-Match": "*"}}, + } { + w := request(t, s, tc.method, tc.path, nil, tc.headers) + if w.Code != 501 || !strings.HasPrefix(w.Body.String(), "feint: ") { + t.Fatal(tc, w.Code, w.Body.String()) + } + } + if w := request(t, s, "PUT", "/bucket/key", []byte("x"), nil); w.Code != 200 { + t.Fatal("accepting half", w.Code, w.Body.String()) + } + if w := request(t, s, "DELETE", "/bucket", nil, nil); w.Code != 501 { + t.Fatal(w.Code, w.Body.String()) + } + if w := request(t, s, "GET", "/bucket", nil, nil); w.Code != 501 { + t.Fatal("bucket listing became a successful empty stub", w.Code, w.Body.String()) + } + objects, err := store.ListObjects(t.Context(), "bucket") + if err != nil || len(objects) != 1 { + t.Fatal("refused request mutated storage", objects, err) + } +} + +func TestLoopbackAndPathStyleBoundary(t *testing.T) { + s, _ := fixture(t) + for _, tc := range []struct { + peer, host string + ok bool + }{ + {"127.0.0.1:1234", "127.0.0.1:4599", true}, + {"[::1]:1234", "[::1]:4599", true}, + {"127.0.0.1:1234", "localhost:4599", true}, + {"192.0.2.1:1234", "127.0.0.1:4599", false}, + {"", "127.0.0.1:4599", false}, + {"127.0.0.1:1234", "bucket.s3.fr-par.scw.cloud", false}, + } { + r := httptest.NewRequest("GET", "http://127.0.0.1/missing", nil) + r.RemoteAddr, r.Host = tc.peer, tc.host + r.Header.Set("X-Forwarded-For", "127.0.0.1") + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if tc.ok && w.Code != 404 || !tc.ok && w.Code != 403 && w.Code != 501 { + t.Fatal(tc, w.Code, w.Body.String()) + } + } +} + +func TestPersistedMetadataIsRevalidatedForHTTP(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + for _, metadata := range []map[string]string{{"purpose": "valid"}, {"purpose": "x\r\nInjected: yes"}, {"bad key": "x"}, {"purpose": "x", "PURPOSE": "y"}} { + _, err := store.PutObject(t.Context(), "bucket", "key", strings.NewReader("x"), + objectstore.PutOptions{Size: 1, ContentType: "application/octet-stream", Metadata: metadata}) + if err != nil { + t.Fatal(err) + } + + w := request(t, s, "GET", "/bucket/key", nil, nil) + if metadata["purpose"] == "valid" { + if w.Code != 200 || w.Body.String() != "x" { + t.Fatal(w.Code, w.Body.String()) + } + } else if w.Code != 500 || w.Header().Get("X-Amz-Meta-Purpose") != "" { + t.Fatal("untrusted persisted headers served", w.Code, w.Header()) + } + } +} + +func TestUploadHeaderValidationAndConstructor(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte("x")) + for _, headers := range []map[string]string{ + {"X-Amz-Meta-Purpose": "valid", "Content-Type": "application/octet-stream"}, + {"X-Amz-Meta-Purpose": "x\ninjected"}, + {"X-Amz-Meta-Bad Key": "x"}, + {"Content-Type": "x\r\ninjected"}, + {"X-Amz-Decoded-Content-Length": "12"}, + {"X-Amz-Checksum-Sha256": "not-base64"}, + {"X-Amz-Sdk-Checksum-Algorithm": "CRC32"}, + {"X-Amz-Content-Sha256": hex.EncodeToString(sum[:]), "X-Amz-Checksum-Sha256": base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{1}, 32))}, + } { + w := request(t, s, "PUT", "/bucket/key", []byte("x"), headers) + valid := headers["X-Amz-Meta-Purpose"] == "valid" + if valid && w.Code != 200 || !valid && w.Code < 400 { + t.Fatal(headers, w.Code, w.Body.String()) + } + if _, err := store.HeadObject(t.Context(), "bucket", "key"); err != nil { + t.Fatal("invalid overwrite destroyed the previously committed object", err) + } + } + for _, change := range []func(*Config){ + func(c *Config) { c.MaxObjectBytes = 0 }, + func(c *Config) { c.MaxObjectBytes = 1<<63 - 1 }, + func(c *Config) { c.Region = "x\r\nInjected" }, + func(c *Config) { c.MissingBucket = 0 }, + func(c *Config) { c.Log = nil }, + } { + cfg := s.cfg + change(&cfg) + if _, err := New(store, cfg); err == nil { + t.Fatal("invalid constructor succeeded", cfg) + } + } + if _, err := New(nil, s.cfg); err == nil { + t.Fatal("nil store accepted") + } +} + +func TestMissingBucketFollowsMeasuredErrorDialects(t *testing.T) { + s, _ := fixture(t) + for _, shape := range []MissingBucketShape{ScalewayMissingBucket, ExoscaleMissingBucket} { + file := "exoscale-no-such-bucket.xml" + if shape == ScalewayMissingBucket { + file = "scaleway-no-such-bucket.xml" + } + // Real anonymous HTTPS GET responses recorded 2026-11-06T20:33:53Z + // at s3.fr-par.scw.cloud and sos-ch-gva-2.exo.io. Response identifiers + // are redacted and a trailing newline added; fields remain recorded. + recording, err := os.ReadFile(filepath.Join("testdata", file)) + if err != nil { + t.Fatal(err) + } + var observed missingBucketError + if err := xml.Unmarshal(recording, &observed); err != nil { + t.Fatal("cannot decode real producer response", err) + } + if observed.Code != "NoSuchBucket" || observed.BucketName == "" { + t.Fatal("recording did not decode into the boundary type", observed) + } + s.cfg.MissingBucket = shape + w := request(t, s, "GET", "/missing", nil, nil) + body := w.Body.String() + var emitted missingBucketError + if err := xml.Unmarshal(w.Body.Bytes(), &emitted); err != nil { + t.Fatal(err) + } + observed.BucketName = emitted.BucketName + if shape == ScalewayMissingBucket { + observed.RequestID, observed.HostID, observed.Resource = emitted.RequestID, emitted.HostID, "/missing" + if emitted.RequestID != emitted.HostID { + t.Fatal("recorded Scaleway identifiers were equal", emitted) + } + } + want, err := xml.Marshal(observed) + if err != nil { + t.Fatal(err) + } + if shape == ExoscaleMissingBucket { + want = append([]byte(strings.TrimSuffix(xml.Header, "\n")), want...) + } + if body != string(want) { + t.Fatal("response differs from the actual producer envelope", body, string(want)) + } + if w.Code != 404 || w.Header().Get("Content-Type") != "application/xml" || + !strings.Contains(body, "NoSuchBucket") || !strings.Contains(body, "missing") { + t.Fatal(w.Code, body, w.Header()) + } + if shape == ScalewayMissingBucket { + if strings.HasPrefix(body, "") || + !strings.Contains(body, "/missing") || + !strings.Contains(body, "The specified bucket does not exist") { + t.Fatal(body) + } + } else if !strings.HasPrefix(body, "") || + !strings.Contains(body, "The specified bucket does not exist.") { + t.Fatal(body) + } + } +} + +func FuzzUploadFraming(f *testing.F) { + f.Add([]byte("1\r\nx\r\n0\r\n\r\n")) + f.Add([]byte("0\r\n\r\n")) + f.Add([]byte("not a chunk")) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 4096 { + return + } + r := httptest.NewRequest("PUT", "http://127.0.0.1/bucket/key", bytes.NewReader(data)) + r.Header.Set("Content-Encoding", "aws-chunked") + r.Header.Set("X-Amz-Decoded-Content-Length", "1") + u, _, err := newUpload(httptest.NewRecorder(), r, 4096) + if err != nil { + t.Fatal(err) + } + _, _ = io.Copy(io.Discard, u) + }) +} + +func TestCanceledUploadDoesNotPublish(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + r := httptest.NewRequest("PUT", "http://127.0.0.1/bucket/key", strings.NewReader("x")).WithContext(ctx) + r.RemoteAddr = "127.0.0.1:1234" + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + _, err := store.HeadObject(t.Context(), "bucket", "key") + if w.Code < 400 || !errors.Is(err, objectstore.ErrObjectNotFound) { + t.Fatal(w.Code, err) + } +} + +func TestAmbiguousUploadHeadersDoNotPublish(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte("x")) + for _, duplicate := range []bool{false, true} { + r := httptest.NewRequest("PUT", "http://127.0.0.1/bucket/key", strings.NewReader("x")) + r.RemoteAddr = "127.0.0.1:1234" + r.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:])) + if duplicate { + r.Header.Add("X-Amz-Content-Sha256", strings.Repeat("0", 64)) + } + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if !duplicate && w.Code != 200 || duplicate && w.Code != 400 { + t.Fatal("ambiguous headers were silently ignored", duplicate, w.Code, w.Body.String()) + } + } +} diff --git a/internal/s3/testdata/exoscale-no-such-bucket.xml b/internal/s3/testdata/exoscale-no-such-bucket.xml new file mode 100644 index 00000000..1e3bdcbe --- /dev/null +++ b/internal/s3/testdata/exoscale-no-such-bucket.xml @@ -0,0 +1 @@ +NoSuchBucketThe specified bucket does not exist.feint-phase0-d45ae1c38d934520a2bf0a08bf2e681f diff --git a/internal/s3/testdata/scaleway-no-such-bucket.xml b/internal/s3/testdata/scaleway-no-such-bucket.xml new file mode 100644 index 00000000..ade73553 --- /dev/null +++ b/internal/s3/testdata/scaleway-no-such-bucket.xml @@ -0,0 +1 @@ +NoSuchBucketThe specified bucket does not existREDACTED_RESPONSE_IDREDACTED_RESPONSE_ID/feint-phase0-9b6bd212d7e34ceaa10827dc6e9b74cafeint-phase0-9b6bd212d7e34ceaa10827dc6e9b74ca diff --git a/internal/s3/upload.go b/internal/s3/upload.go new file mode 100644 index 00000000..b3ae7a5b --- /dev/null +++ b/internal/s3/upload.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" + "net/http/httputil" + "net/textproto" + "strconv" + "strings" +) + +var errUpload = errors.New("invalid upload") + +// Validation must finish before EOF reaches ObjectStore: a checksum failure +// returned after PutObject would be too late to prevent metadata publication. +type uploadReader struct { + body io.Reader + framing *bufio.Reader + crc hash.Hash32 + sha hash.Hash + expected map[string]string + trailers map[string]bool + done bool +} + +func newUpload(w http.ResponseWriter, r *http.Request, max int64) (*uploadReader, int64, error) { + for _, name := range []string{"Content-Type", "Content-Encoding", "X-Amz-Content-Sha256", + "X-Amz-Decoded-Content-Length", "X-Amz-Trailer", "X-Amz-Sdk-Checksum-Algorithm"} { + if len(r.Header.Values(name)) > 1 { + return nil, 0, fmt.Errorf("%w: ambiguous upload header %s", errUpload, name) + } + } + size := r.ContentLength + encoding := r.Header.Get("Content-Encoding") + chunked := encoding == "aws-chunked" + if encoding != "" && !chunked { + return nil, 0, fmt.Errorf("%w: unsupported content encoding", errUpload) + } + if !chunked && r.Header.Get("X-Amz-Decoded-Content-Length") != "" { + return nil, 0, fmt.Errorf("%w: decoded size requires aws-chunked encoding", errUpload) + } + if chunked { + var err error + size, err = strconv.ParseInt(r.Header.Get("X-Amz-Decoded-Content-Length"), 10, 64) + if err != nil { + return nil, 0, fmt.Errorf("%w: exact decoded content length is required", errUpload) + } + } + if size < 0 || size > max { + return nil, 0, fmt.Errorf("%w: decoded upload exceeds the byte bound or has no exact size", errUpload) + } + u := &uploadReader{ + body: http.MaxBytesReader(w, r.Body, max*2+(64<<10)), + crc: crc32.NewIEEE(), sha: sha256.New(), + expected: make(map[string]string), trailers: make(map[string]bool), + } + for _, name := range []string{"X-Amz-Checksum-Crc32", "X-Amz-Checksum-Sha256"} { + if values := r.Header.Values(name); len(values) != 0 { + if len(values) != 1 || !checksumValue(name, values[0]) { + return nil, 0, fmt.Errorf("%w: invalid checksum header", errUpload) + } + u.expected[name] = values[0] + } + } + payload := r.Header.Get("X-Amz-Content-Sha256") + switch payload { + case "", "UNSIGNED-PAYLOAD": + case "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": + if !chunked { + return nil, 0, fmt.Errorf("%w: streaming payload requires aws-chunked encoding", errUpload) + } + default: + digest, err := hex.DecodeString(payload) + if err != nil || len(digest) != sha256.Size || chunked { + return nil, 0, fmt.Errorf("%w: invalid payload hash", errUpload) + } + value := base64.StdEncoding.EncodeToString(digest) + if want := u.expected["X-Amz-Checksum-Sha256"]; want != "" && want != value { + return nil, 0, fmt.Errorf("%w: conflicting SHA256 checksum headers", errUpload) + } + u.expected["X-Amz-Checksum-Sha256"] = value + } + algorithm := r.Header.Get("X-Amz-Sdk-Checksum-Algorithm") + if algorithm != "" && algorithm != "CRC32" && algorithm != "SHA256" { + return nil, 0, fmt.Errorf("%w: only CRC32 and SHA256 checksums are implemented", errUpload) + } + if declared := r.Header.Get("X-Amz-Trailer"); declared != "" { + if !chunked { + return nil, 0, fmt.Errorf("%w: checksum trailers require aws-chunked encoding", errUpload) + } + for _, raw := range strings.Split(declared, ",") { + name := textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(raw)) + if (name != "X-Amz-Checksum-Crc32" && name != "X-Amz-Checksum-Sha256") || u.trailers[name] || u.expected[name] != "" { + return nil, 0, fmt.Errorf("%w: unsupported or ambiguous checksum trailer", errUpload) + } + u.trailers[name] = true + } + } + if algorithm != "" { + name := "X-Amz-Checksum-Crc32" + if algorithm == "SHA256" { + name = "X-Amz-Checksum-Sha256" + } + if u.expected[name] == "" && !u.trailers[name] { + return nil, 0, fmt.Errorf("%w: declared checksum algorithm has no checksum", errUpload) + } + } + if chunked { + u.framing = bufio.NewReader(u.body) + u.body = httputil.NewChunkedReader(u.framing) + } + return u, size, nil +} + +func (u *uploadReader) Read(p []byte) (int, error) { + if u.done { + return 0, io.EOF + } + n, err := u.body.Read(p) + // The standard hash implementations cannot fail their Write. + if _, hashErr := u.crc.Write(p[:n]); hashErr != nil { + return n, hashErr + } + if _, hashErr := u.sha.Write(p[:n]); hashErr != nil { + return n, hashErr + } + if err != nil && !errors.Is(err, io.EOF) { + return n, fmt.Errorf("%w: decode payload: %w", errUpload, err) + } + if errors.Is(err, io.EOF) { + u.done = true + if err := u.finish(); err != nil { + return n, err + } + } + return n, err +} + +func (u *uploadReader) finish() error { + if u.framing != nil { + tail, err := io.ReadAll(io.LimitReader(u.framing, (8<<10)+1)) + if err != nil { + return fmt.Errorf("%w: read checksum trailers: %w", errUpload, err) + } + if len(tail) > 8<<10 || (string(tail) != "\r\n" && !bytes.HasSuffix(tail, []byte("\r\n\r\n"))) { + return fmt.Errorf("%w: oversized or incomplete checksum trailers", errUpload) + } + reader := bufio.NewReader(bytes.NewReader(tail)) + headers, err := textproto.NewReader(reader).ReadMIMEHeader() + if err != nil || reader.Buffered() != 0 || len(headers) != len(u.trailers) { + return fmt.Errorf("%w: invalid checksum trailers", errUpload) + } + for name, values := range headers { + if !u.trailers[name] || len(values) != 1 || !checksumValue(name, values[0]) { + return fmt.Errorf("%w: undeclared or invalid checksum trailer", errUpload) + } + u.expected[name] = values[0] + } + } + for name, want := range u.expected { + got := u.crc32() + if name == "X-Amz-Checksum-Sha256" { + got = base64.StdEncoding.EncodeToString(u.sha.Sum(nil)) + } + if got != want { + return fmt.Errorf("%w: checksum mismatch", errUpload) + } + } + return nil +} + +func (u *uploadReader) crc32() string { + var sum [4]byte + binary.BigEndian.PutUint32(sum[:], u.crc.Sum32()) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func checksumValue(name, value string) bool { + sum, err := base64.StdEncoding.DecodeString(value) + size := 4 + if name == "X-Amz-Checksum-Sha256" { + size = sha256.Size + } + return err == nil && len(sum) == size +} diff --git a/mise.toml b/mise.toml index 1c5896bd..487c89f6 100644 --- a/mise.toml +++ b/mise.toml @@ -50,6 +50,10 @@ run = "./feint serve --addr $FEINT_ADDR --vm ${FEINT_VM:-off}" description = "Run the unit tests with the race detector" run = "go test -race ./..." +[tasks."conformance:s3"] +description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 (required)" +run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" + [tasks.fuzz] description = "Fuzz the request decoders. Longer run: FUZZTIME=5m mise run fuzz" run = "go test ./internal/providers/scaleway/ -run '^$' -fuzz 'FuzzCreateServer' -fuzztime=\"${FUZZTIME:-30s}\"" diff --git a/tools/falsify/specs/s3-gateway.json b/tools/falsify/specs/s3-gateway.json new file mode 100644 index 00000000..23bd4016 --- /dev/null +++ b/tools/falsify/specs/s3-gateway.json @@ -0,0 +1,110 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "transport checksum mismatches publish metadata", + "file": "internal/s3/upload.go", + "find": "if got != want {", + "replace": "if got != want && false {", + "test": "TestUploadChecksumsAndFramingBeforePublication|TestPlainUploadChecksumsAndBound" + }, + { + "label": "the decoded upload byte bound is disabled", + "file": "internal/s3/upload.go", + "find": "if size < 0 || size > max {", + "replace": "if size < 0 || (size > max && false) {", + "test": "TestPlainUploadChecksumsAndBound" + }, + { + "label": "extra bytes after the inner chunk terminator are ignored", + "file": "internal/s3/upload.go", + "find": "if err != nil || reader.Buffered() != 0 || len(headers) != len(u.trailers) {", + "replace": "if err != nil || (reader.Buffered() != 0 && false) || len(headers) != len(u.trailers) {", + "test": "TestUploadChecksumsAndFramingBeforePublication/extra-bytes" + }, + { + "label": "missing declared checksum trailers are ignored", + "file": "internal/s3/upload.go", + "find": "if err != nil || reader.Buffered() != 0 || len(headers) != len(u.trailers) {", + "replace": "if err != nil || reader.Buffered() != 0 || (len(headers) != len(u.trailers) && false) {", + "test": "TestUploadChecksumsAndFramingBeforePublication/missing-trailer" + }, + { + "label": "incomplete checksum trailer framing is accepted", + "file": "internal/s3/upload.go", + "find": "if len(tail) > 8<<10 || (string(tail) != \"\\r\\n\" && !bytes.HasSuffix(tail, []byte(\"\\r\\n\\r\\n\"))) {", + "replace": "if len(tail) > 8<<10 || (string(tail) != \"\\r\\n\" && !bytes.HasSuffix(tail, []byte(\"\\r\\n\\r\\n\")) && false) {", + "test": "TestUploadChecksumsAndFramingBeforePublication/bare-lf-trailer" + }, + { + "label": "the HTTP peer need not be loopback", + "file": "internal/s3/server.go", + "find": "if err != nil || !peer.Addr().Unmap().IsLoopback() {", + "replace": "if err != nil || (!peer.Addr().Unmap().IsLoopback() && false) {", + "test": "TestLoopbackAndPathStyleBoundary" + }, + { + "label": "virtual cloud hostnames are treated as path-style endpoints", + "file": "internal/s3/server.go", + "find": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) {", + "replace": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) && false {", + "test": "TestLoopbackAndPathStyleBoundary" + }, + { + "label": "S3 query features silently become ordinary CRUD", + "file": "internal/s3/server.go", + "find": "if key != \"x-id\" {", + "replace": "if key != \"x-id\" && false {", + "test": "TestRefusalsDoNotMasqueradeAsCloudSupport" + }, + { + "label": "range requests silently return full bytes", + "file": "internal/s3/server.go", + "find": "case \"range\", \"if-match\", \"if-none-match\", \"if-modified-since\", \"if-unmodified-since\", \"if-range\", \"content-md5\":", + "replace": "case \"\\x00never\", \"if-match\", \"if-none-match\", \"if-modified-since\", \"if-unmodified-since\", \"if-range\", \"content-md5\":", + "test": "TestRefusalsDoNotMasqueradeAsCloudSupport" + }, + { + "label": "unsafe restored metadata is emitted as HTTP headers", + "file": "internal/s3/server.go", + "find": "if !headerName(key) || !headerValue(value) || seen[canonical] {", + "replace": "if (!headerName(key) || !headerValue(value) || seen[canonical]) && false {", + "test": "TestPersistedMetadataIsRevalidatedForHTTP" + }, + { + "label": "invalid upload metadata is accepted", + "file": "internal/s3/server.go", + "find": "if len(values) != 1 || !headerName(suffix) || !headerValue(values[0]) {", + "replace": "if (len(values) != 1 || !headerName(suffix) || !headerValue(values[0])) && false {", + "test": "TestUploadHeaderValidationAndConstructor" + }, + { + "label": "conflicting payload and explicit SHA256 headers are ignored", + "file": "internal/s3/upload.go", + "find": "if want := u.expected[\"X-Amz-Checksum-Sha256\"]; want != \"\" && want != value {", + "replace": "if want := u.expected[\"X-Amz-Checksum-Sha256\"]; want != \"\" && want != value && false {", + "test": "TestUploadHeaderValidationAndConstructor" + }, + { + "label": "the constructor admits an overflowing raw byte bound", + "file": "internal/s3/server.go", + "find": "if cfg.MaxObjectBytes <= 0 || cfg.MaxObjectBytes > (math.MaxInt64-(64<<10))/2 {", + "replace": "if cfg.MaxObjectBytes <= 0 || (cfg.MaxObjectBytes > (math.MaxInt64-(64<<10))/2 && false) {", + "test": "TestUploadHeaderValidationAndConstructor" + }, + { + "label": "response checksums no longer describe the stored bytes", + "file": "internal/s3/server.go", + "find": "base64.StdEncoding.EncodeToString(digest)", + "replace": "base64.StdEncoding.EncodeToString(append([]byte{digest[0] ^ 1}, digest[1:]...))", + "test": "TestRoundTripPreservesLogicalKeysAndBytes" + }, + { + "label": "duplicate upload headers silently select the first value", + "file": "internal/s3/upload.go", + "find": "if len(r.Header.Values(name)) > 1 {", + "replace": "if len(r.Header.Values(name)) > 1 && false {", + "test": "TestAmbiguousUploadHeadersDoNotPublish" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 68d23c2a..6c28c5b8 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -236,9 +236,17 @@ var rules = []rule{ }, { Path: "internal/core/objectstore/", - Why: "neutral persisted bytes, not yet connected to a served protocol", - Unproven: "the package tests prove byte round trips and reopen, not S3 responses, " + - "real-client conformance, checkpoint/reset, power-loss recovery or network filesystem locking", + Why: "neutral persisted bytes used by the unmounted S3 component", + Runs: []string{"conformance:s3"}, + Unproven: "the component proof does not mount S3 in the emulator or run provider clients, " + + "checkpoint/reset, power-loss recovery or network filesystem locking", + }, + { + Path: "internal/s3/", + Why: "bounded path-style S3 protocol and real-byte official-client proof", + Runs: []string{"conformance:s3"}, + Unproven: "the explicit component proof covers one AWS CLI version, not provider doors, " + + "virtual hosts, interception, ETag semantics, multipart, public stacks or cumulative quotas", }, // ------------------------------------------------------ what describes it From f129a0f23cd8ddc7446bd98ab3836a87ab4e94cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 6 Nov 2026 23:26:55 +0100 Subject: [PATCH 03/28] feat(objectstore): bound retained bytes and concurrent uploads Add an explicit cumulative blob-byte budget with startup inode inventory, atomic upload reservations and retained-byte accounting after publication or cleanup failures. Keep the neutral format and unlimited Open API. Probe excess input without writing beyond the reserved size; report quota refusals as Feint HTTP 507. Record the accepted same-port S3 door ADR and selected 1 GiB cumulative / 256 MiB simple-PUT deployment limits. Server mounting remains a subsequent property. Validation: repository check, docs check and bounded official AWS CLI HTTPS proof passed. Ten quota mutations, sixteen neutral-byte mutations and fifteen gateway mutations compiled and bit, with restored baselines green. Upstream Scaleway drift remains unrelated and untriaged. Refs: #850 Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/README.md | 2 + docs/architecture.md | 16 ++ docs/object-storage-routing.md | 92 ++++++ internal/core/objectstore/budget.go | 115 ++++++++ internal/core/objectstore/budget_test.go | 295 ++++++++++++++++++++ internal/core/objectstore/files.go | 34 ++- internal/core/objectstore/store.go | 1 + internal/s3/server.go | 2 + internal/s3/server_test.go | 25 +- tools/falsify/specs/objectstore-budget.json | 76 +++++ 10 files changed, 655 insertions(+), 3 deletions(-) create mode 100644 docs/object-storage-routing.md create mode 100644 internal/core/objectstore/budget.go create mode 100644 internal/core/objectstore/budget_test.go create mode 100644 tools/falsify/specs/objectstore-budget.json diff --git a/docs/README.md b/docs/README.md index b336f160..686bdd43 100644 --- a/docs/README.md +++ b/docs/README.md @@ -55,6 +55,8 @@ small enough to read, and no cloud account. - [architecture.md](architecture.md) — the packs, the neutral core, and the boundary between them. +- [object-storage-routing.md](object-storage-routing.md) — the accepted + single-port S3 door decision and its byte budgets; implementation is staged. - [../CONTRIBUTING.md](../CONTRIBUTING.md) — how a change is proved before it is proposed. - [proxy.md](proxy.md) — record a real cloud, read the recording, and turn it diff --git a/docs/architecture.md b/docs/architecture.md index 5cd5c7d7..3b16bcbf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,6 +113,22 @@ The caller must impose storage quotas and unblock its input reader on cancellation; waiting for an existing same-key transaction may also delay cancellation. No structured format passes through a text template. +`OpenWithLimit` adds an explicit cumulative blob-byte budget without changing +the index format or `Open`'s unlimited library behavior. It counts retained and +orphan data on reopen, with hard-linked names counted once, and reserves each +upload's declared size before file creation or input reads. The EOF probe is +read into memory, never written beyond that reservation. Reservations are +released after temporary-file cleanup; new published blobs remain charged even +when metadata publication fails. Failed cleanup retains the partial byte charge. +Deletion does not free blob bytes. Identical uploads still require temporary +space. Metadata and filesystem overhead are outside this byte budget. +See the [accepted door ADR](object-storage-routing.md) for the selected opt-in +defaults (1 GiB cumulative, 256 MiB per simple PUT) and the still-unmounted +server integration. +The gateway renders a budget refusal as a logged Feint HTTP 507 text response, +not an unmeasured S3 XML error. The official-client component proof uses a small +explicit cumulative budget; deployment defaults are not yet CLI options. + ### Explicit S3 component, not shipped Object Storage support [s3](../internal/s3/server.go) is a direct HTTP handler over that store, with no diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md new file mode 100644 index 00000000..4a665a22 --- /dev/null +++ b/docs/object-storage-routing.md @@ -0,0 +1,92 @@ +# ADR: explicit Object Storage doors on the existing port + +Status: **accepted**, 2026-11-06, for #850. Implementation is incremental; this +decision is not a claim of shipped Object Storage support. + +## Context + +The neutral ObjectStore persists verified bytes. The unmounted S3 gateway has +an official AWS CLI HTTPS round-trip proof. Neither gives `feint serve` an S3 +endpoint yet. Raw path-style S3 (`/bucket/key`) can collide with control-plane +paths; `http.ServeMux` also cleans dot segments and repeated slashes, which are +legitimate logical object keys. + +The existing multi-provider server, browser guards and SDK-surface drift checks +must remain effective. S3 operations must not acquire fabricated provider SDK +operation names merely to appear in the control-plane coverage table. + +## Decision + +Keep one listening port. Opt-in S3 doors use explicit endpoint prefixes: + +```text +http://127.0.0.1:4599/_feint/s3/scaleway/fr-par/ +http://127.0.0.1:4599/_feint/s3/exoscale/ch-gva-2/ +``` + +Dispatch a declared S3 prefix before the control-plane mux. Remove only that +prefix, preserving the encoded path and decoding bucket/key once in the S3 +gateway. Never clean the remainder or redirect a key containing dot segments. +Unknown doors are refused, not treated as a default provider or region. +The outer Host/Origin browser guard stays in place. S3 remains loopback-only; +network exposure is not implied by the control-plane exposure option. + +Use one private neutral ObjectStore. Provider/region separation is a declared +gateway namespace, not a provider name embedded in `internal/core`, and never +a bucket/key-derived filesystem path. Provider-specific XML belongs at the +protocol boundary. Declare enabled doors at startup rather than infer a tenant +from an unverified signature. This provides namespace separation, **not IAM**. + +Activation and persistence must be explicit. Wire storage ownership through +serve/start/restart shutdown, configured endpoints through env, and accurate +diagnostics through status/doctor. S3 traffic needs its own honest operation +evidence; do not silently count it as an official control-plane SDK route. + +The selected initial limits, configurable at startup, are: + +- **1 GiB** cumulative blob bytes per store, shared across doors. +- **256 MiB** per decoded simple PUT. + +Count retained blobs, crash leftovers and in-flight upload reservations. +Deleting an object or bucket removes metadata references, not its byte charge. +An upload reserves its declared byte count before reading or creating a file. +It cannot write an extra probe byte beyond that reservation. Distinct uploads +reserve atomically; identical uploads still need temporary space before their +digest is known. Reopen recounts retained bytes, counting hard-linked names of +one file once. No metadata format change or persisted quota counter is needed. +The budget excludes metadata, filesystem block/inode overhead and other files +outside the blob directory; it is not an OS filesystem quota. + +Refuse over-budget uploads without changing existing metadata or deleting +existing data. Reads and metadata deletion remain available at the limit. +Opening existing data over the selected limit fails explicitly; increasing the +configured budget is non-destructive. Do not add automatic garbage collection. + +## Alternatives + +- A second listener would weaken the single-port product contract and require + another endpoint/lifecycle surface. +- S3 hostnames under `.localhost` avoid path collisions but require proving + name resolution, virtual-host addressing and the corresponding client + behavior. They are not needed for the first configurable endpoint. +- Root-level S3 fallback makes URL ownership ambiguous and risks mux redirects. +- Independent provider storage engines duplicate byte, integrity and quota + behavior. Provider-specific wire envelopes do not justify that duplication. + +## Acceptance and deferred work + +Prove declared door dispatch, exact key preservation, namespace separation, +browser refusal, budget thresholds and concurrent reservations. Falsify guards +in an external copy. Drive complete `scw`/`exo` configurable-client lifecycles +with canonical fake credentials in isolated configurations; verify bytes after +restart of the real emulator. Implement ancillary operations from measured +client requests and trustworthy response shapes, not successful empty stubs. + +The configured-client door must be proved against clients that actually support +the chosen endpoint prefix. If a client rejects it, revisit this decision rather +than quietly add a second port or change its workload. + +Virtual hosts, trusted intercepted-host handoff, hardcoded Scaleway Terraform +endpoints, checkpoint/reset, garbage collection and public-stack certification +remain later work. Existing Object Storage support limits stay unchanged until +the mounted-client proofs exist. diff --git a/internal/core/objectstore/budget.go b/internal/core/objectstore/budget.go new file mode 100644 index 00000000..75fb2dba --- /dev/null +++ b/internal/core/objectstore/budget.go @@ -0,0 +1,115 @@ +package objectstore + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "syscall" +) + +// ErrQuota distinguishes a byte-budget refusal from corrupt or invalid data. +var ErrQuota = errors.New("object store blob byte budget exceeded") + +type blobBudget struct { + mu sync.Mutex + limit int64 + used int64 + held int64 +} + +// OpenWithLimit opens the same private format as Open with a positive cumulative +// blob-byte budget. Retained and orphan files count; hard links count once. +// Every upload reserves its full declared size before writing, even a duplicate. +// The limit is a runtime policy, not persisted metadata or a filesystem quota: +// metadata and filesystem overhead are excluded. Existing over-limit stores are +// refused without deleting data. Open retains its unlimited library behavior. +func OpenWithLimit(ctx context.Context, path string, maxBlobBytes int64) (*Store, error) { + if maxBlobBytes <= 0 { + return nil, fmt.Errorf("%w: blob byte budget must be positive", ErrInvalid) + } + s, err := Open(ctx, path) + if err != nil { + return nil, err + } + used, err := s.retainedBytes(ctx, maxBlobBytes) + if err != nil { + return nil, errors.Join(err, s.Close()) + } + s.budget = &blobBudget{limit: maxBlobBytes, used: used} + return s, nil +} + +func (s *Store) retainedBytes(ctx context.Context, limit int64) (used int64, err error) { + dir, err := s.blobs.Open(".") + if err != nil { + return 0, fmt.Errorf("open blobs for budget inventory: %w", err) + } + defer func() { err = errors.Join(err, dir.Close()) }() + seen := make(map[string]bool) + for { + if err := ctx.Err(); err != nil { + return 0, err + } + entries, readErr := dir.ReadDir(128) + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return 0, err + } + file, err := openPrivate(s.blobs, entry.Name()) + if err != nil { + return 0, fmt.Errorf("inspect retained blob: %w", errors.Join(ErrCorrupt, err)) + } + info, statErr := file.Stat() + closeErr := file.Close() + if err := errors.Join(statErr, closeErr); err != nil { + return 0, fmt.Errorf("stat retained blob: %w", err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("%w: retained blob has no file identity", ErrCorrupt) + } + identity := fmt.Sprintf("%v:%v", stat.Dev, stat.Ino) + if seen[identity] { + continue + } + seen[identity] = true + if info.Size() < 0 || info.Size() > limit-used { + return 0, ErrQuota + } + used += info.Size() + } + if readErr != nil { + if !errors.Is(readErr, io.EOF) { + return 0, fmt.Errorf("read blob budget inventory: %w", readErr) + } + return used, nil + } + } +} + +func (s *Store) reserveBlob(size int64) error { + if s.budget == nil { + return nil + } + b := s.budget + b.mu.Lock() + defer b.mu.Unlock() + if size > b.limit-b.used-b.held { + return ErrQuota + } + b.held += size + return nil +} + +func (s *Store) settleBlob(reserved, retained int64) { + if s.budget == nil { + return + } + b := s.budget + b.mu.Lock() + defer b.mu.Unlock() + b.held -= reserved + b.used += retained +} diff --git a/internal/core/objectstore/budget_test.go b/internal/core/objectstore/budget_test.go new file mode 100644 index 00000000..56fd3361 --- /dev/null +++ b/internal/core/objectstore/budget_test.go @@ -0,0 +1,295 @@ +package objectstore + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func limitedStore(t *testing.T, limit int64) (*Store, string) { + t.Helper() + path := filepath.Join(privateDir(t), "objects") + s, err := OpenWithLimit(t.Context(), path, limit) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }) + return s, path +} + +func TestBudgetRetainsDeletedBytesAndAllowsExactThreshold(t *testing.T) { + s, path := limitedStore(t, 8) + mustBucket(t, s, "bucket") + mustPut(t, s, "bucket", "first", "abcd") + if err := s.DeleteObject(t.Context(), "bucket", "first"); err != nil { + t.Fatal(err) + } + // A duplicate fits in the remaining temporary space, then releases it. + mustPut(t, s, "bucket", "duplicate", "abcd") + mustPut(t, s, "bucket", "second", "efgh") + if _, err := s.PutObject(t.Context(), "bucket", "second", strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrQuota) { + t.Fatal("over-budget overwrite was not refused", err) + } + _, data := readObject(t, s, "bucket", "second") + if string(data) != "efgh" { + t.Fatal("quota refusal changed committed bytes") + } + mustPut(t, s, "bucket", "empty", "") + if err := s.DeleteObject(t.Context(), "bucket", "second"); err != nil { + t.Fatal(err) + } + if _, err := s.PutObject(t.Context(), "bucket", "new", strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrQuota) { + t.Fatal("deleting metadata released retained blob bytes", err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenWithLimit(t.Context(), path, 8) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + if _, err := reopened.PutObject(t.Context(), "bucket", "new", strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrQuota) { + t.Fatal("reopen forgot retained bytes", err) + } +} + +type gatedBudgetReader struct { + entered chan struct{} + release chan struct{} + body io.Reader + started bool +} + +func (r *gatedBudgetReader) Read(p []byte) (int, error) { + if !r.started { + r.started = true + close(r.entered) + <-r.release + } + return r.body.Read(p) +} + +type countingBudgetReader struct{ calls int } + +func (r *countingBudgetReader) Read([]byte) (int, error) { + r.calls++ + return 0, io.EOF +} + +func TestBudgetReservesConcurrentUploadsBeforeReading(t *testing.T) { + s, _ := limitedStore(t, 6) + mustBucket(t, s, "bucket") + reader := &gatedBudgetReader{entered: make(chan struct{}), release: make(chan struct{}), body: strings.NewReader("abcd")} + done := make(chan error, 1) + go func() { + _, err := s.PutObject(context.Background(), "bucket", "first", reader, PutOptions{Size: 4}) + done <- err + }() + select { + case <-reader.entered: + case err := <-done: + t.Fatal("first upload did not reach its accepting case", err) + } + probe := &countingBudgetReader{} + _, err := s.PutObject(t.Context(), "bucket", "second", probe, PutOptions{Size: 3}) + // Always release the first upload before asserting so a failing mutation + // cannot leave Close waiting on a test-owned blocked reader. + close(reader.release) + firstErr := <-done + if firstErr != nil || !errors.Is(err, ErrQuota) || probe.calls != 0 { + t.Fatal("reservation did not protect the aggregate limit before input reads", firstErr, err, probe.calls) + } + if _, err := s.HeadObject(t.Context(), "bucket", "second"); !errors.Is(err, ErrObjectNotFound) { + t.Fatal("refused upload published metadata", err) + } + mustPut(t, s, "bucket", "fits", "ef") +} + +func TestFailedUploadReleasesBudgetAndNeverWritesProbeByte(t *testing.T) { + s, path := limitedStore(t, 4) + mustBucket(t, s, "bucket") + for _, data := range []string{"x", "abcde"} { + if _, err := s.PutObject(t.Context(), "bucket", "bad", strings.NewReader(data), PutOptions{Size: 4}); err == nil { + t.Fatal("invalid upload succeeded") + } + entries, err := os.ReadDir(filepath.Join(path, "blobs")) + if err != nil || len(entries) != 0 { + t.Fatal("failed upload retained temporary bytes", entries, err) + } + } + // Observe the temporary file at the EOF probe, not only after cleanup. + reader := &probeSizeReader{t: t, path: filepath.Join(path, "blobs"), body: strings.NewReader("abcd")} + if _, err := s.PutObject(t.Context(), "bucket", "bad", reader, PutOptions{Size: 4}); err == nil { + t.Fatal("extra probe byte was accepted") + } + if reader.observe == nil { + t.Fatal("EOF probe was not exercised") + } + info, statErr := reader.observe.Stat() + closeErr := reader.observe.Close() + if err := errors.Join(statErr, closeErr); err != nil || info.Size() != 4 { + t.Fatal("temporary file wrote beyond its declared reservation", info, err) + } + mustPut(t, s, "bucket", "good", "abcd") +} + +type probeSizeReader struct { + t *testing.T + path string + body *strings.Reader + observe *os.File +} + +func (r *probeSizeReader) Read(p []byte) (int, error) { + if r.body.Len() == 0 { + entries, err := os.ReadDir(r.path) + if err != nil || len(entries) != 1 { + r.t.Fatal(entries, err) + } + r.observe, err = os.Open(filepath.Join(r.path, entries[0].Name())) + if err != nil { + r.t.Fatal(err) + } + return copy(p, []byte("extra")), nil + } + return r.body.Read(p) +} + +func TestBudgetRecountsCrashFilesAndHardLinks(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + obj := mustPut(t, s, "bucket", "key", "abcd") + if err := s.Close(); err != nil { + t.Fatal(err) + } + blobs := filepath.Join(path, "blobs") + if err := os.Link(filepath.Join(blobs, obj.Digest), filepath.Join(blobs, ".pending-alias")); err != nil { + t.Fatal(err) + } + bounded, err := OpenWithLimit(t.Context(), path, 4) + if err != nil { + t.Fatal("one inode was counted twice", err) + } + mustPut(t, bounded, "bucket", "empty", "") + if err := bounded.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(blobs, ".pending-crash"), []byte("xy"), 0o600); err != nil { + t.Fatal(err) + } + if over, err := OpenWithLimit(t.Context(), path, 5); !errors.Is(err, ErrQuota) { + if over != nil { + _ = over.Close() + } + t.Fatal("crash leftovers did not count", err) + } + bounded, err = OpenWithLimit(t.Context(), path, 6) + if err != nil { + t.Fatal("larger explicit limit did not admit existing data", err) + } + if err := bounded.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(blobs, ".pending-crash")); err != nil { + t.Fatal("opening over budget removed existing data", err) + } +} + +func TestBudgetRejectsUnsafeInventoryAndInvalidLimit(t *testing.T) { + s, path := newStore(t) + if err := s.Close(); err != nil { + t.Fatal(err) + } + for _, limit := range []int64{0, -1} { + if got, err := OpenWithLimit(t.Context(), path, limit); !errors.Is(err, ErrInvalid) || got != nil { + t.Fatal(got, err) + } + } + if err := os.Symlink("../index.json", filepath.Join(path, "blobs", ".pending-link")); err != nil { + t.Fatal(err) + } + if got, err := OpenWithLimit(t.Context(), path, 1024); !errors.Is(err, ErrCorrupt) || got != nil { + t.Fatal("unsafe inventory was trusted", got, err) + } + // A refused limited open must release ownership for the original API. + unlimited, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if err := unlimited.Close(); err != nil { + t.Fatal(err) + } +} + +func TestBudgetChargesBlobWithoutMetadataPublication(t *testing.T) { + s, _ := limitedStore(t, 6) + mustBucket(t, s, "bucket") + reader := &gatedBudgetReader{entered: make(chan struct{}), release: make(chan struct{}), body: strings.NewReader("abcd")} + done := make(chan error, 1) + go func() { + _, err := s.PutObject(context.Background(), "bucket", "first", reader, PutOptions{Size: 4}) + done <- err + }() + select { + case <-reader.entered: + case err := <-done: + t.Fatal("upload did not reach its accepting case", err) + } + deleteErr := s.DeleteBucket(t.Context(), "bucket") + close(reader.release) + err := <-done + if deleteErr != nil || !errors.Is(err, ErrBucketNotFound) { + t.Fatal(deleteErr, err) + } + mustBucket(t, s, "bucket") + if _, err := s.PutObject(t.Context(), "bucket", "new", strings.NewReader("xyz"), PutOptions{Size: 3}); !errors.Is(err, ErrQuota) { + t.Fatal("unreferenced published bytes were not charged", err) + } + mustPut(t, s, "bucket", "fits", "xy") +} + +type movedTempReader struct { + body *strings.Reader + path string +} + +func (r *movedTempReader) Read(p []byte) (int, error) { + if r.body.Len() == 0 { + entries, err := os.ReadDir(r.path) + if err != nil || len(entries) != 1 { + return 0, errors.Join(err, errors.New("expected one temporary blob")) + } + // Inject the filesystem failure without permissions depending on UID: + // the open inode stays in blobs, but publish and cleanup lose its name. + if err := os.Rename(filepath.Join(r.path, entries[0].Name()), filepath.Join(r.path, ".pending-moved")); err != nil { + return 0, err + } + } + return r.body.Read(p) +} + +func TestBudgetChargesFailedTemporaryCleanup(t *testing.T) { + s, path := limitedStore(t, 6) + mustBucket(t, s, "bucket") + reader := &movedTempReader{body: strings.NewReader("abcd"), path: filepath.Join(path, "blobs")} + if _, err := s.PutObject(t.Context(), "bucket", "bad", reader, PutOptions{Size: 4}); err == nil { + t.Fatal("filesystem failure became success") + } + if _, err := s.PutObject(t.Context(), "bucket", "new", strings.NewReader("xyz"), PutOptions{Size: 3}); !errors.Is(err, ErrQuota) { + t.Fatal("failed cleanup released bytes still on disk", err) + } + mustPut(t, s, "bucket", "fits", "xy") +} diff --git a/internal/core/objectstore/files.go b/internal/core/objectstore/files.go index 1dc4148e..0fb41170 100644 --- a/internal/core/objectstore/files.go +++ b/internal/core/objectstore/files.go @@ -215,16 +215,44 @@ func tempFile(root *os.Root) (*os.File, string, error) { } func (s *Store) writeBlob(ctx context.Context, body io.Reader, size int64) (_ string, err error) { + if err := s.reserveBlob(size); err != nil { + return "", err + } + var retained int64 + defer func() { s.settleBlob(size, retained) }() file, temp, err := tempFile(s.blobs) if err != nil { return "", err } - defer func() { err = errors.Join(err, s.blobs.Remove(temp)) }() + var n int64 + published := false + defer func() { + removeErr := s.blobs.Remove(temp) + err = errors.Join(err, removeErr) + if published { + retained = size + } else if removeErr != nil { + retained = n + } + }() hash := sha256.New() - n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(&contextReader{ctx: ctx, reader: body}, size+1)) + reader := &contextReader{ctx: ctx, reader: body} + // TestFailedUploadReleasesBudgetAndNeverWritesProbeByte holds the physical + // bound: detect excess input without writing outside the reservation. + n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(reader, size)) if copyErr == nil && n != size { copyErr = fmt.Errorf("%w: expected %d bytes, received %d", ErrInvalid, size, n) } + if copyErr == nil { + var extra [1]byte + count, probeErr := io.ReadFull(reader, extra[:]) + switch { + case count != 0: + copyErr = fmt.Errorf("%w: upload exceeds its declared size", ErrInvalid) + case !errors.Is(probeErr, io.EOF): + copyErr = probeErr + } + } if copyErr == nil { copyErr = ctx.Err() } @@ -246,6 +274,8 @@ func (s *Store) writeBlob(ctx context.Context, body io.Reader, size int64) (_ st if err := existing.Close(); err != nil { return "", fmt.Errorf("close existing blob: %w", err) } + } else { + published = true } dir, err := s.blobs.Open(".") if err != nil { diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index 6fb144ef..35938b9d 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -77,6 +77,7 @@ type Store struct { closed bool fault error domain string + budget *blobBudget } // Close releases directory handles and the exclusive process lock. diff --git a/internal/s3/server.go b/internal/s3/server.go index 6b2d88ac..65f95e6e 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -259,6 +259,8 @@ func (s *Server) storeError(w http.ResponseWriter, r *http.Request, bucket strin status, message := http.StatusInternalServerError, "object storage operation failed" var tooLarge *http.MaxBytesError switch { + case errors.Is(err, objectstore.ErrQuota): + status, message = http.StatusInsufficientStorage, "object storage blob byte budget exceeded" case errors.Is(err, objectstore.ErrObjectNotFound), errors.Is(err, objectstore.ErrBucketExists), errors.Is(err, objectstore.ErrBucketNotEmpty): status, message = http.StatusNotImplemented, "cloud error shape has not been measured for this store outcome" diff --git a/internal/s3/server_test.go b/internal/s3/server_test.go index 71f6236a..4cfeb497 100644 --- a/internal/s3/server_test.go +++ b/internal/s3/server_test.go @@ -25,7 +25,7 @@ func fixture(t *testing.T) (*Server, *objectstore.Store) { if err := os.Chmod(parent, 0700); err != nil { t.Fatal(err) } - store, err := objectstore.Open(t.Context(), filepath.Join(parent, "objects")) + store, err := objectstore.OpenWithLimit(t.Context(), filepath.Join(parent, "objects"), 2<<20) if err != nil { t.Fatal(err) } @@ -425,4 +425,27 @@ func TestAmbiguousUploadHeadersDoNotPublish(t *testing.T) { t.Fatal("ambiguous headers were silently ignored", duplicate, w.Code, w.Body.String()) } } + +} + +func TestQuotaRefusalIsExplicitAndPreservesExistingObject(t *testing.T) { + s, _ := fixture(t) + if w := request(t, s, "PUT", "/bucket", nil, nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + payload := bytes.Repeat([]byte("x"), 1<<20) + if w := request(t, s, "PUT", "/bucket/key", payload, nil); w.Code != 200 { + t.Fatal("exact quota threshold must work", w.Code, w.Body.String()) + } + if w := request(t, s, "PUT", "/bucket/second", bytes.Repeat([]byte("y"), 1<<20), nil); w.Code != 200 { + t.Fatal("remaining quota must work", w.Code, w.Body.String()) + } + w := request(t, s, "PUT", "/bucket/key", []byte("z"), nil) + if w.Code != 507 || !strings.HasPrefix(w.Body.String(), "feint: object storage blob byte budget exceeded") { + t.Fatal("quota became an invented cloud response or success", w.Code, w.Body.String()) + } + w = request(t, s, "GET", "/bucket/key", nil, nil) + if w.Code != 200 || !bytes.Equal(w.Body.Bytes(), payload) { + t.Fatal("quota refusal damaged existing bytes", w.Code) + } } diff --git a/tools/falsify/specs/objectstore-budget.json b/tools/falsify/specs/objectstore-budget.json new file mode 100644 index 00000000..f09bb87c --- /dev/null +++ b/tools/falsify/specs/objectstore-budget.json @@ -0,0 +1,76 @@ +{ + "package": "./internal/core/objectstore/", + "mutations": [ + { + "label": "aggregate allocations exceed the blob-byte limit", + "file": "internal/core/objectstore/budget.go", + "find": "if size > b.limit-b.used-b.held {", + "replace": "if size > b.limit-b.used-b.held && false {", + "test": "TestBudgetRetainsDeletedBytesAndAllowsExactThreshold|TestBudgetReservesConcurrentUploadsBeforeReading" + }, + { + "label": "concurrent uploads never reserve their input size", + "file": "internal/core/objectstore/budget.go", + "find": "b.held += size", + "replace": "b.held += size * 0", + "test": "TestBudgetReservesConcurrentUploadsBeforeReading" + }, + { + "label": "published bytes lose their quota charge", + "file": "internal/core/objectstore/budget.go", + "find": "b.used += retained", + "replace": "b.used += retained * 0", + "test": "TestBudgetRetainsDeletedBytesAndAllowsExactThreshold|TestBudgetChargesBlobWithoutMetadataPublication" + }, + { + "label": "reopen forgets retained blob and crash bytes", + "file": "internal/core/objectstore/budget.go", + "find": "used += info.Size()", + "replace": "used += info.Size() * 0", + "test": "TestBudgetRecountsCrashFilesAndHardLinks" + }, + { + "label": "hard-linked crash names count the same file twice", + "file": "internal/core/objectstore/budget.go", + "find": "if seen[identity] {", + "replace": "if seen[identity] && false {", + "test": "TestBudgetRecountsCrashFilesAndHardLinks" + }, + { + "label": "the EOF probe byte is written beyond the reservation", + "file": "internal/core/objectstore/files.go", + "find": "io.LimitReader(reader, size)", + "replace": "io.LimitReader(reader, size+1)", + "test": "TestFailedUploadReleasesBudgetAndNeverWritesProbeByte" + }, + { + "label": "failed temporary cleanup releases retained bytes", + "file": "internal/core/objectstore/files.go", + "find": "} else if removeErr != nil {", + "replace": "} else if removeErr != nil && false {", + "test": "TestBudgetChargesFailedTemporaryCleanup" + }, + { + "label": "temporary reservations are never released", + "file": "internal/core/objectstore/budget.go", + "find": "b.held -= reserved", + "replace": "b.held -= reserved * 0", + "test": "TestBudgetRetainsDeletedBytesAndAllowsExactThreshold" + }, + { + "label": "an extra EOF probe byte no longer refuses publication", + "file": "internal/core/objectstore/files.go", + "find": "case count != 0:", + "replace": "case count != 0 && false:", + "test": "TestFailedUploadReleasesBudgetAndNeverWritesProbeByte" + }, + { + "label": "the S3 boundary disguises a quota refusal as a generic storage error", + "file": "internal/s3/server.go", + "find": "case errors.Is(err, objectstore.ErrQuota):", + "replace": "case errors.Is(err, objectstore.ErrQuota) && false:", + "test": "TestQuotaRefusalIsExplicitAndPreservesExistingObject", + "package": "./internal/s3/" + } + ] +} From 3d3ed3af9db61ad9d47502f6b16713d13d54fcdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 00:11:48 +0100 Subject: [PATCH 04/28] feat(s3): mount persistent object bytes on the existing port Declare prefixed provider/region doors over one bounded neutral store. Carry explicit storage settings through serve/start/restart, discover only live S3 endpoints for env, and report separate HTTP response evidence without altering SDK coverage. Prove AWS CLI 2.32.17 byte/checksum identity and literal keys through the actual process across restart. Keep ancillary provider CLI operations, hardcoded Terraform endpoints and reset outside this slice. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 19 ++ CHANGELOG.md | 19 ++ RELEASING.md | 1 + docs/README.md | 3 +- docs/architecture.md | 33 ++- docs/confidence.md | 7 +- docs/limits.md | 23 +- docs/object-storage-routing.md | 81 +++++- internal/cli/cli.go | 55 +++- internal/cli/doctor.go | 26 ++ internal/cli/env.go | 16 ++ internal/cli/frozen_test.go | 29 ++- internal/cli/lifecycle.go | 9 + internal/cli/object_storage.go | 235 +++++++++++++++++ .../cli/object_storage_conformance_test.go | 178 +++++++++++++ internal/cli/object_storage_test.go | 237 +++++++++++++++++ internal/cli/status.go | 49 +++- internal/cli/testdata/frozen/cli.json | 230 +++++++++++++++++ internal/cli/testdata/frozen/s3.json | 59 +++++ internal/conformancetest/aws.go | 107 ++++++++ internal/core/objectstore/budget.go | 9 + internal/providers/exoscale/object_storage.go | 10 + internal/providers/scaleway/object_storage.go | 10 + internal/s3/client_conformance_test.go | 82 +----- internal/s3/doors.go | 211 +++++++++++++++ internal/s3/doors_test.go | 242 ++++++++++++++++++ internal/s3/server.go | 31 ++- mise.toml | 4 + .../falsify/specs/mounted-object-storage.json | 178 +++++++++++++ tools/testplan/rules.go | 53 +++- 30 files changed, 2108 insertions(+), 138 deletions(-) create mode 100644 internal/cli/object_storage.go create mode 100644 internal/cli/object_storage_conformance_test.go create mode 100644 internal/cli/object_storage_test.go create mode 100644 internal/cli/testdata/frozen/s3.json create mode 100644 internal/conformancetest/aws.go create mode 100644 internal/providers/exoscale/object_storage.go create mode 100644 internal/providers/scaleway/object_storage.go create mode 100644 internal/s3/doors.go create mode 100644 internal/s3/doors_test.go create mode 100644 tools/falsify/specs/mounted-object-storage.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index 33f651c9..b366c6bd 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -17,6 +17,25 @@ change ni l'un ni l'autre a sa place dans `git log`. ## [Unreleased] +### Ajouté + +- **Des octets S3 simples, persistants et opt-in sur le port existant** (#850) : + `serve`/`start --objectstore`, `--s3-doors`, `--objectstore-budget-mib` + (1024 par défaut) et `--s3-object-limit-mib` (256 par défaut). + Les préfixes provider/région déclarés préservent les clés littérales et + séparent les espaces. `restart` conserve la racine absolue et les limites ; + les snapshots de ressources et le nettoyage ne possèdent pas ces octets. + AWS CLI 2.32.17 prouve l'identité octets/checksum sur le vrai processus avant + et après redémarrage. Les cycles `scw`/`exo` complets, les ETag, le multipart + et les endpoints Terraform codés en dur restent hors périmètre. +- **La découverte S3 active, pas un endpoint deviné quand elle est désactivée** + (#850) : `env --s3 [--s3-region]`, `doctor --s3`, un statut explicite + actif/désactivé/inconnu et le rapport indépendant `/_feint/s3`, schéma **1**. + Ses compteurs HTTP ne gonflent ni la couverture SDK du plan de contrôle ni + ses preuves clientes. La surface CLI additive passe à **26** ; les codes de + sortie et le schéma de santé restent inchangés. `env --unset` retire aussi + les variables d'endpoint S3, sans instance active. + ## [0.14.0] - 2026-10-26 ### Ajouté diff --git a/CHANGELOG.md b/CHANGELOG.md index c8a1f0b6..902f8f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,25 @@ what this project is judged on: **a response shape a client can observe**, and ## [Unreleased] +### Added + +- **Opt-in, persisted simple S3 bytes on the existing port** (#850): + `serve`/`start --objectstore`, `--s3-doors`, `--objectstore-budget-mib` + (1024 by default) and `--s3-object-limit-mib` (256 by default). + Declared provider/region prefixes preserve literal keys and separate + namespaces. `restart` preserves the absolute root and limits; resource + snapshots and cleanup do not own these bytes. AWS CLI 2.32.17 proves + byte/checksum identity through the real process before and after restart. + Complete `scw`/`exo` lifecycles, ETag semantics, multipart and hardcoded + Terraform endpoints remain unsupported. +- **Live S3 discovery, not an endpoint guessed while disabled** (#850): + `env --s3 [--s3-region]`, `doctor --s3`, explicit enabled/disabled/unknown + status, and independent `/_feint/s3` report schema **1**. HTTP response + counts do not inflate control-plane SDK coverage or client evidence. + The additive CLI surface is version **26**; existing exit codes and the + health schema stay unchanged. Ordinary `env --unset` also clears S3 + endpoint variables offline. + ## [0.14.0] - 2026-10-26 ### Added diff --git a/RELEASING.md b/RELEASING.md index b4da776e..e7c6d77e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -173,6 +173,7 @@ its whole behaviour would be promising a moving copy of three clouds. | `/_feint/conformance` | `schema_version` | `internal/cli/testdata/frozen/conformance.json` | | `/_feint/trace` | `schema_version` | `internal/cli/testdata/frozen/trace.json` | | `/_feint/faults` | `schema_version` | `internal/cli/testdata/frozen/faults.json` | +| `/_feint/s3` | independent `schema_version` | `internal/cli/testdata/frozen/s3.json` | | the CLI's verbs, flags and exit codes | `cliSurfaceVersion` | `internal/cli/testdata/frozen/cli.json` | | the state and snapshot formats | the document's own version | `internal/cli/snapshot.go` | diff --git a/docs/README.md b/docs/README.md index 686bdd43..bd0ec8a6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,7 +56,8 @@ small enough to read, and no cloud account. - [architecture.md](architecture.md) — the packs, the neutral core, and the boundary between them. - [object-storage-routing.md](object-storage-routing.md) — the accepted - single-port S3 door decision and its byte budgets; implementation is staged. + single-port S3 doors, byte budgets, activation commands and actual-process + byte proof; complete provider lifecycles remain staged. - [../CONTRIBUTING.md](../CONTRIBUTING.md) — how a change is proved before it is proposed. - [proxy.md](proxy.md) — record a real cloud, read the recording, and turn it diff --git a/docs/architecture.md b/docs/architecture.md index 3b16bcbf..c8ba1dee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,7 +14,8 @@ internal/cli/ the commands: lifecycle, serve, env, doctor, snapsho internal/core/resource/ the neutral resource: ID, Kind, Tenant, State, Attrs, Runtime internal/core/store/ in-memory storage plus JSON snapshot, with no provider knowledge internal/core/objectstore/ neutral persisted object bytes and metadata -internal/s3/ bounded path-style protocol component; not mounted yet +internal/s3/ bounded S3 protocol and same-port explicit door dispatch +internal/conformancetest/ isolated pinned AWS CLI harness, under the conformance tag internal/core/emulator/ Env, Pack, Route, the single-port mount, /_feint/* internal/core/machine/ the machine runtime behind --vm: Incus containers, VMs, OVN internal/core/network/ address plans, subnets, firewall bindings @@ -75,11 +76,11 @@ the pack is what only that provider knows — its image catalogue, the login its machines use, the field its API publishes an address under, or the fact that it publishes none. -## Neutral object bytes, not yet an S3 service +## Neutral object bytes and explicitly mounted S3 doors [objectstore](../internal/core/objectstore/store.go) is an independent library -for #850. It is not wired into the emulator or CLI, and does not change the -Object Storage support limit. Its caller explicitly selects a private root +for #850, activated only by `serve`/`start --objectstore`. It does not imply +complete provider Object Storage support. Its caller explicitly selects a private root whose parent already exists. New roots are mode 0700; files are mode 0600. The pinned parent must belong to this user or root; group/other write permission requires its sticky bit so another UID cannot replace the newly created root @@ -123,18 +124,20 @@ when metadata publication fails. Failed cleanup retains the partial byte charge. Deletion does not free blob bytes. Identical uploads still require temporary space. Metadata and filesystem overhead are outside this byte budget. See the [accepted door ADR](object-storage-routing.md) for the selected opt-in -defaults (1 GiB cumulative, 256 MiB per simple PUT) and the still-unmounted -server integration. +defaults (1 GiB cumulative, 256 MiB per simple PUT) and the mounted +same-port server integration. The gateway renders a budget refusal as a logged Feint HTTP 507 text response, not an unmeasured S3 XML error. The official-client component proof uses a small -explicit cumulative budget; deployment defaults are not yet CLI options. +explicit cumulative budget; deployment limits are configurable CLI options. -### Explicit S3 component, not shipped Object Storage support +### Explicit S3 protocol, not complete Object Storage support [s3](../internal/s3/server.go) is a direct HTTP handler over that store, with no -listener, CLI option, provider route or snapshot integration. The mono-port -server remains unchanged. Mounting provider doors, trusted intercepted host -handoff and virtual-host routing are separate decisions, not implicit promises. +listener, control-plane SDK route or snapshot integration. +[Doors](../internal/s3/doors.go) dispatch declared provider/region prefixes +before the mono-port server's cleaning mux, over one shared store with opaque +namespaces. CLI composition owns activation and shutdown. Trusted intercepted +host handoff and virtual-host routing remain deferred, not implicit promises. It accepts loopback peers and literal loopback/localhost endpoint hosts only. SigV4 credentials are accepted without authenticating their secret; they are never logged. Local callers still have access to the whole selected store. @@ -179,9 +182,11 @@ family to other missing-bucket operations is an adapter choice, not another cloud measurement. NoSuchKey, conflicting creation and BucketNotEmpty still get a logged **Feint 501 text refusal**, not fabricated cloud XML. -The official-client witness proves a protocol component, not an invocation of -`feint serve`, `scw`, `exo`, a surveyed Terraform stack, interception or reset. -Existing Object Storage support limits and the full conformance population are +That witness proves a protocol component. `mise run conformance:s3:mounted` +separately drives the actual `feint start`/`restart` process on both declared +doors and proves bytes, checksums, literal keys and persisted namespaces. +Neither proves complete `scw`/`exo` lifecycles, a surveyed Terraform stack, +interception or reset. The full control-plane conformance population is unchanged. [The declared mutations](../tools/falsify/specs/s3-gateway.json) hold checksum, framing, size, metadata and unsupported-feature guards. diff --git a/docs/confidence.md b/docs/confidence.md index c9f969ba..63cd0f6c 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -40,13 +40,14 @@ prints the reason under the pack that owns it. | Reaching a private address from the host that runs the emulator | **depends on the mode** | keys on `capabilities.private_from_host` — [limits.md](limits.md#a-public-address-is-the-providers-value-made-to-answer-on-the-host) | | A kernel-level test: sysctl, modules, the boot path | **with a VM runtime** | `FEINT_VM=incus-vm`, keying on `capabilities.own_kernel`: a container shares the host kernel and cannot carry any of it — [limits.md](limits.md#docker-was-removed-it-cannot-back-an-emulated-network) | | That a bad image or instance-type id is rejected | **with `serve --strict-catalog`**, against the catalogue you declared | off by default, and the default is the decision; each pack refuses in its own shape, and which of those shapes were recorded is in the table — [limits.md](limits.md#identifiers-are-not-checked-against-anything) | -| Quotas, real capacity, prices | **no** | [limits.md](limits.md#the-catalogue-is-a-whitelist-and-its-values-are-measured) | +| Provider quotas, real capacity, prices | **no** | [limits.md](limits.md#the-catalogue-is-a-whitelist-and-its-values-are-measured) | | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Object Storage, and anything addressed as `s3.` | **no** | [limits.md](limits.md#object-storage-is-not-emulated) | +| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17 | `mise run conformance:s3:mounted` drives the actual process before and after restart; complete `scw`/`exo` lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#implemented-mounted-slice) | +| Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | -| Multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | +| Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | | Traffic actually flowing through a gateway or a NAT | **no** | records move, packets do not — [limits.md](limits.md#outscales-gateways-and-nat-move-records-not-packets) | ## How to read a verdict diff --git a/docs/limits.md b/docs/limits.md index 9b6ff74e..e8417e9e 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -4,7 +4,17 @@ What Feint deliberately does not do, and why. Stating this precisely matters more than the feature list: an emulator that lies about its coverage is worse than one that is small. -## Object Storage is not emulated +## Object Storage: simple explicit doors, not full client workflows + +#850 adds opt-in, persisted simple S3 operations on the existing port, with +declared `/_feint/s3///` endpoints, cumulative byte budgets +and an AWS CLI 2.32.17 restart proof. See the +[door ADR and commands](object-storage-routing.md). +Complete `scw`/`exo` lifecycles, listing, ACL/versioning, multipart, ETags, +checkpoint/reset and hardcoded Terraform redirection remain unsupported. +No MinIO service or dependency is used by this implementation. + +### Why Scaleway Terraform is still not redirected Scaleway Object Storage is S3-compatible, so emulating it is not the hard part (MinIO does it already). The obstacle is **how clients reach it**. @@ -55,11 +65,11 @@ here. The export stays declined; the import no longer shares its refusal. What this emulator records is the intention. A snapshot appears, `available`, carrying the name and project the client gave — the request declares no tags, unlike the create beside it — with no parent volume and a -size of zero, because no bytes moved and none could have. +size of zero, because this handler does not read object bytes. The divergence, stated rather than hidden: the real API refuses an object that -does not exist, and this emulator cannot. There is no Object Storage here to -check a bucket and a key against, so an import naming an object nobody uploaded +does not exist, and this handler does not check the opt-in byte store. The +snapshot-import route is not wired to the S3 gateway, so an import naming an object nobody uploaded answers success here and answers an error upstream. A plan that depends on that refusal is not covered. @@ -344,8 +354,9 @@ wall and no longer an operator ceremony either — it is the coverage argument alone: whether the S3-through-Terraform corner is worth emulating an S3 surface, which is a product call and the only thing still holding the refusal. -The blocker is one product on one client, so the MinIO + `SCW_S3_ENDPOINT` page -remains the right answer for the S3 workflow, and the refusal caps coverage by +That measurement predates #850's explicit byte gateway: a complete configurable +S3 workflow still needs operations beyond its simple byte slice (or a separate +S3 service such as MinIO). The Terraform refusal still caps coverage by exactly one corner rather than quietly bounding the whole project. If Object-Storage-through-Terraform is ever wanted, it is a roadmap item with a named owner and a shape already measured, and number 7 shortened it again: diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 4a665a22..7ee0e694 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -5,9 +5,9 @@ decision is not a claim of shipped Object Storage support. ## Context -The neutral ObjectStore persists verified bytes. The unmounted S3 gateway has -an official AWS CLI HTTPS round-trip proof. Neither gives `feint serve` an S3 -endpoint yet. Raw path-style S3 (`/bucket/key`) can collide with control-plane +At the time of the decision, the neutral ObjectStore persisted verified bytes +and the unmounted S3 gateway had an official AWS CLI HTTPS round-trip proof, +but neither gave `feint serve` an S3 endpoint. Raw path-style S3 (`/bucket/key`) can collide with control-plane paths; `http.ServeMux` also cleans dot segments and repeated slashes, which are legitimate logical object keys. @@ -90,3 +90,78 @@ Virtual hosts, trusted intercepted-host handoff, hardcoded Scaleway Terraform endpoints, checkpoint/reset, garbage collection and public-stack certification remain later work. Existing Object Storage support limits stay unchanged until the mounted-client proofs exist. + +## Implemented mounted slice + +`serve` and `start` now accept the same explicit storage settings; `restart` +replays the recorded absolute storage root even from a different working +directory. The parent must already exist and be safely owned: the command +does not change the permissions of an existing operator directory. + +```bash +mkdir -m 700 object-storage +feint start --objectstore "$PWD/object-storage/data" \ + --objectstore-budget-mib 1024 --s3-object-limit-mib 256 \ + --s3-doors scaleway/fr-par,exoscale/ch-gva-2 +feint env scaleway --s3 +feint env exoscale --s3 +feint status +feint doctor --s3 --vm off +feint restart +feint stop +``` + +Storage is disabled without `--objectstore`. Limits are integer MiB +(1 MiB = 1,048,576 bytes); the defaults are the selected 1 GiB / 256 MiB. +The door list defaults to the two paths above. A declaration selects a +provider dialect and a local region namespace; it is not a promise that the +region's complete cloud API is served or separately measured. + +`env --s3` discovers the running instance on a loopback control-plane endpoint, +without an inherited proxy or redirect. Disabled, missing or ambiguous doors +fail before stdout contains any export. Use `--s3-region` to choose among +multiple enabled regions of a provider. Ordinary `env` remains offline and +control-plane-only. `env --unset` also removes S3 endpoint variables, without +requiring a running instance. All three shell renderers use this same selection. +These are provider CLI variables, not an AWS CLI credential configuration. + +The separate `GET /_feint/s3` report (schema version 1) publishes actual +activation, declared paths, the store's configured blob-byte limit, the +per-object limit and completed HTTP-response counts per declared door/status. +It publishes no credentials, keys, payloads or storage paths. These counts +are **not official-client proof** and do not enter control-plane SDK coverage, +trace or conformance scores. Status reports disabled and unknown distinctly; +`doctor --s3` requires enabled storage. The health schema is unchanged. + +Each mounted PUT installs a 30-second body-read deadline, independently of +the existing control-plane and runtime write deadlines. The server retains +its 10-second header deadline and the standard library's bounded HTTP header +size. The S3 handler rejects browser Origins and non-loopback peers/Hosts even +when the control plane is exposed. +An exposed wildcard listener advertises a usable literal loopback S3 URL, +not `0.0.0.0` or `::`. Activation on a specific non-loopback listen address is +refused: that socket cannot receive the peers the S3 boundary requires. +HTTP shutdown precedes releasing storage +ownership; an unsuccessful graceful shutdown force-closes HTTP connections. +Resource snapshots (`--state`, `/_feint/state`) do not include object bytes, +and `--cleanup` does not delete their persisted blobs. + +`mise run conformance:s3:mounted` builds the real binary, runs `start`, drives +AWS CLI **2.32.17** through both prefixed doors, proves byte/checksum identity +and namespace separation, restarts from another directory, retrieves the +bytes again and independently checks that its literal key survived storage. +The 73,728-byte binary witness has SHA-256 +`4e5fadf6593bcd790f284d715b339960185aa4c760e56bbd79a516d39f975ec1`. +It uses isolated configuration and canonical public fake credentials, with a +non-forwarding proxy and a positive CONNECT refusal control. The proxy is not +a network namespace. It stops only its recorded process and owns its scratch +data. This is an HTTP actual-process witness; +`mise run conformance:s3` separately retains the HTTPS streaming/trailer and +hostile-response-checksum controls. + +This certifies simple CreateBucket, PutObject, GetObject, HeadObject, +DeleteObject and DeleteBucket through the mounted gateway, not complete +`scw`/`exo` lifecycles. Unsupported ancillary operations, missing-object cloud +errors and multipart still receive explicit Feint refusals. No ETag is invented. +The snapshot-import route still records an intention rather than reading these +bytes. Object Storage 1.0 and hardcoded Terraform redirection are not delivered. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 5e4d4c34..7283d33e 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -241,11 +241,14 @@ const ( // in any answer. Two additions to two existing verbs; nothing was removed, // no exit code moved. // +// Version 26 adds opt-in Object Storage flags to serve/start and live S3 +// discovery to env/doctor (#850); existing flags and exit codes stay unchanged. +// // The surface itself is frozen in testdata/frozen/cli.json, compared by // TestTheFrozenSurfacesStillMatchTheirFixture, and a fixture regenerated // without bumping this constant fails TestASurfaceChangeDemandsItsVersionBump. // The procedure for a deliberate change is in RELEASING.md ("Frozen surfaces"). -const cliSurfaceVersion = 25 +const cliSurfaceVersion = 26 // Run executes one command and returns the process exit code. func Run(args []string, stdout, stderr io.Writer) int { @@ -346,6 +349,8 @@ Usage: [--log-level info|debug] [--expose-to-network] [--projects [,...]] [--consistency immediate|eventual] [--strict-catalog ] + [--objectstore ] [--s3-doors ] + [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] Serve the three emulated clouds on one port, in the foreground. --expose-to-network is the only way off loopback, and it disarms the anti-rebinding guard: this @@ -358,6 +363,9 @@ Usage: naming an image, a template or a machine type outside the catalogue you declared, in each cloud's own error shape; without it any identifier is accepted (docs/limits.md). + Object Storage is opt-in, on the same port, through declared + /_feint/s3/// prefixes. Its private bytes + are independent of --state; deletion retains blob charges. feint up [--file feint.yaml] [--runtime off|incus|incus-vm|incus-ovn|auto] [--timeout 2m] [--no-iac] @@ -381,6 +389,8 @@ Usage: [--cleanup] [--resolver ] [--contracts ] [--log-level info|debug] [--projects [,...]] [--strict-catalog ] [--timeout 30s] [--detach] [--foreground] + [--objectstore ] [--s3-doors ] + [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] Same, detached: records the instance, waits until it answers, prints where the log is. Refuses to adopt an instance already running on that address. The serve flags @@ -414,10 +424,11 @@ Usage: feint logs [--addr :4599] [-n 50] [-f] The detached run's log. - feint doctor [--addr :4599] [--vm auto] + feint doctor [--addr :4599] [--vm auto] [--s3] Diagnose the host: the port, the machine runtime and what it can prove, the clients, and the ssh trap. A warning never fails; only a broken thing does. + --s3 requires live, explicitly enabled byte storage. feint images [--vm auto] [--only ubuntu/24.04] [--check] Build the machine images, which carry an ssh daemon so a @@ -426,8 +437,11 @@ Usage: what is missing and exits 2, building nothing. feint env [--shell bash|fish|powershell] [--endpoint ] [--unset] - [--client ] + [--client ] [--s3] [--s3-region ] The environment a real client of that provider needs. + --s3 discovers the live, enabled S3 doors before exporting; + it refuses disabled or ambiguous endpoints. --s3-region + selects among them. --unset needs no running process. Exports on stdout, everything else on stderr, so eval "$(feint env scaleway)" is safe. --client selects a family when a provider's clients disagree about a value: @@ -956,7 +970,7 @@ func checkAddrUnclaimed(addr string) error { addr, describeForeign(id), addr) } -func serve(args []string, stdout io.Writer) error { +func serve(args []string, stdout io.Writer) (result error) { fs := newFlagSet("serve") addr := fs.String("addr", DefaultAddr, "listen address") state := fs.String("state", "", "load and persist the store to this JSON file") @@ -971,9 +985,21 @@ func serve(args []string, stdout io.Writer) error { expose := fs.Bool("expose-to-network", false, "listen off loopback, which disarms the browser guard: read what it costs before setting it") consistency := fs.String("consistency", "immediate", "immediate settles every action at once; eventual walks the states a real cloud passes through, one per read, so a client's waiter has something to observe") strictCatalog := fs.String("strict-catalog", "", "refuse a create naming an image, a template or a machine type outside this declared catalogue (a JSON file, docs/limits.md); unset, the compatibility mode, accepts any identifier") + storageFlags := bindObjectStorageFlags(fs) if err := fs.Parse(args); err != nil { return err } + if err := storageFlags.validate(); err != nil { + return err + } + var storageEndpoint string + if storageFlags.root != "" { + var err error + storageEndpoint, err = storageListenEndpoint(*addr) + if err != nil { + return err + } + } if err := checkListenAddr(*addr, *expose); err != nil { return err @@ -1142,15 +1168,31 @@ func serve(args []string, stdout io.Writer) error { } } + doors, objects, err := mountObjectStorage(context.Background(), srv, storageFlags, env.Log) + if err != nil { + return err + } + if objects != nil { + defer func() { + if err := objects.Close(); err != nil { + result = errors.Join(result, fmt.Errorf("close object storage: %w", err)) + } + }() + } httpSrv := &http.Server{ Addr: *addr, // Wrapped here rather than in the server: this is where the listen // address is known, and the guard's whole job is to compare what the // caller asked for against what this process is bound to. - Handler: emulator.GuardRebinding(srv.Handler(), *addr), + Handler: emulator.GuardRebinding(doors, *addr), ReadHeaderTimeout: 10 * time.Second, WriteTimeout: writeTimeoutFor(rt), } + defer func() { + if err := httpSrv.Close(); err != nil { + result = errors.Join(result, fmt.Errorf("close HTTP server: %w", err)) + } + }() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -1170,6 +1212,9 @@ func serve(args []string, stdout io.Writer) error { fmt.Fprintf(stdout, " %-9s %d routes\n", p.Name(), len(p.Routes())) } fmt.Fprintf(stdout, " machines %s\n", env.RuntimeName()) + for _, door := range doors.Snapshot().Doors { + fmt.Fprintf(stdout, " S3 %s%s\n", storageEndpoint, door.Path) + } if page { fmt.Fprintf(stdout, " page http://%s/_feint/ui\n", *addr) } diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index bcfa72d2..fb52e1b3 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -47,6 +47,7 @@ func doctor(args []string, stdout, stderr io.Writer) int { fs := newFlagSet("doctor") addr := fs.String("addr", DefaultAddr, "the address serve would listen on") vm := fs.String("vm", "auto", "the machine runtime to diagnose: off, incus, incus-vm, incus-ovn, auto") + withS3 := fs.Bool("s3", false, "require and diagnose live S3 activation on this address") if err := fs.Parse(args); err != nil { return exitError } @@ -62,6 +63,9 @@ func doctor(args []string, stdout, stderr io.Writer) int { checks = append(checks, checkEnvHazards()...) checks = append(checks, checkStackHazards()...) checks = append(checks, checkSSHConfig()) + if *withS3 { + checks = append(checks, checkObjectStorage(*addr)) + } failed := 0 for _, c := range checks { @@ -74,6 +78,7 @@ func doctor(args []string, stdout, stderr io.Writer) int { failed++ case verdictOK: } + fmt.Fprintf(stream, " %s %s\n", mark, c.title) if c.detail != "" { fmt.Fprintf(stream, " %s\n", c.detail) @@ -90,6 +95,27 @@ func doctor(args []string, stdout, stderr io.Writer) int { return exitOK } +func checkObjectStorage(addr string) check { + c := check{title: "live S3 storage", state: verdictFail, + fix: "start with --objectstore , then use feint env --s3"} + endpoint, err := storageListenEndpoint(addr) + if err != nil { + c.detail = err.Error() + return c + } + report, err := fetchS3(endpoint) + if err != nil { + c.detail = err.Error() + } else if !report.Enabled { + c.detail = "S3 is explicitly disabled" + } else { + c.state = verdictOK + c.detail = fmt.Sprintf("%d declared door(s); %d blob bytes maximum; %d bytes per simple PUT; ancillary client lifecycles are not certified", + len(report.Doors), report.MaxBlobBytes, report.MaxObjectBytes) + } + return c +} + // checkPort answers the first question anybody asks when serve will not start, // and names the process holding the port rather than leaving it to lsof. func checkPort(addr string) check { diff --git a/internal/cli/env.go b/internal/cli/env.go index f3260d10..6cd0126a 100644 --- a/internal/cli/env.go +++ b/internal/cli/env.go @@ -31,6 +31,8 @@ import ( func envCommand(args []string, stdout, stderr io.Writer) int { fs := newFlagSet("env") + withS3 := fs.Bool("s3", false, "export only live, explicitly enabled S3 endpoints as well as control-plane variables") + s3Region := fs.String("s3-region", "", "select an enabled S3 region when more than one is declared") shell := fs.String("shell", "bash", "shell syntax: bash, fish or powershell") endpoint := fs.String("endpoint", "", "the emulator's address (default: http://)") unset := fs.Bool("unset", false, "print the removals instead of the exports") @@ -89,6 +91,10 @@ func envCommand(args []string, stdout, stderr io.Writer) int { target = defaultEndpoint() } env := pack.Env(target) + if *s3Region != "" && !*withS3 { + fmt.Fprintln(stderr, "feint: --s3-region requires --s3") + return exitError + } if *client != "" { // A client family only means something to a pack whose clients // disagree; anywhere else the flag would silently print the same thing @@ -111,6 +117,16 @@ func envCommand(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "feint: unknown shell %q; bash, fish or powershell\n", *shell) return exitError } + _, hasS3 := pack.(packObjectStorage) + if *withS3 || (*unset && hasS3) { + if err := storageEnvironment(pack, target, *s3Region, *unset, env.Vars); err != nil { + fmt.Fprintf(stderr, "feint: %v\n", err) + return exitError + } + if !*unset { + fmt.Fprintln(stderr, "note: S3 serves simple byte operations only; complete provider CLI lifecycles and hardcoded Terraform endpoints are not certified.") + } + } keys := make([]string, 0, len(env.Vars)) for key := range env.Vars { diff --git a/internal/cli/frozen_test.go b/internal/cli/frozen_test.go index 1d9122a6..6ca999e0 100644 --- a/internal/cli/frozen_test.go +++ b/internal/cli/frozen_test.go @@ -45,6 +45,7 @@ import ( "testing" "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/s3" "github.com/stephrobert/feint/internal/transcript" ) @@ -194,6 +195,7 @@ func observedSurfaces(t *testing.T) []frozenSurface { // a fault from a committed file is a consumer on day one, and the rules // it PUTs are the same document this answers. {name: "faults", version: emulator.FaultsSchemaVersion, content: observeShape(t, "/_feint/faults", nil)}, + {name: "s3", version: s3.ReportVersion, content: observeShape(t, s3.ReportPath, []string{"http_requests"})}, {name: "cli", version: cliSurfaceVersion, content: cliSurfaceContent(t)}, } if os.Getenv("FEINT_UPDATE_FROZEN") != "" { @@ -216,12 +218,24 @@ var frozenBody = func() func(t *testing.T, path string) []byte { return func(t *testing.T, path string) []byte { t.Helper() once.Do(func() { - srv, _, err := newServer(nil) + srv, env, err := newServer(nil) if err != nil { buildErr = err return } - ts := httptest.NewServer(srv.Handler()) + flags := &objectStorageFlags{root: filepath.Join(privateStorageDir(t), "bytes"), + budgetMiB: 3, objectMiB: 1, doors: defaultS3Doors} + doors, store, err := mountObjectStorage(t.Context(), srv, flags, env.Log) + if err != nil { + buildErr = err + return + } + defer func() { + if err := store.Close(); err != nil { + buildErr = err + } + }() + ts := httptest.NewServer(doors) defer ts.Close() drive := func(path string, probe bool) error { @@ -251,6 +265,15 @@ var frozenBody = func() func(t *testing.T, path string) []byte { return } } + resS3, err := ts.Client().Get(ts.URL + s3.DoorPath("scaleway", "fr-par") + "missing-bucket/key") + if err != nil { + buildErr = err + return + } + if err := resS3.Body.Close(); err != nil { + buildErr = err + return + } // One armed rule, and one call that fires it. Both halves are // needed: without the rule the faults payload freezes an empty @@ -283,7 +306,7 @@ var frozenBody = func() func(t *testing.T, path string) []byte { return } - for _, path := range []string{"/_feint/health", "/_feint/routes", "/_feint/conformance", "/_feint/trace", "/_feint/faults"} { + for _, path := range []string{"/_feint/health", "/_feint/routes", "/_feint/conformance", "/_feint/trace", "/_feint/faults", s3.ReportPath} { res, err := ts.Client().Get(ts.URL + path) if err != nil { buildErr = err diff --git a/internal/cli/lifecycle.go b/internal/cli/lifecycle.go index db525186..44c0a39b 100644 --- a/internal/cli/lifecycle.go +++ b/internal/cli/lifecycle.go @@ -41,6 +41,7 @@ type serveFlags struct { // in the background, and a mode `start` could not name is a mode the // lifecycle verbs cannot test (#126). strictCatalog string + storage *objectStorageFlags } // args renders the flags back into a `serve` command line. @@ -64,6 +65,9 @@ func (f serveFlags) args() []string { if f.strictCatalog != "" { out = append(out, "--strict-catalog", f.strictCatalog) } + if f.storage != nil { + out = append(out, f.storage.args()...) + } return out } @@ -84,6 +88,7 @@ func bindServeFlags(fs *flag.FlagSet) *serveFlags { // TestStartRelaysTheDeclaredProjectsToServe fails without this. fs.StringVar(&f.projects, "projects", "", "comma-separated project names the emulated account holds, in order") fs.StringVar(&f.strictCatalog, "strict-catalog", "", "refuse a create naming an image, a template or a machine type outside this declared catalogue (JSON); unset accepts any identifier") + f.storage = bindObjectStorageFlags(fs) return f } @@ -97,6 +102,10 @@ func start(args []string, stdout, stderr io.Writer) int { if err := fs.Parse(args); err != nil { return exitError } + if err := flags.storage.validate(); err != nil { + fmt.Fprintf(stderr, "feint: %v\n", err) + return exitError + } if *foreground { // An alias rather than a duplicate, so a container image has a sane diff --git a/internal/cli/object_storage.go b/internal/cli/object_storage.go new file mode 100644 index 00000000..22491846 --- /dev/null +++ b/internal/cli/object_storage.go @@ -0,0 +1,235 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "math" + "net" + "net/http" + "net/url" + "path/filepath" + "strings" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" + "github.com/stephrobert/feint/internal/s3" +) + +const defaultS3Doors = "scaleway/fr-par,exoscale/ch-gva-2" + +type objectStorageFlags struct { + root string + doors string + budgetMiB int64 + objectMiB int64 +} + +func bindObjectStorageFlags(fs *flag.FlagSet) *objectStorageFlags { + f := &objectStorageFlags{} + fs.StringVar(&f.root, "objectstore", "", "opt in to persistent S3 bytes in this private directory (independent of --state)") + fs.StringVar(&f.doors, "s3-doors", defaultS3Doors, "comma-separated provider/region S3 doors; only these prefixes are served") + fs.Int64Var(&f.budgetMiB, "objectstore-budget-mib", 1024, "cumulative retained and reserved blob budget in MiB; deletion does not free blobs") + fs.Int64Var(&f.objectMiB, "s3-object-limit-mib", 256, "maximum decoded simple PUT size in MiB") + return f +} + +func (f *objectStorageFlags) validate() error { + if f.budgetMiB <= 0 || f.budgetMiB > math.MaxInt64>>20 || + f.objectMiB <= 0 || f.objectMiB > (math.MaxInt64-(64<<10))/2>>20 || + f.objectMiB > f.budgetMiB { + return errors.New("object storage requires positive representable MiB limits, with the object limit no larger than the blob budget") + } + if f.root != "" { + absolute, err := filepath.Abs(f.root) + if err != nil { + return fmt.Errorf("object storage root: %w", err) + } + f.root = absolute + } + return nil +} + +func (f *objectStorageFlags) args() []string { + if f.root == "" { + return nil + } + return []string{"--objectstore", f.root, "--s3-doors", f.doors, + "--objectstore-budget-mib", fmt.Sprint(f.budgetMiB), "--s3-object-limit-mib", fmt.Sprint(f.objectMiB)} +} + +type packObjectStorage interface { + ObjectStorage(region string) s3.Profile +} + +func storageProfiles(packs []emulator.Pack, declaration string) ([]s3.Profile, error) { + var profiles []s3.Profile + seen := make(map[string]bool) + for _, entry := range strings.Split(declaration, ",") { + provider, region, ok := strings.Cut(entry, "/") + if !ok || !s3.ValidToken(provider) || !s3.ValidToken(region) || seen[entry] { + return nil, fmt.Errorf("invalid or duplicate S3 door %q; use provider/region tokens", entry) + } + seen[entry] = true + var found bool + for _, p := range packs { + if p.Name() != provider { + continue + } + if storage, ok := p.(packObjectStorage); ok { + profiles = append(profiles, storage.ObjectStorage(region)) + found = true + } + } + if !found { + return nil, fmt.Errorf("provider %q declares no supported S3 dialect", provider) + } + } + return profiles, nil +} + +func mountObjectStorage(ctx context.Context, srv *emulator.Server, flags *objectStorageFlags, log *slog.Logger) (*s3.Doors, *objectstore.Store, error) { + if flags.root == "" { + doors, err := s3.Mount(srv.Handler(), nil, nil, 0, log) + return doors, nil, err + } + profiles, err := storageProfiles(srv.Packs(), flags.doors) + if err != nil { + return nil, nil, err + } + store, err := objectstore.OpenWithLimit(ctx, flags.root, flags.budgetMiB<<20) + if err != nil { + return nil, nil, fmt.Errorf("open object storage: %w", err) + } + doors, err := s3.Mount(srv.Handler(), store, profiles, flags.objectMiB<<20, log) + if err != nil { + return nil, nil, errors.Join(err, store.Close()) + } + return doors, store, nil +} + +func s3DiscoveryURL(endpoint string) (*url.URL, error) { + base, err := url.Parse(endpoint) + if err != nil || (base.Scheme != "http" && base.Scheme != "https") || + !emulator.LoopbackListen(net.JoinHostPort(base.Hostname(), "0")) || base.User != nil || base.RawQuery != "" || + base.Fragment != "" || base.RawPath != "" || (base.Path != "" && base.Path != "/") { + return nil, errors.New("S3 discovery requires an explicit loopback HTTP(S) control-plane endpoint without credentials, query or path") + } + base.Path = s3.ReportPath + return base, nil +} + +func storageListenEndpoint(addr string) (string, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", fmt.Errorf("S3 listen address: %w", err) + } + switch host { + case "", "0.0.0.0": + host = "127.0.0.1" + case "::": + host = "::1" + } + local := net.JoinHostPort(host, port) + // TestS3ListenConfigurationAlwaysProducesUsableLoopbackEndpoints holds this. + if !emulator.LoopbackListen(local) { + return "", errors.New("S3 requires a loopback or wildcard listener; a specific non-loopback listen address cannot serve its loopback-only doors") + } + return "http://" + local, nil +} + +func fetchS3(endpoint string) (report s3.Report, result error) { + base, err := s3DiscoveryURL(endpoint) + if err != nil { + return report, err + } + // Discovery must not follow a redirect to a cloud or use an inherited proxy. + client := &http.Client{Timeout: statusTimeout, Transport: &http.Transport{Proxy: nil}, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + defer client.CloseIdleConnections() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, base.String(), nil) + if err != nil { + return report, err + } + res, err := client.Do(req) + if err != nil { + return report, err + } + defer func() { + if err := res.Body.Close(); err != nil { + result = errors.Join(result, fmt.Errorf("close S3 report: %w", err)) + } + }() + if res.StatusCode != http.StatusOK { + return report, fmt.Errorf("S3 discovery answered %d", res.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(res.Body, (64<<10)+1)) + if err != nil { + return report, fmt.Errorf("read S3 report: %w", err) + } + if len(raw) > 64<<10 { + return report, errors.New("S3 report exceeds 64 KiB") + } + if err := json.Unmarshal(raw, &report); err != nil { + return report, fmt.Errorf("decode S3 report: %w", err) + } + if report.SchemaVersion != s3.ReportVersion || report.Doors == nil || report.HTTPRequests == nil || + (report.Enabled && (report.MaxBlobBytes <= 0 || report.MaxObjectBytes <= 0 || + report.MaxObjectBytes > report.MaxBlobBytes || len(report.Doors) == 0)) || + (!report.Enabled && len(report.Doors) != 0) { + return report, errors.New("S3 report has an unsupported version or inconsistent activation/limits") + } + seen := make(map[string]bool) + for _, door := range report.Doors { + if !s3.ValidToken(door.Provider) || !s3.ValidToken(door.Region) || + door.Path != s3.DoorPath(door.Provider, door.Region) || seen[door.Path] { + return report, errors.New("S3 report declares a noncanonical or duplicate endpoint") + } + seen[door.Path] = true + } + return report, nil +} + +func storageEnvironment(pack packEnv, endpoint, region string, unset bool, vars map[string]string) error { + storage, ok := pack.(packObjectStorage) + if !ok { + return fmt.Errorf("the %s pack declares no S3 environment", pack.Name()) + } + if unset { + profile := storage.ObjectStorage(region) + for _, key := range profile.Variables { + vars[key] = "" + } + return nil + } + report, err := fetchS3(endpoint) + if err != nil { + return err + } + if !report.Enabled { + return errors.New("S3 is disabled; start with --objectstore before exporting an S3 environment") + } + var selected *s3.Door + for _, door := range report.Doors { + if door.Provider == pack.Name() && (region == "" || door.Region == region) { + if selected != nil { + return errors.New("multiple S3 regions are enabled; select one with --s3-region") + } + copy := door + selected = © + } + } + if selected == nil { + return fmt.Errorf("no enabled S3 door for provider %q and selected region %q", pack.Name(), region) + } + profile := storage.ObjectStorage(selected.Region) + for _, key := range profile.Variables { + vars[key] = strings.TrimSuffix(endpoint, "/") + selected.Path + } + vars[profile.RegionVariable] = selected.Region + return nil +} diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go new file mode 100644 index 00000000..e38ae9f4 --- /dev/null +++ b/internal/cli/object_storage_conformance_test.go @@ -0,0 +1,178 @@ +//go:build conformance + +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stephrobert/feint/internal/conformancetest" + "github.com/stephrobert/feint/internal/core/objectstore" + "github.com/stephrobert/feint/internal/s3" +) + +func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { + var guardHits atomic.Int64 + guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + guardHits.Add(1) + http.Error(w, "external requests refused, never forwarded", http.StatusBadGateway) + })) + defer guard.Close() + if err := conformancetest.ProbeProxyGuard(t.Context(), guard.URL); err != nil || guardHits.Load() != 1 { + t.Fatalf("local proxy control did not record its refusal: %v hits=%d", err, guardHits.Load()) + } + aws := conformancetest.AWS(t, guard.URL, "") + binary := filepath.Join(t.TempDir(), "feint") + buildCtx, cancelBuild := context.WithTimeout(t.Context(), 2*time.Minute) + build := exec.CommandContext(buildCtx, "go", "build", "-o", binary, "../../cmd/feint") + if out, err := build.CombinedOutput(); err != nil { + cancelBuild() + t.Fatalf("build the actual binary: %v\n%s", err, out) + } + cancelBuild() + lease, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := lease.Addr().String() + if err := lease.Close(); err != nil { + t.Fatal(err) + } + endpoint := "http://" + addr + work := privateStorageDir(t) + root := filepath.Join(work, "bytes") + run := func(args ...string) []byte { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Env, cmd.Dir = aws.Env, work + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("feint %s: %v\n%s", strings.Join(args, " "), err, out) + } + return out + } + // Cleanup uses the recorded process identity, not a process-name kill. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, binary, "stop", "--addr", addr) + cmd.Env = aws.Env + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("stop the owned S3 witness: %v\n%s", err, out) + } + }) + run("start", "--addr", addr, "--vm", "off", "--objectstore", "bytes", + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1") + var first statusReport + if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &first); err != nil || + !first.Running || first.PID <= 0 || first.ObjectStorage == nil || + first.ObjectStorage.MaxBlobBytes != 3<<20 || first.ObjectStorage.MaxObjectBytes != 1<<20 { + t.Fatalf("real process/configuration did not become ready: %+v %v", first, err) + } + if len(first.ObjectStorage.Doors) != 2 { + t.Fatal("actual serve did not mount both declared doors") + } + for _, provider := range []string{"scaleway", "exoscale"} { + out := run("env", provider, "--s3", "--endpoint", endpoint) + found := false + for _, door := range first.ObjectStorage.Doors { + if door.Provider == provider && bytes.Contains(out, []byte(endpoint+door.Path)) { + found = true + } + } + if !found { + t.Fatalf("actual env lost the configured %s S3 endpoint: %q", provider, out) + } + } + payload := bytes.Repeat([]byte{0, 255, 'f', 'e', 'i', 'n', 't', '\r', '\n'}, 8192) + key := "folder//a/../percent%2F?plus+#" + bucket := "feint-mounted-bytes" + input, output := filepath.Join(aws.Home, "input"), filepath.Join(aws.Home, "output") + if err := os.WriteFile(input, payload, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(payload) + call := func(door s3.Door, args ...string) []byte { + t.Helper() + return aws.Run(t, append([]string{"--endpoint-url", endpoint + door.Path, "--region", door.Region, + "--no-cli-pager", "s3api"}, args...)...) + } + get := func(door s3.Door, expected []byte) { + t.Helper() + result := call(door, "get-object", "--bucket", bucket, "--key", key, "--checksum-mode", "ENABLED", output) + got, err := os.ReadFile(output) + var meta struct{ ChecksumSHA256 string } + hash := sha256.Sum256(expected) + if err != nil || !bytes.Equal(got, expected) || json.Unmarshal(result, &meta) != nil || + meta.ChecksumSHA256 != base64.StdEncoding.EncodeToString(hash[:]) { + t.Fatalf("official client did not return identical verified bytes: read=%v result=%q", err, result) + } + } + for i, door := range first.ObjectStorage.Doors { + call(door, "create-bucket", "--bucket", bucket) + if i == 1 { + if err := os.WriteFile(input, []byte("separate-sos-scope"), 0600); err != nil { + t.Fatal(err) + } + } + call(door, "put-object", "--bucket", bucket, "--key", key, "--body", input, + "--content-type", "application/octet-stream", "--checksum-algorithm", "CRC32") + } + get(first.ObjectStorage.Doors[0], payload) + get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + // Restart the initially relative-root invocation from a different cwd. + work = privateStorageDir(t) + run("restart", "--addr", addr) + var restarted statusReport + if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &restarted); err != nil || + !restarted.Running || restarted.PID == first.PID || restarted.ObjectStorage == nil || + restarted.ObjectStorage.MaxBlobBytes != 3<<20 || restarted.ObjectStorage.MaxObjectBytes != 1<<20 { + t.Fatalf("restart lost the process or persisted configuration: %+v %v", restarted, err) + } + get(first.ObjectStorage.Doors[0], payload) + get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + // Inspect only our private store while the owner is stopped: GET of the + // same cleaned key twice would otherwise masquerade as key preservation. + run("stop", "--addr", addr) + store, err := objectstore.OpenWithLimit(t.Context(), root, 3<<20) + if err != nil { + t.Fatal(err) + } + for _, door := range first.ObjectStorage.Doors { + if _, err := store.HeadObject(t.Context(), door.Provider+"/"+door.Region+"/"+bucket, key); err != nil { + _ = store.Close() + t.Fatal("the real client key was cleaned or decoded twice:", err) + } + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + run("start", "--addr", addr, "--vm", "off", "--objectstore", root, + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1") + for _, door := range first.ObjectStorage.Doors { + call(door, "head-object", "--bucket", bucket, "--key", key) + call(door, "delete-object", "--bucket", bucket, "--key", key) + call(door, "delete-bucket", "--bucket", bucket) + } + if guardHits.Load() != 1 { + t.Fatalf("official client reached the refusing egress proxy: delta=%d", guardHits.Load()-1) + } + t.Logf("actual feint pid %d -> %d; two same-port namespaces; preserved key; bytes=%d sha256=%s; stop/reopen/restart; proxy delta=0 (not a network namespace)", + first.PID, restarted.PID, len(payload), hex.EncodeToString(sum[:])) +} diff --git a/internal/cli/object_storage_test.go b/internal/cli/object_storage_test.go new file mode 100644 index 00000000..262d05db --- /dev/null +++ b/internal/cli/object_storage_test.go @@ -0,0 +1,237 @@ +package cli + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/s3" +) + +func privateStorageDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + return dir +} + +func TestStartPreservesObjectStorageFlagsAndAbsoluteRoot(t *testing.T) { + fs := newFlagSet("start") + flags := bindServeFlags(fs) + if err := fs.Parse([]string{"--objectstore", "relative-bytes", "--s3-doors", "scaleway/fr-par", + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1"}); err != nil { + t.Fatal(err) + } + if err := flags.storage.validate(); err != nil { + t.Fatal(err) + } + if !filepath.IsAbs(flags.storage.root) { + t.Fatal("a restart from another directory would select another object store") + } + replaySet := newFlagSet("start") + replay := bindServeFlags(replaySet) + if err := replaySet.Parse(flags.args()[1:]); err != nil { + t.Fatal(err) + } + if *replay.storage != *flags.storage { + t.Fatalf("restart loses storage settings: %+v versus %+v", replay.storage, flags.storage) + } +} + +func TestObjectStorageValidatesDefaultsAndLimitsWithoutStartingAnything(t *testing.T) { + fs := newFlagSet("serve") + defaults := bindObjectStorageFlags(fs) + if err := defaults.validate(); err != nil || defaults.budgetMiB<<20 != 1<<30 || defaults.objectMiB<<20 != 256<<20 { + t.Fatalf("approved defaults moved: %+v %v", defaults, err) + } + for _, limits := range [][2]int64{{0, 1}, {-1, 1}, {1, 0}, {1, 2}, {1 << 44, 1}, {1 << 42, 1 << 42}} { + flags := objectStorageFlags{budgetMiB: limits[0], objectMiB: limits[1]} + if err := flags.validate(); err == nil { + t.Fatalf("invalid/overflowing limits accepted: %v", limits) + } + } + if len(defaults.args()) != 0 { + t.Fatal("disabled byte storage rendered activation flags") + } +} + +func TestS3ListenConfigurationAlwaysProducesUsableLoopbackEndpoints(t *testing.T) { + for _, tc := range []struct{ addr, endpoint string }{ + {"127.0.0.1:4599", "http://127.0.0.1:4599"}, + {"localhost:4599", "http://localhost:4599"}, + {":4599", "http://127.0.0.1:4599"}, + {"0.0.0.0:4599", "http://127.0.0.1:4599"}, + {"[::]:4599", "http://[::1]:4599"}, + {"[::1]:4599", "http://[::1]:4599"}, + } { + if got, err := storageListenEndpoint(tc.addr); err != nil || got != tc.endpoint { + t.Fatalf("%s advertised an unusable endpoint: %q %v", tc.addr, got, err) + } + } + for _, addr := range []string{"192.0.2.10:4599", "cloud.example:4599", "invalid-address"} { + if _, err := storageListenEndpoint(addr); err == nil { + t.Fatalf("S3 accepted a listener that cannot receive loopback peers: %s", addr) + } + } +} + +func storageServer(t *testing.T, enabled bool, declaration string) (*httptest.Server, *s3.Doors) { + t.Helper() + srv, env, err := newServer(nil) + if err != nil { + t.Fatal(err) + } + flags := &objectStorageFlags{budgetMiB: 3, objectMiB: 1, doors: declaration} + if enabled { + flags.root = filepath.Join(privateStorageDir(t), "bytes") + } + doors, store, err := mountObjectStorage(t.Context(), srv, flags, env.Log) + if err != nil { + t.Fatal(err) + } + if store != nil { + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }) + } + ts := httptest.NewServer(doors) + t.Cleanup(ts.Close) + return ts, doors +} + +func TestS3DeclarationsRejectUnknownMalformedAndDuplicateDoors(t *testing.T) { + srv, _, err := newServer(nil) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"", "scaleway", "scaleway/fr-par,scaleway/fr-par", "outscale/eu-west-2", + "unknown/fr-par", "scaleway/FR-PAR", "scaleway/fr-par/extra", "scaleway/../fr-par"} { + if _, err := storageProfiles(srv.Packs(), value); err == nil { + t.Fatalf("invalid or unsupported declaration accepted: %q", value) + } + } + profiles, err := storageProfiles(srv.Packs(), defaultS3Doors) + if err != nil || len(profiles) != 2 { + t.Fatalf("approved explicit doors rejected: %+v %v", profiles, err) + } +} + +func TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline(t *testing.T) { + ts, _ := storageServer(t, true, defaultS3Doors+",scaleway/nl-ams") + var out, note strings.Builder + call := func(args ...string) int { + out.Reset() + note.Reset() + return envCommand(args, &out, ¬e) + } + if code := call("scaleway", "--s3", "--endpoint", ts.URL); code != 1 || out.Len() != 0 { + t.Fatalf("ambiguous regions printed a success-shaped environment: %d %q", code, out.String()) + } + for _, shell := range []string{"bash", "fish", "powershell"} { + if code := call("scaleway", "--s3", "--s3-region", "fr-par", "--endpoint", ts.URL, "--shell", shell); code != 0 || + !strings.Contains(out.String(), ts.URL+s3.DoorPath("scaleway", "fr-par")) || + strings.Contains(out.String(), "not certified") || !strings.Contains(note.String(), "not certified") { + t.Fatalf("live selected S3 environment or stderr contract failed: %d %q %q", code, out.String(), note.String()) + } + } + if code := call("exoscale", "--s3", "--endpoint", ts.URL); code != 0 || + !strings.Contains(out.String(), "EXOSCALE_SOS_ENDPOINT") || !strings.Contains(out.String(), "EXOSCALE_STORAGE_API_ENDPOINT") { + t.Fatalf("SOS configurable endpoint names were not wired: %d %q", code, out.String()) + } + if code := call("scaleway", "--s3", "--s3-region", "missing", "--endpoint", ts.URL); code != 1 || out.Len() != 0 { + t.Fatal("an undeclared region exported a fallback endpoint") + } + if code := call("scaleway", "--s3-region", "fr-par"); code != 1 || out.Len() != 0 { + t.Fatal("a region without S3 activation was silently ignored") + } + if code := call("scaleway", "--s3", "--unset", "--endpoint", "http://127.0.0.1:1"); code != 0 || + !strings.Contains(out.String(), "unset SCW_S3_ENDPOINT") { + t.Fatal("unset requires a running emulator or forgets the S3 variable") + } + if code := call("scaleway", "--unset"); code != 0 || !strings.Contains(out.String(), "unset SCW_S3_ENDPOINT") { + t.Fatal("ordinary unset leaves the new S3 endpoint behind") + } + if code := call("scaleway", "--endpoint", ts.URL); code != 0 || strings.Contains(out.String(), "SCW_S3_ENDPOINT") { + t.Fatal("ordinary offline-compatible env unexpectedly activates S3") + } +} + +func TestS3DisabledStateIsHonestInEnvStatusAndDoctor(t *testing.T) { + t.Setenv("XDG_RUNTIME_DIR", t.TempDir()) + ts, _ := storageServer(t, false, "") + var out, note strings.Builder + if code := envCommand([]string{"scaleway", "--s3", "--endpoint", ts.URL}, &out, ¬e); code != 1 || + out.Len() != 0 || !strings.Contains(note.String(), "disabled") { + t.Fatalf("disabled S3 emitted an environment: %d %q %q", code, out.String(), note.String()) + } + addr := strings.TrimPrefix(ts.URL, "http://") + if c := checkObjectStorage(addr); c.state != verdictFail || !strings.Contains(c.detail, "disabled") { + t.Fatalf("doctor claims disabled bytes are available: %+v", c) + } + out.Reset() + note.Reset() + if code := status([]string{"--addr", addr, "--format", "json"}, &out, ¬e); code != 0 { + t.Fatal("status failed:", note.String()) + } + var report statusReport + if err := json.Unmarshal([]byte(out.String()), &report); err != nil || !report.Running || + report.ObjectStorage == nil || report.ObjectStorage.Enabled || report.ObjectStorageError != "" { + t.Fatalf("status loses disabled activation: %+v %v", report, err) + } +} + +func TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints(t *testing.T) { + var destinationHits int + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + destinationHits++ + _, _ = io.WriteString(w, "unexpected") + })) + defer destination.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, destination.URL, http.StatusFound) + })) + defer source.Close() + if _, err := fetchS3(source.URL); err == nil || destinationHits != 0 { + t.Fatal("discovery followed a redirect") + } + for _, endpoint := range []string{"https://s3.fr-par.scw.cloud", "http://127.0.0.1:1/extra", + "http://user:password@127.0.0.1:1", "http://127.0.0.1:1?q=1", "http://127.0.0.1:1#fragment"} { + if _, err := s3DiscoveryURL(endpoint); err == nil { + t.Fatalf("unsafe discovery endpoint accepted: %q", endpoint) + } + } + if target, err := s3DiscoveryURL(source.URL + "/"); err != nil || target.Path != s3.ReportPath { + t.Fatalf("valid loopback discovery URL refused: %v %v", target, err) + } + valid := s3.Report{SchemaVersion: s3.ReportVersion, Enabled: true, MaxBlobBytes: 3, MaxObjectBytes: 1, + Doors: []s3.Door{{Provider: "scaleway", Region: "fr-par", Path: s3.DoorPath("scaleway", "fr-par")}}, + HTTPRequests: map[string]uint64{}} + for _, modify := range []func(*s3.Report){ + func(r *s3.Report) { r.SchemaVersion++ }, + func(r *s3.Report) { r.MaxObjectBytes = 4 }, + func(r *s3.Report) { r.Enabled = false }, + func(r *s3.Report) { r.Doors[0].Path = "/foreign" }, + func(r *s3.Report) { r.Doors = append(r.Doors, r.Doors[0]) }, + } { + report := valid + report.Doors = append([]s3.Door{}, valid.Doors...) + modify(&report) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(report) + })) + if _, err := fetchS3(server.URL); err == nil { + server.Close() + t.Fatalf("invalid report trusted: %+v", report) + } + server.Close() + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 4d60db73..220abad4 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -10,6 +10,7 @@ import ( "github.com/stephrobert/feint/internal/core/emulator" "github.com/stephrobert/feint/internal/core/machine" + "github.com/stephrobert/feint/internal/s3" ) // What is running, and how much of the real API it is standing in for. @@ -57,17 +58,19 @@ type routeResponse struct { // statusReport is what status prints, and what --format json emits verbatim. type statusReport struct { - Running bool `json:"running"` - Addr string `json:"addr"` - PID int `json:"pid,omitempty"` - Started string `json:"started_at,omitempty"` - Stale string `json:"stale,omitempty"` - Log string `json:"log,omitempty"` - Machines string `json:"machines,omitempty"` - Resources int `json:"resources,omitempty"` - Products []productStatus `json:"products,omitempty"` - Providers []providerStatus `json:"providers,omitempty"` - Contracts []string `json:"contracts,omitempty"` + Running bool `json:"running"` + Addr string `json:"addr"` + PID int `json:"pid,omitempty"` + Started string `json:"started_at,omitempty"` + Stale string `json:"stale,omitempty"` + Log string `json:"log,omitempty"` + Machines string `json:"machines,omitempty"` + Resources int `json:"resources,omitempty"` + Products []productStatus `json:"products,omitempty"` + Providers []providerStatus `json:"providers,omitempty"` + Contracts []string `json:"contracts,omitempty"` + ObjectStorage *s3.Report `json:"object_storage,omitempty"` + ObjectStorageError string `json:"object_storage_error,omitempty"` } type providerStatus struct { @@ -123,6 +126,17 @@ func status(args []string, stdout, stderr io.Writer) int { if conf, confErr := fetchConformance(*addr); confErr == nil { report.Contracts = conf.Contracts } + target, storageErr := storageListenEndpoint(*addr) + var storage s3.Report + if storageErr == nil { + storage, storageErr = fetchS3(target) + } + if storageErr != nil { + report.ObjectStorageError = storageErr.Error() + fmt.Fprintf(stderr, "warning: object storage availability is unknown: %v\n", storageErr) + } else { + report.ObjectStorage = &storage + } } report.Products = coverageStatus(*coverageDir) @@ -159,6 +173,19 @@ func writeStatusText(w io.Writer, r statusReport, health healthResponse) { fmt.Fprintf(w, " (pid %d, since %s)", r.PID, r.Started) } fmt.Fprintln(w) + if r.ObjectStorageError != "" { + fmt.Fprintf(w, " S3 unknown (%s)\n", r.ObjectStorageError) + } else if r.ObjectStorage != nil { + if !r.ObjectStorage.Enabled { + fmt.Fprintln(w, " S3 disabled") + } else { + fmt.Fprintf(w, " S3 %d door(s), %d retained/reserved blob bytes maximum, %d bytes per PUT\n", + len(r.ObjectStorage.Doors), r.ObjectStorage.MaxBlobBytes, r.ObjectStorage.MaxObjectBytes) + for _, door := range r.ObjectStorage.Doors { + fmt.Fprintf(w, " %s\n", door.Path) + } + } + } if r.Stale != "" { fmt.Fprintf(w, " warning: %s\n", r.Stale) } diff --git a/internal/cli/testdata/frozen/cli.json b/internal/cli/testdata/frozen/cli.json index a44a4974..5782d1a3 100644 --- a/internal/cli/testdata/frozen/cli.json +++ b/internal/cli/testdata/frozen/cli.json @@ -4457,6 +4457,236 @@ ] } } + }, + { + "schema_version": 26, + "content": { + "exit_codes": { + "drift": 2, + "error": 1, + "ok": 0 + }, + "verbs": { + "catalog": [ + "--format" + ], + "clean": [ + "--check", + "--closing", + "--doorstep", + "--force", + "--format", + "--vm" + ], + "corpus": [ + "--accepted", + "--against-cloud", + "--bind", + "--check", + "--credential", + "--dir", + "--dry-run", + "--endpoint", + "--file", + "--format", + "--mark-stale", + "--timeout" + ], + "coverage": [ + "--artefact", + "--axis", + "--baseline", + "--contract", + "--contract-only", + "--document", + "--evidence", + "--fail-on-unknown", + "--format", + "--gaps", + "--observed", + "--products", + "--provider", + "--sdk", + "--write-baseline" + ], + "docs": [ + "--check", + "--client-pins", + "--confidence", + "--contracts", + "--coverage", + "--file", + "--install", + "--limits", + "--proved", + "--routes", + "--screenshots", + "--ui-manifest", + "--workflow" + ], + "doctor": [ + "--addr", + "--s3", + "--vm" + ], + "down": [ + "--file", + "--keep-emulator" + ], + "env": [ + "--client", + "--endpoint", + "--s3", + "--s3-region", + "--shell", + "--unset" + ], + "evidence": [ + "--allow-narrowing", + "--contracts", + "--endpoint", + "--join", + "--out", + "--reshape", + "--shapes", + "--suites" + ], + "evidence baseline": [ + "--axes", + "--evidence", + "--out" + ], + "evidence verify": [ + "--accepted", + "--baseline", + "--evidence" + ], + "images": [ + "--check", + "--only", + "--vm" + ], + "logs": [ + "--addr", + "-f", + "-n" + ], + "probe": [ + "--contracts", + "--endpoint", + "--provider" + ], + "proxy": [ + "--addr", + "--expose-to-network", + "--forward", + "--intercept", + "--max-body", + "--provider", + "--queue", + "--record", + "--upstream" + ], + "replay": [ + "--endpoint", + "--format", + "--refusals-only", + "--timeout" + ], + "restart": [ + "--addr", + "--timeout" + ], + "serve": [ + "--addr", + "--cleanup", + "--consistency", + "--contracts", + "--coverage", + "--expose-to-network", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-object-limit-mib", + "--shapes", + "--state", + "--strict-catalog", + "--vm" + ], + "shapes": [ + "--check", + "--dir", + "--dry-run", + "--profile", + "--provider", + "--record" + ], + "snapshot list": [ + "--format" + ], + "snapshot load": [ + "--addr" + ], + "snapshot save": [ + "--addr", + "--force" + ], + "start": [ + "--addr", + "--cleanup", + "--contracts", + "--detach", + "--foreground", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-object-limit-mib", + "--state", + "--strict-catalog", + "--timeout", + "--vm" + ], + "status": [ + "--addr", + "--coverage", + "--format" + ], + "stop": [ + "--addr", + "--timeout" + ], + "transcript": [ + "--against", + "--contract", + "--format", + "--sanitise", + "--shape" + ], + "ui": [ + "--addr", + "--print" + ], + "up": [ + "--file", + "--no-iac", + "--runtime", + "--timeout" + ], + "version": [ + "--check" + ], + "wait": [ + "--addr", + "--timeout" + ] + } + } } ] } diff --git a/internal/cli/testdata/frozen/s3.json b/internal/cli/testdata/frozen/s3.json new file mode 100644 index 00000000..b78dfc62 --- /dev/null +++ b/internal/cli/testdata/frozen/s3.json @@ -0,0 +1,59 @@ +{ + "history": [ + { + "schema_version": 1, + "content": { + "fields": [ + { + "path": "doors", + "type": "array" + }, + { + "path": "doors[]", + "type": "object" + }, + { + "path": "doors[].path", + "type": "string" + }, + { + "path": "doors[].provider", + "type": "string" + }, + { + "path": "doors[].region", + "type": "string" + }, + { + "path": "enabled", + "type": "bool" + }, + { + "path": "http_requests", + "type": "object" + }, + { + "path": "http_requests.*", + "type": "array" + }, + { + "path": "http_requests.*[]", + "type": "number" + }, + { + "path": "max_blob_bytes", + "type": "number" + }, + { + "path": "max_object_bytes", + "type": "number" + }, + { + "path": "schema_version", + "type": "number" + } + ] + } + } + ] +} diff --git a/internal/conformancetest/aws.go b/internal/conformancetest/aws.go new file mode 100644 index 00000000..89f0345b --- /dev/null +++ b/internal/conformancetest/aws.go @@ -0,0 +1,107 @@ +//go:build conformance + +// Package conformancetest contains the isolated, pinned official-client harness. +package conformancetest + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// AWSCLI is an isolated AWS CLI installation, not a host credential profile. +type AWSCLI struct { + Path string + Home string + Env []string +} + +// AWS builds a private environment from the repository's public fake keys. +func AWS(t *testing.T, proxy, ca string) AWSCLI { + t.Helper() + binary, err := exec.LookPath("aws") + if err != nil { + t.Fatal("install AWS CLI 2.32.17 for this explicit real-client proof:", err) + } + home := t.TempDir() + if err := os.Chmod(home, 0700); err != nil { //nolint:gosec // a private directory, not a file + t.Fatal(err) + } + creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") + if err != nil { + t.Fatal(err) + } + fake := make(map[string]string) + for _, line := range strings.Split(string(creds), "\n") { + if key, value, ok := strings.Cut(line, "="); ok && strings.HasPrefix(key, "SCW_") { + fake[key] = value + } + } + if fake["SCW_ACCESS_KEY"] == "" || fake["SCW_SECRET_KEY"] == "" { + t.Fatal("canonical public fake credentials are absent") + } + cli := AWSCLI{Path: binary, Home: home, Env: []string{ + "HOME=" + home, "PATH=" + filepath.Dir(binary) + ":/usr/bin:/bin", + "XDG_STATE_HOME=" + home, "XDG_RUNTIME_DIR=" + home, + "AWS_ACCESS_KEY_ID=" + fake["SCW_ACCESS_KEY"], "AWS_SECRET_ACCESS_KEY=" + fake["SCW_SECRET_KEY"], + "AWS_DEFAULT_REGION=fr-par", "AWS_EC2_METADATA_DISABLED=true", + "AWS_CONFIG_FILE=" + filepath.Join(home, "config"), "AWS_SHARED_CREDENTIALS_FILE=" + filepath.Join(home, "credentials"), + "AWS_PAGER=", "AWS_CLI_AUTO_PROMPT=off", "AWS_MAX_ATTEMPTS=1", + "HTTP_PROXY=" + proxy, "HTTPS_PROXY=" + proxy, "ALL_PROXY=" + proxy, + "NO_PROXY=127.0.0.1,localhost,::1", "LANG=C.UTF-8", + }} + if ca != "" { + cli.Env = append(cli.Env, "AWS_CA_BUNDLE="+ca) + } + version := string(cli.Run(t, "--version")) + if !strings.HasPrefix(version, "aws-cli/2.32.17 ") { + t.Fatal("proof requires the measured AWS CLI 2.32.17, got:", version) + } + t.Log(strings.TrimSpace(version)) + return cli +} + +// Run executes the pinned client with a bounded lifetime and no shell. +func (c AWSCLI) Run(t *testing.T, args ...string) []byte { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, c.Path, args...) //nolint:gosec // pinned official client, explicit arguments, no shell + cmd.Env = c.Env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("AWS CLI %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return out +} + +// ProbeProxyGuard sends a local CONNECT refusal control, never a cloud request. +func ProbeProxyGuard(ctx context.Context, endpoint string) error { + proxy, err := url.Parse(endpoint) + if err != nil { + return err + } + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxy)}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + r, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://s3.fr-par.scw.cloud/never-forward", nil) + if err != nil { + return err + } + response, err := client.Do(r) + if response != nil { + if closeErr := response.Body.Close(); closeErr != nil { + return closeErr + } + } + if err == nil { + return fmt.Errorf("guard unexpectedly permitted CONNECT") + } + return nil +} diff --git a/internal/core/objectstore/budget.go b/internal/core/objectstore/budget.go index 75fb2dba..3950ab2f 100644 --- a/internal/core/objectstore/budget.go +++ b/internal/core/objectstore/budget.go @@ -9,6 +9,15 @@ import ( "syscall" ) +// BlobLimit returns the configured cumulative blob-byte limit, or zero for an +// unlimited store. The limit is immutable for the lifetime of an open instance. +func (s *Store) BlobLimit() int64 { + if s.budget == nil { + return 0 + } + return s.budget.limit +} + // ErrQuota distinguishes a byte-budget refusal from corrupt or invalid data. var ErrQuota = errors.New("object store blob byte budget exceeded") diff --git a/internal/providers/exoscale/object_storage.go b/internal/providers/exoscale/object_storage.go new file mode 100644 index 00000000..a6247682 --- /dev/null +++ b/internal/providers/exoscale/object_storage.go @@ -0,0 +1,10 @@ +package exoscale + +import "github.com/stephrobert/feint/internal/s3" + +// ObjectStorage declares the measured SOS dialect and configurable CLI endpoints. +// It does not advertise control-plane bucket usage or multipart support. +func (p *Pack) ObjectStorage(region string) s3.Profile { + return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ExoscaleMissingBucket, + Variables: []string{"EXOSCALE_SOS_ENDPOINT", "EXOSCALE_STORAGE_API_ENDPOINT"}, RegionVariable: "EXOSCALE_ZONE"} +} diff --git a/internal/providers/scaleway/object_storage.go b/internal/providers/scaleway/object_storage.go new file mode 100644 index 00000000..7727e893 --- /dev/null +++ b/internal/providers/scaleway/object_storage.go @@ -0,0 +1,10 @@ +package scaleway + +import "github.com/stephrobert/feint/internal/s3" + +// ObjectStorage declares the measured S3 dialect and configurable CLI endpoint. +// It does not advertise Terraform redirection or ancillary S3 operations. +func (p *Pack) ObjectStorage(region string) s3.Profile { + return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ScalewayMissingBucket, + Variables: []string{"SCW_S3_ENDPOINT"}, RegionVariable: "SCW_DEFAULT_REGION"} +} diff --git a/internal/s3/client_conformance_test.go b/internal/s3/client_conformance_test.go index c575d957..3c9b37e5 100644 --- a/internal/s3/client_conformance_test.go +++ b/internal/s3/client_conformance_test.go @@ -10,10 +10,8 @@ import ( "encoding/hex" "encoding/json" "encoding/pem" - "fmt" "net/http" "net/http/httptest" - "net/url" "os" "os/exec" "path/filepath" @@ -22,22 +20,20 @@ import ( "sync/atomic" "testing" "time" + + "github.com/stephrobert/feint/internal/conformancetest" ) func TestOfficialAWSCLIBytes(t *testing.T) { - aws, err := exec.LookPath("aws") - if err != nil { - t.Fatal("install AWS CLI 2.32.17 for this explicit real-client proof:", err) - } s, store := fixture(t) var guardHits atomic.Int64 guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { guardHits.Add(1) - http.Error(w, "feint conformance: external requests are refused, never forwarded", 502) + http.Error(w, "feint conformance: external requests are refused, never forwarded", http.StatusBadGateway) })) defer guard.Close() // A positive control proves the proxy actually records and refuses CONNECT. - if err := probeProxyGuard(t.Context(), guard.URL); err != nil { + if err := conformancetest.ProbeProxyGuard(t.Context(), guard.URL); err != nil { t.Fatal(err) } if guardHits.Load() != 1 { @@ -74,49 +70,10 @@ func TestOfficialAWSCLIBytes(t *testing.T) { if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: endpoint.Certificate().Raw}), 0600); err != nil { t.Fatal(err) } - creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") - if err != nil { - t.Fatal(err) - } - fake := make(map[string]string) - for _, line := range strings.Split(string(creds), "\n") { - if key, value, ok := strings.Cut(line, "="); ok && strings.HasPrefix(key, "SCW_") { - fake[key] = value - } - } - if fake["SCW_ACCESS_KEY"] == "" || fake["SCW_SECRET_KEY"] == "" { - t.Fatal("canonical public fake credentials are absent") - } - env := []string{ - "HOME=" + home, "PATH=" + filepath.Dir(aws) + ":/usr/bin:/bin", - "AWS_ACCESS_KEY_ID=" + fake["SCW_ACCESS_KEY"], "AWS_SECRET_ACCESS_KEY=" + fake["SCW_SECRET_KEY"], - "AWS_DEFAULT_REGION=fr-par", "AWS_EC2_METADATA_DISABLED=true", - "AWS_CONFIG_FILE=" + filepath.Join(home, "config"), "AWS_SHARED_CREDENTIALS_FILE=" + filepath.Join(home, "credentials"), - "AWS_PAGER=", "AWS_CLI_AUTO_PROMPT=off", "AWS_MAX_ATTEMPTS=1", - "AWS_CA_BUNDLE=" + ca, - "HTTP_PROXY=" + guard.URL, "HTTPS_PROXY=" + guard.URL, "ALL_PROXY=" + guard.URL, - "NO_PROXY=127.0.0.1,localhost,::1", "LANG=C.UTF-8", - } - run := func(args ...string) []byte { - t.Helper() - ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, aws, args...) - cmd.Env = env - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("AWS CLI %s failed: %v\n%s", strings.Join(args, " "), err, out) - } - return out - } - version := string(run("--version")) - if !strings.HasPrefix(version, "aws-cli/2.32.17 ") { - t.Fatal("proof requires the measured AWS CLI 2.32.17, got:", version) - } - t.Log(strings.TrimSpace(version)) + cli := conformancetest.AWS(t, guard.URL, ca) call := func(args ...string) []byte { t.Helper() - return run(append([]string{"--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api"}, args...)...) + return cli.Run(t, append([]string{"--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api"}, args...)...) } payload := bytes.Repeat([]byte{0, 255, 'f', 'e', 'i', 'n', 't', '\r', '\n'}, 8192) input, output := filepath.Join(home, "input"), filepath.Join(home, "output") @@ -150,9 +107,9 @@ func TestOfficialAWSCLIBytes(t *testing.T) { // Alter only the response checksum, retaining the same successful bytes. corruptChecksum.Store(true) ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) - cmd := exec.CommandContext(ctx, aws, "--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api", + cmd := exec.CommandContext(ctx, cli.Path, "--endpoint-url", endpoint.URL, "--no-cli-pager", "s3api", "get-object", "--bucket", "feint-byte-proof", "--key", "proof/data", "--checksum-mode", "ENABLED", filepath.Join(home, "corrupt-output")) - cmd.Env = env + cmd.Env = cli.Env control, controlErr := cmd.CombinedOutput() cancel() corruptChecksum.Store(false) @@ -200,26 +157,3 @@ func (w *statusWriter) WriteHeader(status int) { } w.ResponseWriter.WriteHeader(status) } - -func probeProxyGuard(ctx context.Context, endpoint string) error { - proxy, err := url.Parse(endpoint) - if err != nil { - return err - } - client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxy)}, Timeout: 5 * time.Second} - defer client.CloseIdleConnections() - r, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://s3.fr-par.scw.cloud/never-forward", nil) - if err != nil { - return err - } - response, err := client.Do(r) - if response != nil { - if closeErr := response.Body.Close(); closeErr != nil { - return closeErr - } - } - if err == nil { - return fmt.Errorf("guard unexpectedly permitted CONNECT") - } - return nil -} diff --git a/internal/s3/doors.go b/internal/s3/doors.go new file mode 100644 index 00000000..4fb41966 --- /dev/null +++ b/internal/s3/doors.go @@ -0,0 +1,211 @@ +package s3 + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +// ReportPath is the separate S3 configuration/evidence endpoint. Its counters +// describe HTTP requests, not official-client proof or control-plane coverage. +const ReportPath = "/_feint/s3" + +// ReportVersion versions the S3 report independently of control-plane schemas. +const ReportVersion = 1 + +// UploadReadTimeout bounds a simple PUT body without changing the runtime's +// independent write deadline or the control-plane request-body policy. +const UploadReadTimeout = 30 * time.Second + +// Profile is a provider's declared wire dialect and client environment names. +// Variables stay local to the composition root, never taken from an HTTP report. +type Profile struct { + Provider string + Region string + MissingBucket MissingBucketShape + Variables []string + RegionVariable string +} + +// Door describes an enabled path-style endpoint, without filesystem paths. +type Door struct { + Provider string `json:"provider"` + Region string `json:"region"` + Path string `json:"path"` +} + +// Report owns the wire shape consumed by env/status/doctor and conformance. +type Report struct { + SchemaVersion int `json:"schema_version"` + Enabled bool `json:"enabled"` + MaxBlobBytes int64 `json:"max_blob_bytes"` + MaxObjectBytes int64 `json:"max_object_bytes"` + Doors []Door `json:"doors"` + HTTPRequests map[string]uint64 `json:"http_requests"` +} + +// DoorPath returns the declared endpoint prefix; provider and region must be +// validated tokens before using it to dispatch a request or export a variable. +func DoorPath(provider, region string) string { + return ReportPath + "/" + provider + "/" + region + "/" +} + +// ValidToken limits namespace components to short lower-case ASCII tokens. +func ValidToken(value string) bool { + if value == "" || len(value) > 32 { + return false + } + for _, c := range value { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' { + return false + } + } + return true +} + +// Doors dispatches declared S3 paths before any path-cleaning control-plane mux. +// The caller must keep its outer Host/Origin guard and own store shutdown. +type Doors struct { + next http.Handler + log *slog.Logger + handlers map[string]*Server + mu sync.Mutex + report Report +} + +// Mount composes a control-plane handler and explicitly selected S3 doors. +// A nil store and no profiles publishes disabled S3 without activating storage. +func Mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxObjectBytes int64, log *slog.Logger) (*Doors, error) { + if next == nil || log == nil || (store == nil) != (len(profiles) == 0) { + return nil, errors.New("s3: handler, logger and consistent explicit storage activation are required") + } + d := &Doors{next: next, log: log, handlers: make(map[string]*Server), report: Report{ + SchemaVersion: ReportVersion, Enabled: store != nil, Doors: []Door{}, HTTPRequests: make(map[string]uint64), + }} + if store == nil { + return d, nil + } + maxBlobBytes := store.BlobLimit() + if maxBlobBytes <= 0 || maxObjectBytes > maxBlobBytes { + return nil, errors.New("s3: positive blob budget must cover the per-object limit") + } + d.report.MaxBlobBytes, d.report.MaxObjectBytes = maxBlobBytes, maxObjectBytes + for _, p := range profiles { + if !ValidToken(p.Provider) || !ValidToken(p.Region) { + return nil, errors.New("s3: invalid declared provider or region") + } + path := DoorPath(p.Provider, p.Region) + if _, exists := d.handlers[path]; exists { + return nil, fmt.Errorf("s3: duplicate door %s", path) + } + server, err := New(store, Config{Region: p.Region, MaxObjectBytes: maxObjectBytes, + MissingBucket: p.MissingBucket, Namespace: p.Provider + "/" + p.Region, Log: log}) + if err != nil { + return nil, err + } + d.handlers[path] = server + d.report.Doors = append(d.report.Doors, Door{Provider: p.Provider, Region: p.Region, Path: path}) + } + return d, nil +} + +// Snapshot returns detached report data so readers cannot mutate live evidence. +func (d *Doors) Snapshot() Report { + d.mu.Lock() + defer d.mu.Unlock() + report := d.report + report.Doors = append([]Door{}, d.report.Doors...) + report.HTTPRequests = make(map[string]uint64, len(d.report.HTTPRequests)) + for key, value := range d.report.HTTPRequests { + report.HTTPRequests[key] = value + } + return report +} + +// ServeHTTP never routes a reserved S3 path through the cleaning mux. +func (d *Doors) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == ReportPath { + if r.Method != http.MethodGet || r.URL.EscapedPath() != ReportPath || r.URL.RawQuery != "" { + d.refuse(w, r, "invalid S3 report request") + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(d.Snapshot()); err != nil { + d.log.ErrorContext(r.Context(), "S3 report encoding failed", "error", err) + } + return + } + if !strings.HasPrefix(r.URL.Path, ReportPath+"/") { + d.next.ServeHTTP(w, r) + return + } + raw := r.URL.EscapedPath() + for prefix, handler := range d.handlers { + if remainder, ok := strings.CutPrefix(raw, prefix); ok { + path, err := url.PathUnescape("/" + remainder) + if err != nil { + d.refuse(w, r, "invalid encoded S3 path") + return + } + request := r.Clone(r.Context()) + request.URL.Path, request.URL.RawPath = path, "/"+remainder + observed := &doorWriter{ResponseWriter: w} + defer func() { + // TestAnInterruptedDoorDoesNotInventResponseEvidence holds this. + if observed.status == 0 { + d.log.ErrorContext(r.Context(), "S3 request ended without response headers") + return + } + d.mu.Lock() + key := prefix + " " + fmt.Sprint(observed.status) + d.report.HTTPRequests[key]++ + d.mu.Unlock() + }() + if r.Method == http.MethodPut { + if err := http.NewResponseController(w).SetReadDeadline(time.Now().Add(UploadReadTimeout)); err != nil { + d.log.ErrorContext(r.Context(), "S3 upload read deadline unavailable", "error", err) + http.Error(observed, "feint: S3 upload read deadline unavailable", http.StatusServiceUnavailable) + return + } + } + handler.ServeHTTP(observed, request) + return + } + } + d.refuse(w, r, "S3 door is not enabled or its prefix is not canonical") +} + +func (d *Doors) refuse(w http.ResponseWriter, r *http.Request, reason string) { + d.log.WarnContext(r.Context(), "S3 dispatch refused", "reason", reason) + http.Error(w, "feint: "+reason, http.StatusNotFound) +} + +type doorWriter struct { + http.ResponseWriter + status int +} + +func (w *doorWriter) WriteHeader(status int) { + if w.status != 0 { + return + } + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func (w *doorWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(p) +} + +func (w *doorWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } diff --git a/internal/s3/doors_test.go b/internal/s3/doors_test.go new file mode 100644 index 00000000..eeec166b --- /dev/null +++ b/internal/s3/doors_test.go @@ -0,0 +1,242 @@ +package s3 + +import ( + "bytes" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" +) + +func mountedFixture(t *testing.T) (*Doors, *objectstore.Store) { + t.Helper() + _, store := fixture(t) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + control := http.NewServeMux() + control.HandleFunc("GET /native", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("control")) }) + // A positive fallback witness makes a reserved-path leak observable even + // when the real control plane would happen to answer the same HTTP 404. + control.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("control-fallback")) }) + doors, err := Mount(control, store, []Profile{ + {Provider: "scaleway", Region: "fr-par", MissingBucket: ScalewayMissingBucket}, + {Provider: "scaleway", Region: "nl-ams", MissingBucket: ScalewayMissingBucket}, + {Provider: "exoscale", Region: "ch-gva-2", MissingBucket: ExoscaleMissingBucket}, + }, 512, log) + if err != nil { + t.Fatal(err) + } + return doors, store +} + +func doorRequest(t *testing.T, client *http.Client, method, target, body string, status int) []byte { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), method, target, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(res.Body) + closeErr := res.Body.Close() + if err != nil || closeErr != nil || res.StatusCode != status || res.Header.Get("Location") != "" { + t.Fatalf("%s %s: status=%d want=%d read=%v close=%v location=%q body=%q", + method, target, res.StatusCode, status, err, closeErr, res.Header.Get("Location"), raw) + } + return raw +} + +func TestMountedDoorsPreserveRawKeysAndIsolateScopes(t *testing.T) { + doors, store := mountedFixture(t) + ts := httptest.NewServer(emulator.GuardRebinding(doors, "127.0.0.1:4599")) + defer ts.Close() + client := ts.Client() + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + wireKey := "/a/../%2F%252F%00%3F%23" + logicalKey := "/a/..//%2F\x00?#" + for i, door := range doors.Snapshot().Doors { + prefix := ts.URL + door.Path + doorRequest(t, client, "PUT", prefix+"bucket", "", 200) + payload := strings.Repeat(string(rune('a'+i)), i+1) + doorRequest(t, client, "PUT", prefix+"bucket/"+wireKey, payload, 200) + got := doorRequest(t, client, "GET", prefix+"bucket/"+wireKey, "", 200) + if string(got) != payload { + t.Fatalf("scope %s returned %q, want %q", door.Path, got, payload) + } + obj, err := store.HeadObject(t.Context(), door.Provider+"/"+door.Region+"/bucket", logicalKey) + if err != nil || obj.Size != int64(len(payload)) { + t.Fatalf("key was cleaned, decoded twice or stored in the wrong scope: %+v %v", obj, err) + } + } + for i, door := range doors.Snapshot().Doors { + got := doorRequest(t, client, "GET", ts.URL+door.Path+"bucket/"+wireKey, "", 200) + if string(got) != strings.Repeat(string(rune('a'+i)), i+1) { + t.Fatalf("later write contaminated %s: %q", door.Path, got) + } + } + report := doors.Snapshot() + if !report.Enabled || report.MaxBlobBytes != store.BlobLimit() || report.MaxObjectBytes != 512 || len(report.HTTPRequests) == 0 { + t.Fatalf("incorrect live configuration/evidence: %+v", report) + } + report.Doors[0].Path = "/wrong" + clear(report.HTTPRequests) + if doors.Snapshot().Doors[0].Path == "/wrong" || len(doors.Snapshot().HTTPRequests) == 0 { + t.Fatal("snapshot aliases live configuration/evidence") + } + if got := doorRequest(t, client, "GET", ts.URL+"/native", "", 200); string(got) != "control" { + t.Fatal("mount displaced the control plane") + } +} + +func TestMountedDoorRefusesUnknownAndEncodedPrefixesWithoutRedirect(t *testing.T) { + doors, _ := mountedFixture(t) + ts := httptest.NewServer(doors) + defer ts.Close() + for _, path := range []string{ + "/_feint/s3/outscale/eu-west-2/bucket/key", + "/_feint/s3/scaleway/not-declared/bucket/key", + "/_feint/s3/scaleway/fr-par", + "/_feint/s3/scaleway/../fr-par/bucket/key", + "/_feint/s3%2Fscaleway/fr-par/bucket/key", + "/_feint/s3/scaleway%2Ffr-par/bucket/key", + } { + doorRequest(t, ts.Client(), "GET", ts.URL+path, "", 404) + } + for _, door := range doors.Snapshot().Doors { + body := doorRequest(t, ts.Client(), "GET", ts.URL+door.Path+"missing-bucket/key", "", 404) + if !bytes.Contains(body, []byte("missing-bucket")) || + bytes.Contains(body, []byte(door.Provider+"/"+door.Region)) { + t.Fatalf("private namespace leaked into missing-bucket dialect: %q", body) + } + doorRequest(t, ts.Client(), "GET", ts.URL+doors.Snapshot().Doors[0].Path+"bucket%2Fother/key", "", 400) + } +} + +type deadlineWriter struct { + *httptest.ResponseRecorder + deadline time.Time +} + +func (w *deadlineWriter) SetReadDeadline(deadline time.Time) error { + w.deadline = deadline + return nil +} + +func TestMountedPutRequiresAndInstallsBodyReadDeadline(t *testing.T) { + doors, _ := mountedFixture(t) + request := func() *http.Request { + req := httptest.NewRequest("PUT", "http://localhost/_feint/s3/scaleway/fr-par/bucket", nil) + req.RemoteAddr = "127.0.0.1:1234" + return req + } + unsupported := httptest.NewRecorder() + doors.ServeHTTP(unsupported, request()) + if unsupported.Code != 503 { + t.Fatal("a PUT without deadline support was accepted:", unsupported.Code) + } + writer := &deadlineWriter{ResponseRecorder: httptest.NewRecorder()} + before := time.Now() + doors.ServeHTTP(writer, request()) + if writer.Code != 200 || writer.deadline.Before(before.Add(UploadReadTimeout)) || + writer.deadline.After(time.Now().Add(UploadReadTimeout)) { + t.Fatalf("valid PUT did not install the bounded read deadline: %d %v", writer.Code, writer.deadline) + } +} + +func TestMountedS3CannotDisarmItsOriginGuard(t *testing.T) { + doors, _ := mountedFixture(t) + // Deliberately no outer guard: this also covers an exposed control plane. + ts := httptest.NewServer(doors) + defer ts.Close() + req, err := http.NewRequestWithContext(t.Context(), "PUT", ts.URL+doors.Snapshot().Doors[0].Path+"bucket", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", "https://hostile.example") + res, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + if err := res.Body.Close(); err != nil { + t.Fatal(err) + } + if res.StatusCode != 403 { + t.Fatal("exposing the control plane admitted a browser S3 mutation:", res.StatusCode) + } + doorRequest(t, ts.Client(), "PUT", req.URL.String(), "", 200) +} + +func TestMountRequiresActualQuotaAndExplicitActivation(t *testing.T) { + doors, store := mountedFixture(t) + _ = doors + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + next := http.NotFoundHandler() + profile := Profile{Provider: "scaleway", Region: "fr-par", MissingBucket: ScalewayMissingBucket} + parent := t.TempDir() + if err := os.Chmod(parent, 0700); err != nil { + t.Fatal(err) + } + unlimited, err := objectstore.Open(t.Context(), filepath.Join(parent, "unlimited")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := unlimited.Close(); err != nil { + t.Error(err) + } + }() + for _, tc := range []struct { + store *objectstore.Store + profiles []Profile + limit int64 + }{ + {nil, []Profile{profile}, 512}, {store, nil, 512}, {unlimited, []Profile{profile}, 512}, + {store, []Profile{profile}, store.BlobLimit() + 1}, {store, []Profile{profile}, 0}, + {store, []Profile{profile, profile}, 512}, + {store, []Profile{{Provider: "scaleway", Region: "../region", MissingBucket: ScalewayMissingBucket}}, 512}, + } { + if _, err := Mount(next, tc.store, tc.profiles, tc.limit, log); err == nil { + t.Fatalf("invalid activation/configuration accepted: %+v", tc) + } + } + disabled, err := Mount(next, nil, nil, 0, log) + if err != nil { + t.Fatal(err) + } + if disabled.Snapshot().Enabled || disabled.Snapshot().MaxBlobBytes != 0 { + t.Fatal("disabled storage advertised an active budget") + } + if err := store.HeadBucket(t.Context(), "bucket"); !errors.Is(err, objectstore.ErrBucketNotFound) { + t.Fatal("construction unexpectedly changed storage:", err) + } +} + +func TestAnInterruptedDoorDoesNotInventResponseEvidence(t *testing.T) { + doors, _ := mountedFixture(t) + prefix := doors.Snapshot().Doors[0].Path + // Force an interrupted handler before it writes any response header. + doors.handlers[prefix] = nil + req := httptest.NewRequest("GET", "http://localhost"+prefix+"bucket/key", nil) + req.RemoteAddr = "127.0.0.1:1234" + func() { + defer func() { + if recover() == nil { + t.Fatal("the interruption control did not interrupt its handler") + } + }() + doors.ServeHTTP(httptest.NewRecorder(), req) + }() + if len(doors.Snapshot().HTTPRequests) != 0 { + t.Fatal("a handler without response headers acquired completed-response evidence") + } +} diff --git a/internal/s3/server.go b/internal/s3/server.go index 65f95e6e..8a6dbaae 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -46,6 +46,8 @@ type Config struct { MaxObjectBytes int64 MissingBucket MissingBucketShape Log *slog.Logger + // Namespace is an opaque deployment scope, not an authenticated tenant. + Namespace string } // Server is a direct HTTP handler. Do not put it behind a path-cleaning mux: @@ -57,7 +59,8 @@ type Server struct { // New validates explicit dependencies; it neither opens storage nor listens. func New(store *objectstore.Store, cfg Config) (*Server, error) { - if store == nil || cfg.Log == nil || cfg.Region == "" || !headerValue(cfg.Region) { + if store == nil || cfg.Log == nil || cfg.Region == "" || !headerValue(cfg.Region) || + len(cfg.Namespace) > 128 || !headerValue(cfg.Namespace) { return nil, errors.New("s3: store, logger and a header-safe region are required") } if cfg.MaxObjectBytes <= 0 || cfg.MaxObjectBytes > (math.MaxInt64-(64<<10))/2 { @@ -73,6 +76,10 @@ func New(store *objectstore.Store, cfg Config) (*Server, error) { // Other features get an explicit Feint refusal, never a successful empty stub // or an invented cloud XML error. Signatures and credentials are never logged. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if len(r.Header.Values("Origin")) != 0 { + s.refuse(w, r, http.StatusForbidden, "browser Origin is not supported", nil) + return + } peer, err := netip.ParseAddrPort(r.RemoteAddr) if err != nil || !peer.Addr().Unmap().IsLoopback() { s.refuse(w, r, http.StatusForbidden, "only direct loopback peers are supported", err) @@ -119,7 +126,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case http.MethodGet, http.MethodHead: s.get(w, r, bucket, key) case http.MethodDelete: - err := s.store.DeleteObject(r.Context(), bucket, key) + err := s.store.DeleteObject(r.Context(), s.name(bucket), key) if err != nil && !errors.Is(err, objectstore.ErrObjectNotFound) { s.storeError(w, r, bucket, err) return @@ -142,14 +149,14 @@ func (s *Server) bucket(w http.ResponseWriter, r *http.Request, name string) { s.refuse(w, r, http.StatusNotImplemented, "CreateBucket configuration is not implemented", readErr) return } - err = s.store.CreateBucket(r.Context(), name) + err = s.store.CreateBucket(r.Context(), s.name(name)) case http.MethodHead: - err = s.store.HeadBucket(r.Context(), name) + err = s.store.HeadBucket(r.Context(), s.name(name)) case http.MethodDelete: - err = s.store.DeleteBucket(r.Context(), name) + err = s.store.DeleteBucket(r.Context(), s.name(name)) status = http.StatusNoContent case http.MethodGet: - err = s.store.HeadBucket(r.Context(), name) + err = s.store.HeadBucket(r.Context(), s.name(name)) if err == nil { s.refuse(w, r, http.StatusNotImplemented, "bucket listing is not implemented", nil) return @@ -186,7 +193,7 @@ func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) s.refuse(w, r, http.StatusBadRequest, "invalid S3 upload", err) return } - obj, err := s.store.PutObject(r.Context(), bucket, key, upload, objectstore.PutOptions{ + obj, err := s.store.PutObject(r.Context(), s.name(bucket), key, upload, objectstore.PutOptions{ Size: size, ContentType: r.Header.Get("Content-Type"), Metadata: metadata, }) if err != nil { @@ -202,7 +209,7 @@ func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) } func (s *Server) get(w http.ResponseWriter, r *http.Request, bucket, key string) { - obj, body, err := s.store.GetObject(r.Context(), bucket, key) + obj, body, err := s.store.GetObject(r.Context(), s.name(bucket), key) if err != nil { s.storeError(w, r, bucket, err) return @@ -328,6 +335,7 @@ func bucketName(name string) bool { if len(name) < 3 || len(name) > 63 || name[0] == '.' || name[0] == '-' || name[len(name)-1] == '.' || name[len(name)-1] == '-' { return false } + for _, c := range name { if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '.' && c != '-' { return false @@ -336,6 +344,13 @@ func bucketName(name string) bool { return !strings.Contains(name, "..") } +func (s *Server) name(bucket string) string { + if s.cfg.Namespace == "" { + return bucket + } + return s.cfg.Namespace + "/" + bucket +} + func headerValue(value string) bool { if !utf8.ValidString(value) { return false diff --git a/mise.toml b/mise.toml index 487c89f6..9bbc8da4 100644 --- a/mise.toml +++ b/mise.toml @@ -54,6 +54,10 @@ run = "go test -race ./..." description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 (required)" run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" +[tasks."conformance:s3:mounted"] +description = "Prove prefixed S3 bytes through the actual feint start/stop/restart process with AWS CLI 2.32.17" +run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" + [tasks.fuzz] description = "Fuzz the request decoders. Longer run: FUZZTIME=5m mise run fuzz" run = "go test ./internal/providers/scaleway/ -run '^$' -fuzz 'FuzzCreateServer' -fuzztime=\"${FUZZTIME:-30s}\"" diff --git a/tools/falsify/specs/mounted-object-storage.json b/tools/falsify/specs/mounted-object-storage.json new file mode 100644 index 00000000..3f7f8b6e --- /dev/null +++ b/tools/falsify/specs/mounted-object-storage.json @@ -0,0 +1,178 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "S3 activates on a listener which cannot receive loopback peers", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if !emulator.LoopbackListen(local) {", + "replace": "if !emulator.LoopbackListen(local) && false {", + "test": "TestS3ListenConfigurationAlwaysProducesUsableLoopbackEndpoints" + }, + { + "label": "the IPv4 wildcard listener is no longer mapped to usable loopback S3", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "case \"\", \"0.0.0.0\":", + "replace": "case \"\", \"\\x00never\":", + "test": "TestS3ListenConfigurationAlwaysProducesUsableLoopbackEndpoints" + }, + { + "label": "interrupted handlers invent completed HTTP response evidence", + "file": "internal/s3/doors.go", + "find": "if observed.status == 0 {\n\t\t\t\t\td.log.ErrorContext", + "replace": "if observed.status == 0 && false {\n\t\t\t\t\td.log.ErrorContext", + "test": "TestAnInterruptedDoorDoesNotInventResponseEvidence" + }, + { + "label": "provider and region namespaces collapse onto one bucket", + "file": "internal/s3/server.go", + "find": "return s.cfg.Namespace + \"/\" + bucket", + "replace": "return strings.TrimSuffix(s.cfg.Namespace, s.cfg.Namespace) + bucket", + "test": "TestMountedDoorsPreserveRawKeysAndIsolateScopes" + }, + { + "label": "dispatch uses an already decoded path", + "file": "internal/s3/doors.go", + "find": "raw := r.URL.EscapedPath()", + "replace": "raw := r.URL.Path", + "test": "TestMountedDoorsPreserveRawKeysAndIsolateScopes|TestMountedDoorRefusesUnknownAndEncodedPrefixesWithoutRedirect" + }, + { + "label": "the mounted remainder is path-cleaned", + "file": "internal/s3/doors.go", + "find": "request.URL.Path, request.URL.RawPath = path, \"/\"+remainder", + "replace": "request.URL.Path, request.URL.RawPath = strings.ReplaceAll(path, \"//\", \"/\"), strings.ReplaceAll(\"/\"+remainder, \"//\", \"/\")", + "test": "TestMountedDoorsPreserveRawKeysAndIsolateScopes" + }, + { + "label": "encoded bucket separators lose their boundary", + "file": "internal/s3/doors.go", + "find": "request.URL.Path, request.URL.RawPath = path, \"/\"+remainder", + "replace": "request.URL.Path, request.URL.RawPath = path, strings.TrimSuffix(\"/\"+remainder, \"/\"+remainder)", + "test": "TestMountedDoorsPreserveRawKeysAndIsolateScopes|TestMountedDoorRefusesUnknownAndEncodedPrefixesWithoutRedirect" + }, + { + "label": "reserved unknown S3 paths reach the cleaning control-plane mux", + "file": "internal/s3/doors.go", + "find": "d.refuse(w, r, \"S3 door is not enabled or its prefix is not canonical\")", + "replace": "d.next.ServeHTTP(w, r)", + "test": "TestMountedDoorRefusesUnknownAndEncodedPrefixesWithoutRedirect" + }, + { + "label": "an unlimited store advertises and accepts mounted quotas", + "file": "internal/s3/doors.go", + "find": "if maxBlobBytes <= 0 || maxObjectBytes > maxBlobBytes {", + "replace": "if (maxBlobBytes <= 0 || maxObjectBytes > maxBlobBytes) && false {", + "test": "TestMountRequiresActualQuotaAndExplicitActivation" + }, + { + "label": "declared namespace tokens are not validated", + "file": "internal/s3/doors.go", + "find": "if !ValidToken(p.Provider) || !ValidToken(p.Region) {", + "replace": "if (!ValidToken(p.Provider) || !ValidToken(p.Region)) && false {", + "test": "TestMountRequiresActualQuotaAndExplicitActivation" + }, + { + "label": "an exposed control plane disarms S3 browser protection", + "file": "internal/s3/server.go", + "find": "if len(r.Header.Values(\"Origin\")) != 0 {", + "replace": "if len(r.Header.Values(\"Origin\")) != 0 && false {", + "test": "TestMountedS3CannotDisarmItsOriginGuard" + }, + { + "label": "PUT proceeds without read-deadline support", + "file": "internal/s3/doors.go", + "find": "err != nil {\n\t\t\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", + "replace": "err != nil && false {\n\t\t\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", + "test": "TestMountedPutRequiresAndInstallsBodyReadDeadline" + }, + { + "label": "the PUT body read deadline collapses to immediate", + "file": "internal/s3/doors.go", + "find": "SetReadDeadline(time.Now().Add(UploadReadTimeout))", + "replace": "SetReadDeadline(time.Now().Add(UploadReadTimeout * 0))", + "test": "TestMountedPutRequiresAndInstallsBodyReadDeadline" + }, + { + "label": "start forgets byte-storage settings before recording serve args", + "package": "./internal/cli/", + "file": "internal/cli/lifecycle.go", + "find": "out = append(out, f.storage.args()...)", + "replace": "out = append(out, f.storage.args()[:0]...)", + "test": "TestStartPreservesObjectStorageFlagsAndAbsoluteRoot" + }, + { + "label": "restart retains a relative object-store root", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "f.root = absolute", + "replace": "f.root = filepath.Join(\".\", filepath.Base(absolute))", + "test": "TestStartPreservesObjectStorageFlagsAndAbsoluteRoot" + }, + { + "label": "invalid and overflowing byte limits are accepted", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if f.budgetMiB <= 0 || f.budgetMiB > math.MaxInt64>>20 ||\n\t\tf.objectMiB <= 0 || f.objectMiB > (math.MaxInt64-(64<<10))/2>>20 ||\n\t\tf.objectMiB > f.budgetMiB {", + "replace": "if (f.budgetMiB <= 0 || f.budgetMiB > math.MaxInt64>>20 ||\n\t\tf.objectMiB <= 0 || f.objectMiB > (math.MaxInt64-(64<<10))/2>>20 ||\n\t\tf.objectMiB > f.budgetMiB) && false {", + "test": "TestObjectStorageValidatesDefaultsAndLimitsWithoutStartingAnything" + }, + { + "label": "malformed and duplicate CLI door declarations are accepted", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if !ok || !s3.ValidToken(provider) || !s3.ValidToken(region) || seen[entry] {", + "replace": "if (!ok || !s3.ValidToken(provider) || !s3.ValidToken(region) || seen[entry]) && false {", + "test": "TestS3DeclarationsRejectUnknownMalformedAndDuplicateDoors" + }, + { + "label": "a provider without an S3 dialect becomes a silent empty declaration", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if !found {", + "replace": "if !found && false {", + "test": "TestS3DeclarationsRejectUnknownMalformedAndDuplicateDoors" + }, + { + "label": "foreign or credential-bearing discovery URLs are accepted", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if err != nil || (base.Scheme != \"http\" && base.Scheme != \"https\") ||\n\t\t!emulator.LoopbackListen(net.JoinHostPort(base.Hostname(), \"0\")) || base.User != nil || base.RawQuery != \"\" ||\n\t\tbase.Fragment != \"\" || base.RawPath != \"\" || (base.Path != \"\" && base.Path != \"/\") {", + "replace": "if (err != nil || (base.Scheme != \"http\" && base.Scheme != \"https\") ||\n\t\t!emulator.LoopbackListen(net.JoinHostPort(base.Hostname(), \"0\")) || base.User != nil || base.RawQuery != \"\" ||\n\t\tbase.Fragment != \"\" || base.RawPath != \"\" || (base.Path != \"\" && base.Path != \"/\")) && false {", + "test": "TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints" + }, + { + "label": "discovery follows a redirect rather than refusing it", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "return http.ErrUseLastResponse", + "replace": "return errors.Unwrap(http.ErrUseLastResponse)", + "test": "TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints" + }, + { + "label": "unknown report versions and inconsistent limits become valid discovery", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if report.SchemaVersion != s3.ReportVersion || report.Doors == nil || report.HTTPRequests == nil ||\n\t\t(report.Enabled && (report.MaxBlobBytes <= 0 || report.MaxObjectBytes <= 0 ||\n\t\t\treport.MaxObjectBytes > report.MaxBlobBytes || len(report.Doors) == 0)) ||\n\t\t(!report.Enabled && len(report.Doors) != 0) {", + "replace": "if (report.SchemaVersion != s3.ReportVersion || report.Doors == nil || report.HTTPRequests == nil ||\n\t\t(report.Enabled && (report.MaxBlobBytes <= 0 || report.MaxObjectBytes <= 0 ||\n\t\t\treport.MaxObjectBytes > report.MaxBlobBytes || len(report.Doors) == 0)) ||\n\t\t(!report.Enabled && len(report.Doors) != 0)) && false {", + "test": "TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints" + }, + { + "label": "ordinary unset leaves the S3 endpoint installed", + "package": "./internal/cli/", + "file": "internal/cli/env.go", + "find": "if *withS3 || (*unset && hasS3) {", + "replace": "if *withS3 || (*unset && hasS3 && false) {", + "test": "TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline" + }, + { + "label": "ambiguous S3 regions silently select the last endpoint", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if selected != nil {", + "replace": "if selected != nil && false {", + "test": "TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 6c28c5b8..eefb3059 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -79,6 +79,18 @@ const prepushIsTheWholeGate = "" var rules = []rule{ // ---------------------------------------------------------------- packs + { + Path: "internal/providers/scaleway/object_storage.go", + Why: "declared S3 dialect and configurable endpoint variables, outside control-plane SDK coverage", + Runs: []string{"conformance:s3", "conformance:s3:mounted"}, + Unproven: "AWS CLI byte operations, not a complete scw lifecycle or hardcoded Terraform S3 redirection", + }, + { + Path: "internal/providers/exoscale/object_storage.go", + Why: "declared SOS dialect and configurable endpoint variables, outside control-plane SDK coverage", + Runs: []string{"conformance:s3", "conformance:s3:mounted"}, + Unproven: "AWS CLI byte operations, not a complete exo lifecycle, bucket-usage API or multipart support", + }, { Path: "internal/providers/scaleway/", Why: "the Scaleway pack — routes, response shapes, errors", @@ -235,18 +247,39 @@ var rules = []rule{ }}, }, { - Path: "internal/core/objectstore/", - Why: "neutral persisted bytes used by the unmounted S3 component", - Runs: []string{"conformance:s3"}, - Unproven: "the component proof does not mount S3 in the emulator or run provider clients, " + - "checkpoint/reset, power-loss recovery or network filesystem locking", + Path: "internal/core/objectstore/", + Why: "neutral persisted bytes used by the S3 component and the real emulator process", + Runs: []string{"conformance:s3", "conformance:s3:mounted"}, + Unproven: "complete provider-client lifecycles, checkpoint/reset, power-loss recovery and network filesystem locking", + }, + { + Path: "internal/s3/", + Why: "bounded path-style S3 protocol and real-byte official-client proof", + Runs: []string{"conformance:s3", "conformance:s3:mounted"}, + Unproven: "one AWS CLI version, not complete scw/exo lifecycles, virtual hosts, interception, ETag semantics, multipart or public stacks", + }, + { + Path: "internal/conformancetest/", + Why: "isolated pinned AWS CLI environment and non-forwarding egress control", + Runs: []string{"conformance:s3", "conformance:s3:mounted"}, + Unproven: "the proxy is not a network namespace and does not constrain arbitrary clients which ignore it", + }, + { + Path: "internal/cli/object_storage.go", + Why: "S3 activation, discovery and actual-process persistence", + Runs: []string{"conformance:s3:mounted"}, + Unproven: "complete provider CLI lifecycles, declarative feint.yaml storage configuration and hardcoded Terraform endpoints", + Cites: []claim{{ + About: "declarative feint.yaml storage configuration", + In: "internal/cli/up.go", + Absent: []string{"--objectstore", "--s3-doors"}, + }}, }, { - Path: "internal/s3/", - Why: "bounded path-style S3 protocol and real-byte official-client proof", - Runs: []string{"conformance:s3"}, - Unproven: "the explicit component proof covers one AWS CLI version, not provider doors, " + - "virtual hosts, interception, ETag semantics, multipart, public stacks or cumulative quotas", + Path: "internal/cli/object_storage_conformance_test.go", + Why: "the actual-process S3 client witness, excluded from ordinary unit tests", + Runs: []string{"conformance:s3:mounted"}, + Unproven: "complete provider CLI lifecycles and hardcoded Terraform endpoints", }, // ------------------------------------------------------ what describes it From 7c3e087fb09bf3eadc3589cc434fca4fa7dac9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 08:53:55 +0100 Subject: [PATCH 05/28] feat(s3): prove private Scaleway bucket lifecycle Persist neutral bucket attributes and the explicit Suspended configuration, serve measured private ACL/tagging/list shapes, and export SDK-compatible bare endpoints. Prove scw create/list/get/delete alongside AWS byte checks through an actual restart. Twenty new mutations and all sixty-four inherited mutations compile and bite. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 11 +- CHANGELOG.md | 10 +- docs/confidence.md | 2 +- docs/limits-acks.json | 2 +- docs/limits.md | 14 +- docs/object-storage-routing.md | 57 +++- internal/cli/env.go | 2 +- internal/cli/object_storage.go | 2 +- .../cli/object_storage_conformance_test.go | 49 +++- internal/cli/object_storage_test.go | 3 +- internal/conformancetest/aws.go | 44 +-- internal/conformancetest/scw.go | 51 ++++ internal/core/objectstore/bucket_metadata.go | 71 +++++ .../core/objectstore/bucket_metadata_test.go | 147 ++++++++++ internal/core/objectstore/files.go | 2 +- internal/core/objectstore/manifest.go | 8 +- internal/core/objectstore/store.go | 16 +- internal/core/objectstore/store_test.go | 4 +- internal/providers/scaleway/object_storage.go | 6 +- internal/s3/bucket_configuration.go | 272 ++++++++++++++++++ internal/s3/bucket_configuration_test.go | 175 +++++++++++ internal/s3/doors.go | 22 +- internal/s3/doors_test.go | 23 ++ internal/s3/server.go | 52 +++- mise.toml | 2 +- tools/falsify/specs/neutral-object-bytes.json | 4 +- .../specs/s3-bucket-configuration.json | 153 ++++++++++ tools/falsify/specs/s3-gateway.json | 4 +- tools/testplan/rules.go | 8 +- 29 files changed, 1145 insertions(+), 71 deletions(-) create mode 100644 internal/conformancetest/scw.go create mode 100644 internal/core/objectstore/bucket_metadata.go create mode 100644 internal/core/objectstore/bucket_metadata_test.go create mode 100644 internal/s3/bucket_configuration.go create mode 100644 internal/s3/bucket_configuration_test.go create mode 100644 tools/falsify/specs/s3-bucket-configuration.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index b366c6bd..a89be4d5 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -26,7 +26,7 @@ change ni l'un ni l'autre a sa place dans `git log`. séparent les espaces. `restart` conserve la racine absolue et les limites ; les snapshots de ressources et le nettoyage ne possèdent pas ces octets. AWS CLI 2.32.17 prouve l'identité octets/checksum sur le vrai processus avant - et après redémarrage. Les cycles `scw`/`exo` complets, les ETag, le multipart + et après redémarrage. Les cycles `exo` complets, les ETag, le multipart et les endpoints Terraform codés en dur restent hors périmètre. - **La découverte S3 active, pas un endpoint deviné quand elle est désactivée** (#850) : `env --s3 [--s3-region]`, `doctor --s3`, un statut explicite @@ -35,6 +35,15 @@ change ni l'un ni l'autre a sa place dans `git log`. ses preuves clientes. La surface CLI additive passe à **26** ; les codes de sortie et le schéma de santé restent inchangés. `env --unset` retire aussi les variables d'endpoint S3, sans instance active. +- **Le cycle mesuré des buckets Scaleway privés** (#850) : + `scw` 2.62.0 `create/list/get/delete`, avec les octets vérifiés par AWS CLI, + survit au redémarrage réel. Région XML, checksums de transport bornés, + versioning `Suspended` persistant, ACL privée et `NoSuchTagSet` enregistré + sont servis ; historique de versions, écriture des tags et droits publics + restent refusés. Les endpoints exportés omettent le slash refusé par le SDK ; + la liste accepte sa racine canonique nue sans redirection. Le format privé + neutre version 2 lit la version 1 et la migre à la prochaine mutation ; + les anciens binaires ne peuvent pas relire cette version 2. ## [0.14.0] - 2026-10-26 diff --git a/CHANGELOG.md b/CHANGELOG.md index 902f8f29..bfee54db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ what this project is judged on: **a response shape a client can observe**, and namespaces. `restart` preserves the absolute root and limits; resource snapshots and cleanup do not own these bytes. AWS CLI 2.32.17 proves byte/checksum identity through the real process before and after restart. - Complete `scw`/`exo` lifecycles, ETag semantics, multipart and hardcoded + Complete `exo` lifecycles, ETag semantics, multipart and hardcoded Terraform endpoints remain unsupported. - **Live S3 discovery, not an endpoint guessed while disabled** (#850): `env --s3 [--s3-region]`, `doctor --s3`, explicit enabled/disabled/unknown @@ -33,6 +33,14 @@ what this project is judged on: **a response shape a client can observe**, and The additive CLI surface is version **26**; existing exit codes and the health schema stay unchanged. Ordinary `env --unset` also clears S3 endpoint variables offline. +- **Measured private Scaleway bucket lifecycle** (#850): `scw` 2.62.0 + `create/list/get/delete`, alongside verified AWS CLI bytes, survives actual + process restart. Region XML, bounded transport checksums, persisted + `Suspended` versioning, private canonical ACL and recorded `NoSuchTagSet` + are served; version histories, writable tags and public grants are refused. + S3 env endpoints omit the slash the SDK rejects; canonical bare bucket-list + roots are dispatched without redirects. Neutral private metadata version 2 + reads version 1 and upgrades on mutation; older binaries cannot read it. ## [0.14.0] - 2026-10-26 diff --git a/docs/confidence.md b/docs/confidence.md index 63cd0f6c..7c3ff52e 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -44,7 +44,7 @@ prints the reason under the pack that owns it. | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17 | `mise run conformance:s3:mounted` drives the actual process before and after restart; complete `scw`/`exo` lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#implemented-mounted-slice) | +| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17 and scw 2.62.0 | `mise run conformance:s3:mounted` drives bytes and private non-versioned Scaleway buckets before and after actual process restart; complete Exoscale lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#measured-scaleway-bucket-workflow) | | Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | diff --git a/docs/limits-acks.json b/docs/limits-acks.json index d2ed9750..312bcf51 100644 --- a/docs/limits-acks.json +++ b/docs/limits-acks.json @@ -21,7 +21,7 @@ "Identifiers are not checked against anything": "2026-09-02", "Lifecycle transitions are immediate, unless you ask for otherwise": "2026-09-03", "Managed Kubernetes is not emulated, and a CRUD-only version is refused (#283)": "2026-08-27", - "Object Storage is not emulated": "2026-08-27", + "Object Storage: simple explicit doors, not full client workflows": "2026-11-07", "Outscale and Exoscale trail Scaleway, and the generated tables say by how much": "2026-08-27", "Placement is recorded, never enforced (#285)": "2026-08-27", "ReadTags does not list an internet service, because upstream names no type for one": "2026-08-27", diff --git a/docs/limits.md b/docs/limits.md index e8417e9e..d43a122b 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -8,10 +8,12 @@ than one that is small. #850 adds opt-in, persisted simple S3 operations on the existing port, with declared `/_feint/s3///` endpoints, cumulative byte budgets -and an AWS CLI 2.32.17 restart proof. See the +and an AWS CLI 2.32.17 / scw 2.62.0 restart proof. The Scaleway private, +non-versioned bucket `create/list/get/delete` workflow is served. See the [door ADR and commands](object-storage-routing.md). -Complete `scw`/`exo` lifecycles, listing, ACL/versioning, multipart, ETags, -checkpoint/reset and hardcoded Terraform redirection remain unsupported. +Complete Exoscale workflows, object listing, writable tags/public ACLs, +version histories, multipart, ETags, checkpoint/reset and hardcoded Terraform +redirection remain unsupported. No MinIO service or dependency is used by this implementation. ### Why Scaleway Terraform is still not redirected @@ -36,9 +38,9 @@ for the number; the section below is that number, measured. The short version: the certificate half is cheap and safe, the DNS half is neither, and the whole blocker reduces to this one product on this one client. -The SDK and the CLI are better off: they honour `SCW_S3_ENDPOINT`. So an S3 -workflow driven by `scw` or by an SDK can already point at MinIO today; only the -Terraform path is blocked. +The SDK and the CLI are better off: they honour `SCW_S3_ENDPOINT`. The private, +non-versioned bucket workflow now points at Feint's explicit same-port door +and passes with the pinned CLI; the Terraform path remains blocked. The consequence has since been observed live, from a stranger's stack rather than a fixture ([#262](https://github.com/stephrobert/feint/issues/262), diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 7ee0e694..3095c419 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -160,8 +160,59 @@ data. This is an HTTP actual-process witness; hostile-response-checksum controls. This certifies simple CreateBucket, PutObject, GetObject, HeadObject, -DeleteObject and DeleteBucket through the mounted gateway, not complete -`scw`/`exo` lifecycles. Unsupported ancillary operations, missing-object cloud -errors and multipart still receive explicit Feint refusals. No ETag is invented. +DeleteObject and DeleteBucket through the mounted gateway, plus the bounded +Scaleway bucket workflow below, not complete `exo` lifecycles. Unsupported +ancillary operations, missing-object cloud errors and multipart still receive +explicit Feint refusals. No ETag is invented. The snapshot-import route still records an intention rather than reading these bytes. Object Storage 1.0 and hardcoded Terraform redirection are not delivered. + +## Measured Scaleway bucket workflow + +The declared Scaleway doors additionally serve `scw` **2.62.0** bucket +`create/list/get/delete`, with the ordinary defaults: private ACL, no Object +Lock and `enable-versioning=false`. `mise run conformance:s3:mounted` now +requires that pinned client as well as AWS CLI. It creates the bucket with +`scw`, uploads/downloads the binary witness with AWS CLI (the Scaleway CLI has +no object upload command), and repeats both bucket reads and byte retrieval +after the real process restarts. Creation dates and the canonical local owner +remain stable; listing reads the actual namespace-scoped inventory. + +The wire contract comes from the official CLI source and historical HTTP +cassettes at revision `a2a40172bb27582c2a0fe37a4837f949cb10e6a6`, +particularly +[simple create](https://github.com/scaleway/scaleway-cli/blob/a2a40172bb27582c2a0fe37a4837f949cb10e6a6/internal/namespaces/object/v1/testdata/test-bucket-create-simple.cassette.yaml) +and +[list](https://github.com/scaleway/scaleway-cli/blob/a2a40172bb27582c2a0fe37a4837f949cb10e6a6/internal/namespaces/object/v1/testdata/test-bucket-list-simple.cassette.yaml). +This is historical Scaleway evidence, not an Exoscale response oracle or a +new authenticated cloud recording. + +`CreateBucket` checks the optional XML region against its declared door. +`PutBucketVersioning` verifies its bounded body/checksums, including the +client's legacy Content-MD5, before persisting `Suspended`. A new bucket has +no explicit versioning status until that write. `Enabled`, tags writes, +non-private permissions and Object Lock remain logged Feint refusals. +`GetBucketTagging` uses the recorded `NoSuchTagSet` response because tag +creation is not supported; `GetBucketAcl` describes the only accepted policy. +The stable owner is a local fixture identity, not authenticated IAM. +The isolated SCW cache has a fresh `last-cli-checks` marker, using the pinned +client's own predicate to suppress unrelated GitHub version/API-key checks. +The non-forwarding proxy remains enabled; no host cache or credentials are used. + +The SDK refuses a trailing slash in its configured endpoint, and ListBuckets +uses that bare root. `env --s3` therefore exports endpoints **without** their +terminal slash, for every shell. The report retains canonical slash-suffixed +door prefixes. The dispatcher accepts a canonical bare GET for the declared +bucket workflow, without redirecting, cleaning paths or accepting encoded +aliases. Literal object-key handling is unchanged. + +Bucket attributes belong to the neutral byte store, in private metadata +version **2**. Version 1 indexes remain readable without a read-time rewrite; +the next mutation upgrades them. An older binary cannot read the upgraded +index. Existing bytes, generations and quotas remain unchanged. Legacy buckets +without a stored creation date receive an explicit listing refusal rather +than an invented date. Resource snapshots still do not own this metadata. + +Exoscale upload/download/delete, its mandatory ListMultipartUploads during +bucket removal, native bucket-usage API, and full Object Storage 1.0 remain +separate follow-up work. No empty multipart success stub was added. diff --git a/internal/cli/env.go b/internal/cli/env.go index 6cd0126a..c41eb9de 100644 --- a/internal/cli/env.go +++ b/internal/cli/env.go @@ -124,7 +124,7 @@ func envCommand(args []string, stdout, stderr io.Writer) int { return exitError } if !*unset { - fmt.Fprintln(stderr, "note: S3 serves simple byte operations only; complete provider CLI lifecycles and hardcoded Terraform endpoints are not certified.") + fmt.Fprintln(stderr, "note: S3 supports simple bytes and private non-versioned Scaleway buckets; Exoscale ancillary workflows and hardcoded Terraform endpoints are not certified.") } } diff --git a/internal/cli/object_storage.go b/internal/cli/object_storage.go index 22491846..b518790e 100644 --- a/internal/cli/object_storage.go +++ b/internal/cli/object_storage.go @@ -228,7 +228,7 @@ func storageEnvironment(pack packEnv, endpoint, region string, unset bool, vars } profile := storage.ObjectStorage(selected.Region) for _, key := range profile.Variables { - vars[key] = strings.TrimSuffix(endpoint, "/") + selected.Path + vars[key] = strings.TrimSuffix(endpoint, "/") + strings.TrimSuffix(selected.Path, "/") } vars[profile.RegionVariable] = selected.Region return nil diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index e38ae9f4..1b0bbbfe 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -27,8 +27,9 @@ import ( func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { var guardHits atomic.Int64 - guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { guardHits.Add(1) + t.Logf("refusing proxy observed method=%s host=%s (never forwarded)", r.Method, r.Host) http.Error(w, "external requests refused, never forwarded", http.StatusBadGateway) })) defer guard.Close() @@ -92,7 +93,8 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { out := run("env", provider, "--s3", "--endpoint", endpoint) found := false for _, door := range first.ObjectStorage.Doors { - if door.Provider == provider && bytes.Contains(out, []byte(endpoint+door.Path)) { + if door.Provider == provider && bytes.Contains(out, []byte(endpoint+strings.TrimSuffix(door.Path, "/"))) && + !bytes.Contains(out, []byte(endpoint+door.Path)) { found = true } } @@ -113,6 +115,24 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { return aws.Run(t, append([]string{"--endpoint-url", endpoint + door.Path, "--region", door.Region, "--no-cli-pager", "s3api"}, args...)...) } + scw := conformancetest.SCW(t, aws, endpoint+strings.TrimSuffix(s3.DoorPath("scaleway", "fr-par"), "/")) + bucketCall := func(verb string, name bool) []byte { + t.Helper() + args := []string{"-o", "json", "object", "bucket", verb} + if name { + args = append(args, bucket) + } + return scw.Run(t, append(args, "region=fr-par")...) + } + checkVersioning := func() { + t.Helper() + var configuration struct{ Status string } + out := call(s3.Door{Provider: "scaleway", Region: "fr-par", Path: s3.DoorPath("scaleway", "fr-par")}, + "get-bucket-versioning", "--bucket", bucket) + if err := json.Unmarshal(out, &configuration); err != nil || configuration.Status != "Suspended" { + t.Fatalf("actual persisted versioning status was lost or invented: %v %q", err, out) + } + } get := func(door s3.Door, expected []byte) { t.Helper() result := call(door, "get-object", "--bucket", bucket, "--key", key, "--checksum-mode", "ENABLED", output) @@ -125,7 +145,11 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } } for i, door := range first.ObjectStorage.Doors { - call(door, "create-bucket", "--bucket", bucket) + if door.Provider == "scaleway" { + bucketCall("create", true) + } else { + call(door, "create-bucket", "--bucket", bucket) + } if i == 1 { if err := os.WriteFile(input, []byte("separate-sos-scope"), 0600); err != nil { t.Fatal(err) @@ -136,6 +160,12 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } get(first.ObjectStorage.Doors[0], payload) get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + beforeBucket := bucketCall("get", true) + checkVersioning() + beforeList := bucketCall("list", false) + if !bytes.Contains(beforeList, []byte(bucket)) { + t.Fatal("Scaleway CLI list omitted its created bucket:", string(beforeList)) + } // Restart the initially relative-root invocation from a different cwd. work = privateStorageDir(t) run("restart", "--addr", addr) @@ -147,6 +177,13 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } get(first.ObjectStorage.Doors[0], payload) get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + if after := bucketCall("get", true); !bytes.Equal(after, beforeBucket) { + t.Fatalf("Scaleway bucket configuration changed across restart: before=%q after=%q", beforeBucket, after) + } + checkVersioning() + if after := bucketCall("list", false); !bytes.Equal(after, beforeList) { + t.Fatalf("Scaleway bucket identity/creation time changed across restart: before=%q after=%q", beforeList, after) + } // Inspect only our private store while the owner is stopped: GET of the // same cleaned key twice would otherwise masquerade as key preservation. run("stop", "--addr", addr) @@ -168,7 +205,11 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { for _, door := range first.ObjectStorage.Doors { call(door, "head-object", "--bucket", bucket, "--key", key) call(door, "delete-object", "--bucket", bucket, "--key", key) - call(door, "delete-bucket", "--bucket", bucket) + if door.Provider == "scaleway" { + bucketCall("delete", true) + } else { + call(door, "delete-bucket", "--bucket", bucket) + } } if guardHits.Load() != 1 { t.Fatalf("official client reached the refusing egress proxy: delta=%d", guardHits.Load()-1) diff --git a/internal/cli/object_storage_test.go b/internal/cli/object_storage_test.go index 262d05db..a8e0f843 100644 --- a/internal/cli/object_storage_test.go +++ b/internal/cli/object_storage_test.go @@ -138,7 +138,8 @@ func TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline(t *testing.T) { } for _, shell := range []string{"bash", "fish", "powershell"} { if code := call("scaleway", "--s3", "--s3-region", "fr-par", "--endpoint", ts.URL, "--shell", shell); code != 0 || - !strings.Contains(out.String(), ts.URL+s3.DoorPath("scaleway", "fr-par")) || + !strings.Contains(out.String(), ts.URL+strings.TrimSuffix(s3.DoorPath("scaleway", "fr-par"), "/")) || + strings.Contains(out.String(), ts.URL+s3.DoorPath("scaleway", "fr-par")) || strings.Contains(out.String(), "not certified") || !strings.Contains(note.String(), "not certified") { t.Fatalf("live selected S3 environment or stderr contract failed: %d %q %q", code, out.String(), note.String()) } diff --git a/internal/conformancetest/aws.go b/internal/conformancetest/aws.go index 89f0345b..5d172c7a 100644 --- a/internal/conformancetest/aws.go +++ b/internal/conformancetest/aws.go @@ -16,15 +16,15 @@ import ( "time" ) -// AWSCLI is an isolated AWS CLI installation, not a host credential profile. -type AWSCLI struct { +// Client is a pinned official client in an isolated, public-fake environment. +type Client struct { Path string Home string Env []string } // AWS builds a private environment from the repository's public fake keys. -func AWS(t *testing.T, proxy, ca string) AWSCLI { +func AWS(t *testing.T, proxy, ca string) Client { t.Helper() binary, err := exec.LookPath("aws") if err != nil { @@ -34,20 +34,8 @@ func AWS(t *testing.T, proxy, ca string) AWSCLI { if err := os.Chmod(home, 0700); err != nil { //nolint:gosec // a private directory, not a file t.Fatal(err) } - creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") - if err != nil { - t.Fatal(err) - } - fake := make(map[string]string) - for _, line := range strings.Split(string(creds), "\n") { - if key, value, ok := strings.Cut(line, "="); ok && strings.HasPrefix(key, "SCW_") { - fake[key] = value - } - } - if fake["SCW_ACCESS_KEY"] == "" || fake["SCW_SECRET_KEY"] == "" { - t.Fatal("canonical public fake credentials are absent") - } - cli := AWSCLI{Path: binary, Home: home, Env: []string{ + fake := fakeCredentials(t) + cli := Client{Path: binary, Home: home, Env: []string{ "HOME=" + home, "PATH=" + filepath.Dir(binary) + ":/usr/bin:/bin", "XDG_STATE_HOME=" + home, "XDG_RUNTIME_DIR=" + home, "AWS_ACCESS_KEY_ID=" + fake["SCW_ACCESS_KEY"], "AWS_SECRET_ACCESS_KEY=" + fake["SCW_SECRET_KEY"], @@ -68,8 +56,26 @@ func AWS(t *testing.T, proxy, ca string) AWSCLI { return cli } +func fakeCredentials(t *testing.T) map[string]string { + t.Helper() + creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") + if err != nil { + t.Fatal(err) + } + fake := make(map[string]string) + for _, line := range strings.Split(string(creds), "\n") { + if key, value, ok := strings.Cut(line, "="); ok && strings.HasPrefix(key, "SCW_") { + fake[key] = value + } + } + if fake["SCW_ACCESS_KEY"] == "" || fake["SCW_SECRET_KEY"] == "" { + t.Fatal("canonical public fake credentials are absent") + } + return fake +} + // Run executes the pinned client with a bounded lifetime and no shell. -func (c AWSCLI) Run(t *testing.T, args ...string) []byte { +func (c Client) Run(t *testing.T, args ...string) []byte { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) defer cancel() @@ -77,7 +83,7 @@ func (c AWSCLI) Run(t *testing.T, args ...string) []byte { cmd.Env = c.Env out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("AWS CLI %s failed: %v\n%s", strings.Join(args, " "), err, out) + t.Fatalf("%s %s failed: %v\n%s", filepath.Base(c.Path), strings.Join(args, " "), err, out) } return out } diff --git a/internal/conformancetest/scw.go b/internal/conformancetest/scw.go new file mode 100644 index 00000000..61e709d8 --- /dev/null +++ b/internal/conformancetest/scw.go @@ -0,0 +1,51 @@ +//go:build conformance + +package conformancetest + +import ( + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// SCW reuses the witness's private HOME and refusing proxy, never a host +// credential profile. S3 requests target only the explicit local endpoint. +func SCW(t *testing.T, base Client, endpoint string) Client { + t.Helper() + binary, err := exec.LookPath("scw") + if err != nil { + t.Fatal("install Scaleway CLI 2.62.0 for this explicit bucket proof:", err) + } + client := Client{Path: binary, Home: base.Home, Env: append([]string{}, base.Env...)} + api, err := url.Parse(endpoint) + if err != nil { + t.Fatal("invalid witness endpoint:", err) + } + api.Path, api.RawPath = "", "" + cache := filepath.Join(base.Home, "scw-cache") + if err := os.Mkdir(cache, 0700); err != nil { //nolint:gosec // owned witness directory, not a file + t.Fatal(err) + } + // core/checks.go at the pinned revision skips unrelated GitHub/API-key + // checks when this private marker is fresh. The refusing proxy stays on. + if err := os.WriteFile(filepath.Join(cache, "last-cli-checks"), nil, 0600); err != nil { + t.Fatal(err) + } + client.Env = append(client.Env, "PATH="+filepath.Dir(binary)+":/usr/bin:/bin", + "SCW_S3_ENDPOINT="+endpoint, "SCW_API_URL="+api.String(), "SCW_INSECURE=true", "SCW_CACHE_DIR="+cache) + for key, value := range fakeCredentials(t) { + if key != "SCW_API_URL" { + client.Env = append(client.Env, key+"="+value) + } + } + version := string(client.Run(t, "version")) + fields := strings.Fields(version) + if len(fields) < 2 || fields[0] != "Version" || fields[1] != "2.62.0" { + t.Fatal("proof requires the measured Scaleway CLI 2.62.0, got:", version) + } + t.Log("Scaleway CLI 2.62.0; isolated public fake credentials") + return client +} diff --git a/internal/core/objectstore/bucket_metadata.go b/internal/core/objectstore/bucket_metadata.go new file mode 100644 index 00000000..8b3ca71b --- /dev/null +++ b/internal/core/objectstore/bucket_metadata.go @@ -0,0 +1,71 @@ +package objectstore + +import ( + "context" + "maps" +) + +// BucketMetadata returns a detached copy of a bucket's opaque UTF-8 attributes. +// A legacy bucket with no attributes returns an empty map. +func (s *Store) BucketMetadata(ctx context.Context, name string) (map[string]string, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + b, err := s.lookup(name) + if err != nil { + return nil, err + } + out := make(map[string]string, len(b.Metadata)) + maps.Copy(out, b.Metadata) + return out, nil +} + +// ListBucketMetadata returns detached attributes for one consistent inventory. +// Deletion cannot race a per-bucket lookup into a partially read result. +func (s *Store) ListBucketMetadata(ctx context.Context) (map[string]map[string]string, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[string]map[string]string, len(s.state.Buckets)) + for name, b := range s.state.Buckets { + metadata := make(map[string]string, len(b.Metadata)) + maps.Copy(metadata, b.Metadata) + out[name] = metadata + } + return out, nil +} + +// SetBucketMetadata durably replaces one opaque attribute, preserving the +// others. The whole bucket's raw UTF-8 metadata is bounded to 65,536 bytes. +// Metadata version 1 is read without changes and upgraded on the next write; +// version 1 readers cannot read the resulting version 2 index. +func (s *Store) SetBucketMetadata(ctx context.Context, name, key, value string) error { + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(name) + if err != nil { + return err + } + metadata := make(map[string]string, len(b.Metadata)+1) + maps.Copy(metadata, b.Metadata) + metadata[key] = value + if err := validMetadata(Object{Metadata: metadata}); err != nil { + return err + } + next := s.clone() + next.Buckets[name].Metadata = metadata + return s.commit(ctx, next) +} diff --git a/internal/core/objectstore/bucket_metadata_test.go b/internal/core/objectstore/bucket_metadata_test.go new file mode 100644 index 00000000..a2eea681 --- /dev/null +++ b/internal/core/objectstore/bucket_metadata_test.go @@ -0,0 +1,147 @@ +package objectstore + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestBucketMetadataPersistsAndDoesNotAlias(t *testing.T) { + s, path := newStore(t) + initial := map[string]string{"initial": "creation"} + if err := s.CreateBucketWithMetadata(t.Context(), "bucket", initial); err != nil { + t.Fatal(err) + } + initial["initial"] = "outside" + if err := s.SetBucketMetadata(t.Context(), "bucket", "first", "one"); err != nil { + t.Fatal(err) + } + got, err := s.BucketMetadata(t.Context(), "bucket") + if err != nil || got["first"] != "one" { + t.Fatal(got, err) + } + got["first"] = "outside" + listed, err := s.ListBucketMetadata(t.Context()) + if err != nil || len(listed) != 1 || listed["bucket"]["first"] != "one" { + t.Fatal(listed, err) + } + listed["bucket"]["first"] = "outside-list" + mustPut(t, s, "bucket", "key", "bytes") + if err := s.SetBucketMetadata(t.Context(), "bucket", "second", "two"); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + got, err = reopened.BucketMetadata(t.Context(), "bucket") + if err != nil || got["initial"] != "creation" || got["first"] != "one" || got["second"] != "two" || len(got) != 3 { + t.Fatal("attributes lost, aliased, or not persisted:", got, err) + } + if _, err := reopened.HeadObject(t.Context(), "bucket", "key"); err != nil { + t.Fatal("attribute update lost an object:", err) + } +} + +func TestBucketMetadataBoundsAndConcurrentFields(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "bucket") + if err := s.SetBucketMetadata(t.Context(), "bucket", "", strings.Repeat("x", 65536)); err != nil { + t.Fatal("exact metadata bound rejected:", err) + } + if err := s.SetBucketMetadata(t.Context(), "bucket", "more", "x"); !errors.Is(err, ErrInvalid) { + t.Fatal("cumulative metadata bound not enforced:", err) + } + if err := s.SetBucketMetadata(t.Context(), "bucket", "", ""); err != nil { + t.Fatal(err) + } + for _, invalid := range []string{"\xff", strings.Repeat("x", 65537)} { + if err := s.SetBucketMetadata(t.Context(), "bucket", "bad", invalid); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid metadata admitted:", err) + } + if err := s.CreateBucketWithMetadata(t.Context(), "invalid", map[string]string{"bad": invalid}); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid initial metadata admitted:", err) + } + } + var wg sync.WaitGroup + for _, key := range []string{"first", "second"} { + wg.Go(func() { + if err := s.SetBucketMetadata(t.Context(), "bucket", key, key); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + got, err := s.BucketMetadata(t.Context(), "bucket") + if err != nil || got["first"] != "first" || got["second"] != "second" || len(got) != 3 { + t.Fatal("concurrent field update lost metadata or a failed write persisted:", got, err) + } + if err := s.SetBucketMetadata(t.Context(), "missing", "key", "value"); !errors.Is(err, ErrBucketNotFound) { + t.Fatal("missing bucket accepted:", err) + } +} + +func TestLegacyBucketMetadataUpgradeAndValidation(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + mustPut(t, s, "bucket", "key", "legacy") + legacy := s.clone() + legacy.Version = 1 + body, err := json.Marshal(legacy) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + index := filepath.Join(path, "index.json") + if err := os.WriteFile(index, body, 0600); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + got, err := reopened.BucketMetadata(t.Context(), "bucket") + if err != nil || len(got) != 0 { + t.Fatal("legacy bucket refused:", got, err) + } + unchanged, err := os.ReadFile(index) + if err != nil || string(unchanged) != string(body) { + t.Fatal("read silently rewrote legacy metadata:", err) + } + if err := reopened.SetBucketMetadata(t.Context(), "bucket", "key", "value"); err != nil { + t.Fatal(err) + } + if reopened.state.Version != 2 { + t.Fatal("write did not upgrade metadata version") + } + if _, err := reopened.HeadObject(t.Context(), "bucket", "key"); err != nil { + t.Fatal("upgrade lost legacy bytes:", err) + } + for _, invalid := range []string{ + `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"g","objects":{},"metadata":{"key":"value"}}}}`, + `{"format":"feint-objectstore","version":2,"buckets":{"b":{"generation":"g","objects":{},"metadata":{"key":"` + strings.Repeat("x", 65537) + `"}}}}`, + } { + if _, err := decodeManifest([]byte(invalid)); !errors.Is(err, ErrCorrupt) { + t.Fatal("restored metadata validation did not reject incompatible/oversized attributes:", err) + } + } +} diff --git a/internal/core/objectstore/files.go b/internal/core/objectstore/files.go index 0fb41170..68d3fa04 100644 --- a/internal/core/objectstore/files.go +++ b/internal/core/objectstore/files.go @@ -99,7 +99,7 @@ func Open(ctx context.Context, path string) (result *Store, err error) { if err := root.Mkdir("blobs", 0o700); err != nil { return nil, fmt.Errorf("create blob directory: %w", err) } - s.state = manifest{Format: storeFormat, Version: 1, Buckets: make(map[string]*bucket)} + s.state = manifest{Format: storeFormat, Version: 2, Buckets: make(map[string]*bucket)} if err := s.commit(ctx, s.state); err != nil { return nil, err } diff --git a/internal/core/objectstore/manifest.go b/internal/core/objectstore/manifest.go index 011b3b3f..8762a8e1 100644 --- a/internal/core/objectstore/manifest.go +++ b/internal/core/objectstore/manifest.go @@ -47,7 +47,7 @@ func decodeManifest(body []byte) (manifest, error) { if state.Format != storeFormat { return manifest{}, ErrUnowned } - if state.Version != 1 || state.Buckets == nil { + if (state.Version != 1 && state.Version != 2) || state.Buckets == nil { return manifest{}, fmt.Errorf("%w: unsupported version or missing buckets", ErrCorrupt) } for name, b := range state.Buckets { @@ -57,6 +57,12 @@ func decodeManifest(body []byte) (manifest, error) { if b == nil || b.Objects == nil || b.Generation == "" || len(b.Generation) > 128 || !utf8.ValidString(b.Generation) { return manifest{}, fmt.Errorf("%w: invalid bucket metadata", ErrCorrupt) } + if state.Version == 1 && b.Metadata != nil { + return manifest{}, fmt.Errorf("%w: bucket attributes require metadata version 2", ErrCorrupt) + } + if err := validMetadata(Object{Metadata: b.Metadata}); err != nil { + return manifest{}, errors.Join(ErrCorrupt, err) + } for key, obj := range b.Objects { if err := validKey(key); err != nil { return manifest{}, errors.Join(ErrCorrupt, err) diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index 35938b9d..b2401f1d 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -55,6 +55,7 @@ type PutOptions struct { type bucket struct { Generation string `json:"generation"` Objects map[string]Object `json:"objects"` + Metadata map[string]string `json:"metadata,omitempty"` } type manifest struct { @@ -114,9 +115,18 @@ func (s *Store) begin(ctx context.Context) (func(), error) { // CreateBucket durably creates a logical bucket. Its name never becomes a path. func (s *Store) CreateBucket(ctx context.Context, name string) error { + return s.CreateBucketWithMetadata(ctx, name, nil) +} + +// CreateBucketWithMetadata atomically creates a bucket with detached, opaque +// UTF-8 attributes. Attributes share SetBucketMetadata's 65,536-byte bound. +func (s *Store) CreateBucketWithMetadata(ctx context.Context, name string, metadata map[string]string) error { if err := validName(name); err != nil { return err } + if err := validMetadata(Object{Metadata: metadata}); err != nil { + return err + } end, err := s.begin(ctx) if err != nil { return err @@ -128,7 +138,7 @@ func (s *Store) CreateBucket(ctx context.Context, name string) error { return ErrBucketExists } next := s.clone() - next.Buckets[name] = &bucket{Generation: rand.Text(), Objects: make(map[string]Object)} + next.Buckets[name] = &bucket{Generation: rand.Text(), Objects: make(map[string]Object), Metadata: maps.Clone(metadata)} return s.commit(ctx, next) } @@ -348,9 +358,9 @@ func (s *Store) lookup(name string) (*bucket, error) { } func (s *Store) clone() manifest { - next := manifest{Format: s.state.Format, Version: s.state.Version, Buckets: make(map[string]*bucket)} + next := manifest{Format: s.state.Format, Version: 2, Buckets: make(map[string]*bucket)} for name, b := range s.state.Buckets { - next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects)} + next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects), Metadata: maps.Clone(b.Metadata)} } return next } diff --git a/internal/core/objectstore/store_test.go b/internal/core/objectstore/store_test.go index e113eac5..fcdfffd7 100644 --- a/internal/core/objectstore/store_test.go +++ b/internal/core/objectstore/store_test.go @@ -753,7 +753,7 @@ func TestManifestIsClosedVersionedAndValidated(t *testing.T) { } for _, body := range []string{ `null`, `{}`, `{"format":"operator","version":1,"buckets":{}}`, - `{"format":"feint-objectstore","version":2,"buckets":{}}`, + `{"format":"feint-objectstore","version":3,"buckets":{}}`, `{"format":"feint-objectstore","version":1,"buckets":null}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":null}}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"","objects":{}}}}`, @@ -820,7 +820,7 @@ func TestFailedOpenReleasesOwnershipLock(t *testing.T) { if err != nil { t.Fatal(err) } - if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":2,"buckets":{}}`), 0o600); err != nil { + if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":3,"buckets":{}}`), 0o600); err != nil { t.Fatal(err) } if broken, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || broken != nil { diff --git a/internal/providers/scaleway/object_storage.go b/internal/providers/scaleway/object_storage.go index 7727e893..528a9779 100644 --- a/internal/providers/scaleway/object_storage.go +++ b/internal/providers/scaleway/object_storage.go @@ -3,8 +3,10 @@ package scaleway import "github.com/stephrobert/feint/internal/s3" // ObjectStorage declares the measured S3 dialect and configurable CLI endpoint. -// It does not advertise Terraform redirection or ancillary S3 operations. +// It enables the measured private, non-versioned bucket CLI workflow, not +// Terraform redirection, public permissions or object version histories. func (p *Pack) ObjectStorage(region string) s3.Profile { return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ScalewayMissingBucket, - Variables: []string{"SCW_S3_ENDPOINT"}, RegionVariable: "SCW_DEFAULT_REGION"} + Variables: []string{"SCW_S3_ENDPOINT"}, RegionVariable: "SCW_DEFAULT_REGION", + BucketConfiguration: true} } diff --git a/internal/s3/bucket_configuration.go b/internal/s3/bucket_configuration.go new file mode 100644 index 00000000..122f3bbd --- /dev/null +++ b/internal/s3/bucket_configuration.go @@ -0,0 +1,272 @@ +package s3 + +import ( + "bytes" + "crypto/md5" //nolint:gosec // Content-MD5 is the client's transport checksum, not an ETag or authentication. + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "slices" + "strings" + "time" +) + +const ( + s3XMLNamespace = "http://s3.amazonaws.com/doc/2006-03-01/" + bucketVersioning = "s3.versioning" + bucketCreated = "s3.created" +) + +// Bucket configuration remains separate from object upload header support. +// Only the measured create defaults and versioning checksum are admitted. +func (s *Server) bucketHeaders(r *http.Request, query url.Values, bucket bool) (*http.Request, error) { + if !s.cfg.BucketConfiguration || !bucket || r.Method != http.MethodPut { + return r, nil + } + checked := r.Clone(r.Context()) + if query.Has("versioning") { + checked.Header.Del("Content-MD5") + return checked, nil + } + if len(query) != 0 && (len(query) != 1 || !query.Has("x-id")) { + return checked, nil + } + for name, allowed := range map[string]string{"X-Amz-Acl": "private", "X-Amz-Bucket-Object-Lock-Enabled": "false"} { + values := r.Header.Values(name) + if len(values) != 0 { + if len(values) != 1 || values[0] != allowed { + return nil, fmt.Errorf("unsupported bucket default for %s", name) + } + checked.Header.Del(name) + } + } + return checked, nil +} + +func configurationBody(w http.ResponseWriter, r *http.Request) ([]byte, error) { + upload, _, err := newUpload(w, r, 64<<10) + if err != nil { + return nil, err + } + body, err := io.ReadAll(upload) + if err != nil { + return nil, err + } + if values := r.Header.Values("Content-MD5"); len(values) != 0 { + digest := md5.Sum(body) //nolint:gosec // Verify the legacy S3 transport checksum; never use it as identity. + if len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:]) { + return nil, errors.New("bucket configuration Content-MD5 mismatch") + } + } + return body, nil +} + +// configurationValue accepts one leaf, not an XML document with silently +// discarded fields. TestBucketConfigurationRefusals holds that closed shape. +func configurationValue(body []byte, rootName, leafName string) (string, error) { + decoder := xml.NewDecoder(bytes.NewReader(body)) //nolint:gosec // Bounded body, no external entity resolver, directives rejected below. + depth, roots, leaves := 0, 0, 0 + var value strings.Builder + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", err + } + switch t := token.(type) { + case xml.StartElement: + if (t.Name.Space != "" && t.Name.Space != s3XMLNamespace) || + (depth == 0 && (roots != 0 || t.Name.Local != rootName)) || + (depth == 1 && (leaves != 0 || t.Name.Local != leafName)) || depth > 1 { + return "", errors.New("unexpected bucket configuration element") + } + for _, attr := range t.Attr { + if attr.Name.Space != "" || attr.Name.Local != "xmlns" || attr.Value != s3XMLNamespace { + return "", errors.New("unexpected bucket configuration attribute") + } + } + if depth == 0 { + roots++ + } else { + leaves++ + } + depth++ + case xml.EndElement: + depth-- + case xml.CharData: + if depth == 2 { + value.Write(t) + } else if strings.TrimSpace(string(t)) != "" { + return "", errors.New("unexpected bucket configuration text") + } + case xml.ProcInst: + if roots != 0 || t.Target != "xml" { + return "", errors.New("unexpected bucket configuration instruction") + } + default: + return "", errors.New("unexpected bucket configuration token") + } + } + if depth != 0 || roots != 1 || leaves != 1 { + return "", errors.New("bucket configuration requires exactly one value") + } + return value.String(), nil +} + +type versioningConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration"` + Status string `xml:",omitempty"` +} + +func (s *Server) bucketConfiguration(w http.ResponseWriter, r *http.Request, name string, query url.Values) { + feature := "" + for key, values := range query { + if key == "x-id" { + continue + } + if feature != "" || len(values) != 1 || values[0] != "" { + s.refuse(w, r, http.StatusBadRequest, "one empty bucket configuration query is required", nil) + return + } + feature = key + } + metadata, err := s.store.BucketMetadata(r.Context(), s.name(name)) + if err != nil { + s.storeError(w, r, name, err) + return + } + switch { + case feature == "versioning" && r.Method == http.MethodPut: + body, readErr := configurationBody(w, r) + if readErr != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid bucket versioning body", readErr) + return + } + status, decodeErr := configurationValue(body, "VersioningConfiguration", "Status") + if decodeErr != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid bucket versioning XML", decodeErr) + return + } + if status != "Suspended" { + s.refuse(w, r, http.StatusNotImplemented, "object version histories are not implemented", nil) + return + } + if err := s.store.SetBucketMetadata(r.Context(), s.name(name), bucketVersioning, status); err != nil { + s.storeError(w, r, name, err) + return + } + w.WriteHeader(http.StatusOK) + case feature == "versioning" && r.Method == http.MethodGet: + status := metadata[bucketVersioning] + if status != "" && status != "Suspended" { + s.refuse(w, r, http.StatusInternalServerError, "persisted bucket versioning is not supported", nil) + return + } + s.writeXML(w, r, http.StatusOK, versioningConfiguration{Status: status}) + case feature == "acl" && r.Method == http.MethodGet: + owner := s.bucketOwner() + s.writeXML(w, r, http.StatusOK, accessControlPolicy{Owner: owner, Grant: aclGrant{ + Grantee: aclGrantee{Type: "CanonicalUser", XSI: "http://www.w3.org/2001/XMLSchema-instance", + ID: owner.ID, DisplayName: owner.DisplayName}, Permission: "FULL_CONTROL", + }}) + case feature == "tagging" && r.Method == http.MethodGet: + id := rand.Text() + w.Header().Set("X-Amz-Request-Id", id) + w.Header().Set("X-Amz-Id-2", id) + s.writeXML(w, r, http.StatusNotFound, missingBucketError{Code: "NoSuchTagSet", + Message: "The TagSet does not exist", RequestID: id, HostID: id, Resource: "/" + name, BucketName: name}) + default: + s.refuse(w, r, http.StatusNotImplemented, "bucket configuration operation is not implemented", nil) + } +} + +// These paths, namespaces and the canonical private grant follow the official +// v2.62.0 create cassette. The owner is a stable local identity, not cloud IAM. +type bucketOwner struct { + DisplayName string + ID string +} + +type aclGrantee struct { + XSI string `xml:"xmlns:xsi,attr"` + Type string `xml:"xsi:type,attr"` + ID string + DisplayName string +} + +type aclGrant struct { + Grantee aclGrantee + Permission string +} + +type accessControlPolicy struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"` + Owner bucketOwner + Grant aclGrant `xml:"AccessControlList>Grant"` +} + +func (s *Server) bucketOwner() bucketOwner { + sum := sha256.Sum256([]byte("feint-local-owner/" + s.cfg.Namespace)) + return bucketOwner{DisplayName: "feint", ID: hex.EncodeToString(sum[:])} +} + +type listedBucket struct { + Name string + CreationDate string +} + +type listedBuckets struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"` + Owner bucketOwner + Buckets []listedBucket `xml:"Buckets>Bucket"` +} + +func (s *Server) listBuckets(w http.ResponseWriter, r *http.Request) { + inventory, err := s.store.ListBucketMetadata(r.Context()) + if err != nil { + s.storeError(w, r, "", err) + return + } + result := listedBuckets{Owner: s.bucketOwner()} + for _, name := range slices.Sorted(maps.Keys(inventory)) { + wireName := name + if s.cfg.Namespace != "" { + var matches bool + wireName, matches = strings.CutPrefix(name, s.cfg.Namespace+"/") + if !matches { + continue + } + } + metadata := inventory[name] + created := metadata[bucketCreated] + if _, err := time.Parse(time.RFC3339Nano, created); err != nil || !bucketName(wireName) { + s.refuse(w, r, http.StatusNotImplemented, "legacy bucket listing has no measured creation metadata", err) + return + } + result.Buckets = append(result.Buckets, listedBucket{Name: wireName, CreationDate: created}) + } + s.writeXML(w, r, http.StatusOK, result) +} + +func (s *Server) writeXML(w http.ResponseWriter, r *http.Request, status int, value any) { + body, err := xml.Marshal(value) + if err != nil { + s.refuse(w, r, http.StatusInternalServerError, "bucket response encoding failed", err) + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + if _, err := w.Write(body); err != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 bucket response write failed", "error", err) + } +} diff --git a/internal/s3/bucket_configuration_test.go b/internal/s3/bucket_configuration_test.go new file mode 100644 index 00000000..6b4fa64d --- /dev/null +++ b/internal/s3/bucket_configuration_test.go @@ -0,0 +1,175 @@ +package s3 + +import ( + "crypto/md5" //nolint:gosec // Fixture for the measured legacy transport checksum. + "encoding/base64" + "encoding/xml" + "errors" + "net/http" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +const suspendedXML = `Suspended` + +func bucketFixture(t *testing.T) *Server { + t.Helper() + s, _ := fixture(t) + s.cfg.BucketConfiguration = true + s.cfg.MissingBucket = ScalewayMissingBucket + s.cfg.Namespace = "scaleway/fr-par" + return s +} + +func TestBucketConfigurationRoundTrip(t *testing.T) { + s := bucketFixture(t) + body := []byte(`fr-par`) + w := request(t, s, http.MethodPut, "/bucket", body, map[string]string{ + "X-Amz-Acl": "private", "X-Amz-Bucket-Object-Lock-Enabled": "false", + }) + if w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + w = request(t, s, http.MethodGet, "/bucket?versioning=", nil, nil) + var initial versioningConfiguration + if w.Code != http.StatusOK || xml.Unmarshal(w.Body.Bytes(), &initial) != nil || initial.Status != "" { + t.Fatal("new bucket claimed an unrequested versioning configuration:", w.Code, w.Body.String()) + } + sum := md5.Sum([]byte(suspendedXML)) //nolint:gosec // S3 request transport checksum. + w = request(t, s, http.MethodPut, "/bucket?versioning=", []byte(suspendedXML), + map[string]string{"Content-MD5": base64.StdEncoding.EncodeToString(sum[:])}) + if w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + metadata, err := s.store.BucketMetadata(t.Context(), s.name("bucket")) + if err != nil || metadata[bucketVersioning] != "Suspended" { + t.Fatal("successful configuration did not persist its state:", metadata, err) + } + if _, err := time.Parse(time.RFC3339Nano, metadata[bucketCreated]); err != nil { + t.Fatal("bucket creation has no durable creation time:", err) + } +} + +func TestBucketPrivateACLTagsAndScopedListing(t *testing.T) { + s := bucketFixture(t) + for _, name := range []string{"bucket", "second"} { + if w := request(t, s, http.MethodPut, "/"+name, nil, nil); w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + } + if err := s.store.CreateBucketWithMetadata(t.Context(), "exoscale/ch-gva-2/foreign", + map[string]string{bucketCreated: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil { + t.Fatal(err) + } + w := request(t, s, http.MethodGet, "/", nil, nil) + var listed listedBuckets + if w.Code != http.StatusOK || xml.Unmarshal(w.Body.Bytes(), &listed) != nil || len(listed.Buckets) != 2 || + listed.Buckets[0].Name != "bucket" || listed.Buckets[1].Name != "second" || listed.Buckets[0].CreationDate == "" { + t.Fatal("bucket listing is not the actual scoped inventory:", w.Code, w.Body.String()) + } + w = request(t, s, http.MethodGet, "/bucket?acl=", nil, nil) + var acl accessControlPolicy + if w.Code != http.StatusOK || xml.Unmarshal(w.Body.Bytes(), &acl) != nil || + acl.Owner.ID == "" || acl.Owner != s.bucketOwner() || acl.Grant.Permission != "FULL_CONTROL" || + acl.Grant.Grantee.ID != acl.Owner.ID || acl.Grant.Grantee.DisplayName != acl.Owner.DisplayName || + !strings.Contains(w.Body.String(), `xsi:type="CanonicalUser"`) { + t.Fatal("default private canonical grant is not truthful:", w.Code, w.Body.String()) + } + w = request(t, s, http.MethodGet, "/bucket?tagging=", nil, nil) + var missing missingBucketError + if w.Code != http.StatusNotFound || xml.Unmarshal(w.Body.Bytes(), &missing) != nil || + missing.Code != "NoSuchTagSet" || missing.Message != "The TagSet does not exist" || + missing.BucketName != "bucket" || missing.RequestID == "" || missing.RequestID != missing.HostID { + t.Fatal("absent tags do not follow the recorded cloud error:", w.Code, w.Body.String()) + } +} + +func TestBucketConfigurationRefusals(t *testing.T) { + s := bucketFixture(t) + if w := request(t, s, http.MethodPut, "/bucket", nil, nil); w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + for _, tc := range []struct { + name, method, path, body string + headers map[string]string + status int + }{ + {"region", "PUT", "/wrong-region", `nl-ams`, nil, 400}, + {"public-acl", "PUT", "/public-bucket", "", map[string]string{"X-Amz-Acl": "public-read"}, 501}, + {"lock", "PUT", "/locked-bucket", "", map[string]string{"X-Amz-Bucket-Object-Lock-Enabled": "true"}, 501}, + {"enabled", "PUT", "/bucket?versioning=", strings.ReplaceAll(suspendedXML, "Suspended", "Enabled"), nil, 501}, + {"checksum", "PUT", "/bucket?versioning=", suspendedXML, map[string]string{"Content-MD5": "AAAAAAAAAAAAAAAAAAAAAA=="}, 400}, + {"unknown-field", "PUT", "/bucket?versioning=", strings.ReplaceAll(suspendedXML, "Status", "Unknown"), nil, 400}, + {"unknown-attribute", "PUT", "/bucket?versioning=", strings.ReplaceAll(suspendedXML, "", ``), nil, 400}, + {"duplicate-field", "PUT", "/bucket?versioning=", `SuspendedSuspended`, nil, 400}, + {"missing-field", "PUT", "/bucket?versioning=", ``, nil, 400}, + {"multiple-query", "GET", "/bucket?versioning=&acl=", "", nil, 400}, + {"duplicate-query", "GET", "/bucket?versioning=&versioning=", "", nil, 400}, + {"query-value", "GET", "/bucket?versioning=other", "", nil, 400}, + {"object-query", "PUT", "/bucket/key?versioning=", "bytes", nil, 501}, + {"tags-write", "PUT", "/bucket?tagging=", ``, nil, 501}, + {"large", "PUT", "/bucket?versioning=", strings.Repeat("x", 65537), nil, 400}, + } { + t.Run(tc.name, func(t *testing.T) { + w := request(t, s, tc.method, tc.path, []byte(tc.body), tc.headers) + if w.Code != tc.status || !strings.HasPrefix(w.Body.String(), "feint: ") { + t.Fatal("unsupported configuration acquired a cloud success:", w.Code, w.Body.String()) + } + }) + } + metadata, err := s.store.BucketMetadata(t.Context(), s.name("bucket")) + if err != nil || metadata[bucketVersioning] != "" { + t.Fatal("refused writes changed bucket state:", metadata, err) + } + for _, name := range []string{"wrong-region", "public-bucket", "locked-bucket"} { + if err := s.store.HeadBucket(t.Context(), s.name(name)); !errors.Is(err, objectstore.ErrBucketNotFound) { + t.Fatal("refused create published a bucket:", name, err) + } + } +} + +func TestBucketConfigurationValueRequiresOneLeaf(t *testing.T) { + for _, body := range []string{ + ``, + `Suspended`, + `Suspended`, + `Suspended`, + } { + if _, err := configurationValue([]byte(body), "VersioningConfiguration", "Status"); err == nil { + t.Fatal("invalid closed configuration acquired a parsed value:", body) + } + } + if status, err := configurationValue([]byte(suspendedXML), "VersioningConfiguration", "Status"); err != nil || status != "Suspended" { + t.Fatal("the accepting configuration failed:", status, err) + } +} + +func TestRestoredBucketConfigurationIsRevalidated(t *testing.T) { + s := bucketFixture(t) + if err := s.store.CreateBucket(t.Context(), s.name("legacy")); err != nil { + t.Fatal(err) + } + w := request(t, s, http.MethodGet, "/", nil, nil) + if w.Code != http.StatusNotImplemented { + t.Fatal("legacy creation date was invented:", w.Code, w.Body.String()) + } + if err := s.store.SetBucketMetadata(t.Context(), s.name("legacy"), bucketVersioning, "Enabled"); err != nil { + t.Fatal(err) + } + w = request(t, s, http.MethodGet, "/legacy?versioning=", nil, nil) + if w.Code != http.StatusInternalServerError { + t.Fatal("restored version histories were advertised:", w.Code, w.Body.String()) + } +} + +func FuzzBucketConfigurationValue(f *testing.F) { + f.Add(suspendedXML) + f.Add(``) + f.Add(`Suspended`) + f.Fuzz(func(t *testing.T, body string) { + _, _ = configurationValue([]byte(body), "VersioningConfiguration", "Status") + }) +} diff --git a/internal/s3/doors.go b/internal/s3/doors.go index 4fb41966..923452df 100644 --- a/internal/s3/doors.go +++ b/internal/s3/doors.go @@ -28,11 +28,12 @@ const UploadReadTimeout = 30 * time.Second // Profile is a provider's declared wire dialect and client environment names. // Variables stay local to the composition root, never taken from an HTTP report. type Profile struct { - Provider string - Region string - MissingBucket MissingBucketShape - Variables []string - RegionVariable string + Provider string + Region string + MissingBucket MissingBucketShape + Variables []string + RegionVariable string + BucketConfiguration bool } // Door describes an enabled path-style endpoint, without filesystem paths. @@ -107,7 +108,8 @@ func Mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxO return nil, fmt.Errorf("s3: duplicate door %s", path) } server, err := New(store, Config{Region: p.Region, MaxObjectBytes: maxObjectBytes, - MissingBucket: p.MissingBucket, Namespace: p.Provider + "/" + p.Region, Log: log}) + MissingBucket: p.MissingBucket, Namespace: p.Provider + "/" + p.Region, Log: log, + BucketConfiguration: p.BucketConfiguration}) if err != nil { return nil, err } @@ -149,7 +151,13 @@ func (d *Doors) ServeHTTP(w http.ResponseWriter, r *http.Request) { } raw := r.URL.EscapedPath() for prefix, handler := range d.handlers { - if remainder, ok := strings.CutPrefix(raw, prefix); ok { + remainder, ok := strings.CutPrefix(raw, prefix) + // SCW ListBuckets uses the canonical bare endpoint rather than its + // slash-suffixed root. No redirect or path cleaning is involved. + if !ok && r.Method == http.MethodGet && handler.cfg.BucketConfiguration && raw == strings.TrimSuffix(prefix, "/") { + remainder, ok = "", true + } + if ok { path, err := url.PathUnescape("/" + remainder) if err != nil { d.refuse(w, r, "invalid encoded S3 path") diff --git a/internal/s3/doors_test.go b/internal/s3/doors_test.go index eeec166b..1216146e 100644 --- a/internal/s3/doors_test.go +++ b/internal/s3/doors_test.go @@ -122,6 +122,29 @@ func TestMountedDoorRefusesUnknownAndEncodedPrefixesWithoutRedirect(t *testing.T } } +func TestMountedBucketListingAcceptsOnlyItsCanonicalBareEndpoint(t *testing.T) { + doors, _ := mountedFixture(t) + prefix := DoorPath("scaleway", "fr-par") + doors.handlers[prefix].cfg.BucketConfiguration = true + ts := httptest.NewServer(doors) + defer ts.Close() + doorRequest(t, ts.Client(), "PUT", ts.URL+prefix+"bucket", "", 200) + bare := strings.TrimSuffix(prefix, "/") + for _, path := range []string{prefix, bare, prefix + "?x-id=ListBuckets", bare + "?x-id=ListBuckets"} { + body := doorRequest(t, ts.Client(), "GET", ts.URL+path, "", 200) + if !bytes.Contains(body, []byte("bucket")) { + t.Fatal("canonical bucket root omitted actual inventory:", string(body)) + } + } + for _, path := range []string{ + strings.Replace(bare, "/scaleway/", "/scaleway%2F", 1), + bare + "-foreign", + } { + doorRequest(t, ts.Client(), "GET", ts.URL+path, "", 404) + } + doorRequest(t, ts.Client(), "PUT", ts.URL+bare, "", 404) +} + type deadlineWriter struct { *httptest.ResponseRecorder deadline time.Time diff --git a/internal/s3/server.go b/internal/s3/server.go index 8a6dbaae..4f6fe916 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -21,6 +21,7 @@ import ( "net/url" "strconv" "strings" + "time" "unicode" "unicode/utf8" @@ -48,6 +49,9 @@ type Config struct { Log *slog.Logger // Namespace is an opaque deployment scope, not an authenticated tenant. Namespace string + // BucketConfiguration enables the measured private, non-versioned bucket + // workflow, independently of the missing-bucket error dialect. + BucketConfiguration bool } // Server is a direct HTTP handler. Do not put it behind a path-cleaning mux: @@ -100,26 +104,44 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } for key := range query { - if key != "x-id" { + if key != "x-id" && (!s.cfg.BucketConfiguration || (key != "versioning" && key != "acl" && key != "tagging")) { s.refuse(w, r, http.StatusNotImplemented, "S3 query features are not implemented", nil) return } } - if err := supportedHeaders(r); err != nil { - s.refuse(w, r, http.StatusNotImplemented, "S3 header feature is not implemented", err) - return - } rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") bucket, bucketErr := url.PathUnescape(rawBucket) key, keyErr := url.PathUnescape(rawKey) + checked, err := s.bucketHeaders(r, query, rawKey == "") + if err != nil { + s.refuse(w, r, http.StatusNotImplemented, "S3 bucket header feature is not implemented", err) + return + } + if err := supportedHeaders(checked); err != nil { + s.refuse(w, r, http.StatusNotImplemented, "S3 header feature is not implemented", err) + return + } + if rawBucket == "" && rawKey == "" && s.cfg.BucketConfiguration && + (len(query) == 0 || (len(query) == 1 && query.Has("x-id"))) && r.Method == http.MethodGet { + s.listBuckets(w, r) + return + } if bucketErr != nil || keyErr != nil || !bucketName(bucket) || !utf8.ValidString(key) || len(key) > 1024 { s.refuse(w, r, http.StatusBadRequest, "invalid bucket or object key", errors.Join(bucketErr, keyErr)) return } if key == "" { + if len(query) != 0 && (len(query) != 1 || !query.Has("x-id")) { + s.bucketConfiguration(w, r, bucket, query) + return + } s.bucket(w, r, bucket) return } + if query.Has("versioning") || query.Has("acl") || query.Has("tagging") { + s.refuse(w, r, http.StatusNotImplemented, "bucket configuration queries require a bucket, not an object", nil) + return + } switch r.Method { case http.MethodPut: s.put(w, r, bucket, key) @@ -142,8 +164,24 @@ func (s *Server) bucket(w http.ResponseWriter, r *http.Request, name string) { status := http.StatusOK switch r.Method { case http.MethodPut: - // This slice accepts only the empty CreateBucket request. Region XML - // configuration must be measured before enabling that request shape. + if s.cfg.BucketConfiguration { + body, readErr := configurationBody(w, r) + if readErr != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid CreateBucket body", readErr) + return + } + if len(body) != 0 { + region, decodeErr := configurationValue(body, "CreateBucketConfiguration", "LocationConstraint") + if decodeErr != nil || region != s.cfg.Region { + s.refuse(w, r, http.StatusBadRequest, "CreateBucket region must match the declared door", decodeErr) + return + } + } + err = s.store.CreateBucketWithMetadata(r.Context(), s.name(name), map[string]string{ + bucketCreated: time.Now().UTC().Format(time.RFC3339Nano), + }) + break + } body, readErr := io.ReadAll(http.MaxBytesReader(w, r.Body, 1)) if readErr != nil || len(body) != 0 { s.refuse(w, r, http.StatusNotImplemented, "CreateBucket configuration is not implemented", readErr) diff --git a/mise.toml b/mise.toml index 9bbc8da4..b01b7ad4 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove prefixed S3 bytes through the actual feint start/stop/restart process with AWS CLI 2.32.17" +description = "Prove prefixed bytes and private Scaleway buckets through real start/stop/restart with AWS CLI 2.32.17 and scw 2.62.0" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks.fuzz] diff --git a/tools/falsify/specs/neutral-object-bytes.json b/tools/falsify/specs/neutral-object-bytes.json index d0cca6d4..ed4bac0d 100644 --- a/tools/falsify/specs/neutral-object-bytes.json +++ b/tools/falsify/specs/neutral-object-bytes.json @@ -88,8 +88,8 @@ { "label": "a future metadata version is silently accepted", "file": "internal/core/objectstore/manifest.go", - "find": "if state.Version != 1 || state.Buckets == nil {", - "replace": "if (state.Version != 1 && false) || state.Buckets == nil {", + "find": "if (state.Version != 1 && state.Version != 2) || state.Buckets == nil {", + "replace": "if ((state.Version != 1 && state.Version != 2) && false) || state.Buckets == nil {", "test": "TestManifestIsClosedVersionedAndValidated" }, { diff --git a/tools/falsify/specs/s3-bucket-configuration.json b/tools/falsify/specs/s3-bucket-configuration.json new file mode 100644 index 00000000..01918cd9 --- /dev/null +++ b/tools/falsify/specs/s3-bucket-configuration.json @@ -0,0 +1,153 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "bucket create aliases the supplied attributes", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/store.go", + "find": "Metadata: maps.Clone(metadata)}", + "replace": "Metadata: func() map[string]string { _ = maps.Clone(metadata); return metadata }()}", + "test": "TestBucketMetadataPersistsAndDoesNotAlias" + }, + { + "label": "initial bucket metadata bypasses its UTF-8 and byte bound", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/store.go", + "find": "if err := validMetadata(Object{Metadata: metadata}); err != nil {", + "replace": "if err := validMetadata(Object{Metadata: metadata}); err != nil && false {", + "test": "TestBucketMetadataBoundsAndConcurrentFields" + }, + { + "label": "bucket metadata updates bypass their cumulative bound", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/bucket_metadata.go", + "find": "if err := validMetadata(Object{Metadata: metadata}); err != nil {", + "replace": "if err := validMetadata(Object{Metadata: metadata}); err != nil && false {", + "test": "TestBucketMetadataBoundsAndConcurrentFields" + }, + { + "label": "bucket attribute reads alias persisted state", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/bucket_metadata.go", + "find": "maps.Copy(out, b.Metadata)", + "replace": "maps.Copy(out, b.Metadata)\n\tout = b.Metadata", + "test": "TestBucketMetadataPersistsAndDoesNotAlias" + }, + { + "label": "inventory attributes alias persisted state", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/bucket_metadata.go", + "find": "out[name] = metadata", + "replace": "out[name] = metadata\n\t\tout[name] = b.Metadata", + "test": "TestBucketMetadataPersistsAndDoesNotAlias" + }, + { + "label": "restored bucket attributes bypass validation", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/manifest.go", + "find": "if err := validMetadata(Object{Metadata: b.Metadata}); err != nil {", + "replace": "if err := validMetadata(Object{Metadata: b.Metadata}); err != nil && false {", + "test": "TestLegacyBucketMetadataUpgradeAndValidation" + }, + { + "label": "version 1 silently accepts version 2 attributes", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/manifest.go", + "find": "if state.Version == 1 && b.Metadata != nil {", + "replace": "if state.Version == 1 && b.Metadata != nil && false {", + "test": "TestLegacyBucketMetadataUpgradeAndValidation" + }, + { + "label": "the create region need not match its declared door", + "file": "internal/s3/server.go", + "find": "if decodeErr != nil || region != s.cfg.Region {", + "replace": "if decodeErr != nil || (region != s.cfg.Region && false) {", + "test": "TestBucketConfigurationRefusals/region" + }, + { + "label": "non-private ACLs and Object Lock are acknowledged without implementation", + "file": "internal/s3/bucket_configuration.go", + "find": "if len(values) != 1 || values[0] != allowed {", + "replace": "if (len(values) != 1 || values[0] != allowed) && false {", + "test": "TestBucketConfigurationRefusals/public-acl|TestBucketConfigurationRefusals/lock" + }, + { + "label": "invalid configuration MD5 is acknowledged", + "file": "internal/s3/bucket_configuration.go", + "find": "if len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:]) {", + "replace": "if (len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:])) && false {", + "test": "TestBucketConfigurationRefusals/checksum" + }, + { + "label": "unknown XML configuration leaves are silently accepted", + "file": "internal/s3/bucket_configuration.go", + "find": "(depth == 1 && (leaves != 0 || t.Name.Local != leafName))", + "replace": "(depth == 1 && (leaves != 0 || (t.Name.Local != leafName && false)))", + "test": "TestBucketConfigurationRefusals/unknown-field" + }, + { + "label": "unknown XML configuration attributes are discarded", + "file": "internal/s3/bucket_configuration.go", + "find": "if attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace {", + "replace": "if (attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace) && false {", + "test": "TestBucketConfigurationRefusals/unknown-attribute" + }, + { + "label": "an empty XML document produces a configuration value", + "file": "internal/s3/bucket_configuration.go", + "find": "if depth != 0 || roots != 1 || leaves != 1 {", + "replace": "if (depth != 0 || roots != 1 || leaves != 1) && false {", + "test": "TestBucketConfigurationValueRequiresOneLeaf" + }, + { + "label": "version histories are falsely enabled", + "file": "internal/s3/bucket_configuration.go", + "find": "if status != \"Suspended\" {", + "replace": "if status != \"Suspended\" && false {", + "test": "TestBucketConfigurationRefusals/enabled" + }, + { + "label": "restored unsupported versioning is published", + "file": "internal/s3/bucket_configuration.go", + "find": "if status != \"\" && status != \"Suspended\" {", + "replace": "if status != \"\" && status != \"Suspended\" && false {", + "test": "TestRestoredBucketConfigurationIsRevalidated" + }, + { + "label": "duplicate query values silently select a bucket feature", + "file": "internal/s3/bucket_configuration.go", + "find": "if feature != \"\" || len(values) != 1 || values[0] != \"\" {", + "replace": "if (feature != \"\" || len(values) != 1 || values[0] != \"\") && false {", + "test": "TestBucketConfigurationRefusals/duplicate-query" + }, + { + "label": "bucket configuration queries become ordinary object writes", + "file": "internal/s3/server.go", + "find": "if query.Has(\"versioning\") || query.Has(\"acl\") || query.Has(\"tagging\") {", + "replace": "if (query.Has(\"versioning\") || query.Has(\"acl\") || query.Has(\"tagging\")) && false {", + "test": "TestBucketConfigurationRefusals/object-query" + }, + { + "label": "legacy buckets receive an invented listing date", + "file": "internal/s3/bucket_configuration.go", + "find": "if _, err := time.Parse(time.RFC3339Nano, created); err != nil || !bucketName(wireName) {", + "replace": "if _, err := time.Parse(time.RFC3339Nano, created); (err != nil || !bucketName(wireName)) && false {", + "test": "TestRestoredBucketConfigurationIsRevalidated" + }, + { + "label": "the bare endpoint retains the original unmatched path", + "file": "internal/s3/doors.go", + "find": "remainder, ok = \"\", true", + "replace": "remainder, ok = strings.TrimPrefix(raw, \"\\x00never\"), true", + "test": "TestMountedBucketListingAcceptsOnlyItsCanonicalBareEndpoint" + }, + { + "label": "SDK endpoint exports retain the refused terminal slash", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "vars[key] = strings.TrimSuffix(endpoint, \"/\") + strings.TrimSuffix(selected.Path, \"/\")", + "replace": "vars[key] = strings.TrimSuffix(endpoint, \"/\") + selected.Path", + "test": "TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline" + } + ] +} diff --git a/tools/falsify/specs/s3-gateway.json b/tools/falsify/specs/s3-gateway.json index 23bd4016..84002d23 100644 --- a/tools/falsify/specs/s3-gateway.json +++ b/tools/falsify/specs/s3-gateway.json @@ -53,8 +53,8 @@ { "label": "S3 query features silently become ordinary CRUD", "file": "internal/s3/server.go", - "find": "if key != \"x-id\" {", - "replace": "if key != \"x-id\" && false {", + "find": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\")) {", + "replace": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\")) && false {", "test": "TestRefusalsDoNotMasqueradeAsCloudSupport" }, { diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index eefb3059..4be64841 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -83,7 +83,7 @@ var rules = []rule{ Path: "internal/providers/scaleway/object_storage.go", Why: "declared S3 dialect and configurable endpoint variables, outside control-plane SDK coverage", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, - Unproven: "AWS CLI byte operations, not a complete scw lifecycle or hardcoded Terraform S3 redirection", + Unproven: "AWS CLI bytes and private non-versioned scw buckets, not other bucket options or hardcoded Terraform S3 redirection", }, { Path: "internal/providers/exoscale/object_storage.go", @@ -254,13 +254,13 @@ var rules = []rule{ }, { Path: "internal/s3/", - Why: "bounded path-style S3 protocol and real-byte official-client proof", + Why: "bounded path-style S3 protocol, private Scaleway buckets and real-byte official-client proof", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, - Unproven: "one AWS CLI version, not complete scw/exo lifecycles, virtual hosts, interception, ETag semantics, multipart or public stacks", + Unproven: "one AWS CLI and one scw version, not complete Exoscale lifecycles, public ACLs, version histories, virtual hosts, interception, ETags, multipart or public stacks", }, { Path: "internal/conformancetest/", - Why: "isolated pinned AWS CLI environment and non-forwarding egress control", + Why: "isolated pinned AWS/scw environments and non-forwarding egress control", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "the proxy is not a network namespace and does not constrain arbitrary clients which ignore it", }, From be0463d86e0fc0236a82848b505e2700cc7b9fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 09:07:29 +0100 Subject: [PATCH 06/28] test(s3): measure Exoscale ignoring endpoint prefixes Pin exo 1.95.1 and prove locally that both unsigned bucket discovery and subsequent upload omit the configured path prefix. The measurement listener refuses PUT deliberately; this does not certify an Exoscale lifecycle. Preserve the mounted AWS/scw witness and document the addressing blocker before changing the accepted door design. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 6 ++ CHANGELOG.md | 6 ++ docs/object-storage-routing.md | 27 ++++++ internal/cli/exo_endpoint_conformance_test.go | 95 +++++++++++++++++++ internal/conformancetest/exo.go | 31 ++++++ mise.toml | 4 + tools/testplan/rules.go | 6 ++ 7 files changed, 175 insertions(+) create mode 100644 internal/cli/exo_endpoint_conformance_test.go create mode 100644 internal/conformancetest/exo.go diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index a89be4d5..b72894ee 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,12 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **L'incompatibilité Exoscale avec les préfixes S3, mesurée** (#850) : + `conformance:s3:exo:endpoint` prouve qu'exo 1.95.1 retire le préfixe + configuré de la découverte du bucket comme de l'upload. Ce listener refuse + volontairement le PUT ; il ne certifie pas un cycle d'octets. Les portes + préfixées restent prouvées avec AWS CLI/scw, pas exo. + - **Des octets S3 simples, persistants et opt-in sur le port existant** (#850) : `serve`/`start --objectstore`, `--s3-doors`, `--objectstore-budget-mib` (1024 par défaut) et `--s3-object-limit-mib` (256 par défaut). diff --git a/CHANGELOG.md b/CHANGELOG.md index bfee54db..e46999ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Measured Exoscale S3 prefix incompatibility** (#850): + `conformance:s3:exo:endpoint` proves that exo 1.95.1 removes the configured + URL prefix from both bucket discovery and object upload. This listener + deliberately refuses the PUT; it does not certify a byte lifecycle. + Prefixed mounted doors remain proved with AWS CLI/scw, not exo. + - **Opt-in, persisted simple S3 bytes on the existing port** (#850): `serve`/`start --objectstore`, `--s3-doors`, `--objectstore-budget-mib` (1024 by default) and `--s3-object-limit-mib` (256 by default). diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 3095c419..528b202e 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -216,3 +216,30 @@ than an invented date. Resource snapshots still do not own this metadata. Exoscale upload/download/delete, its mandatory ListMultipartUploads during bucket removal, native bucket-usage API, and full Object Storage 1.0 remain separate follow-up work. No empty multipart success stub was added. + +## Exoscale prefix incompatibility measured + +`mise run conformance:s3:exo:endpoint` pins **exo 1.95.1**, revision +`7aaf76d0b13b8e10a343ec88c9cb03116f09b4c1`, with an isolated HOME, +pack-owned public fake credentials and a positively tested refusing proxy. +A local measurement listener receives the commands; it never forwards them. + +With `EXOSCALE_SOS_ENDPOINT` and `EXOSCALE_STORAGE_API_ENDPOINT` set to +`http://127.0.0.1:/_feint/s3/exoscale/ch-gva-2/`, `storage upload` +issues **HEAD /feint-bucket**, without the configured prefix. When the +measurement listener supplies the bucket-region header, the next request is +**PUT /feint-bucket/exo-byte-proof**, also without that prefix. +The PUT is deliberately refused with 501: this is an addressing measurement, +not an upload success or a SOS cloud-response fixture. + +The same client's normal `storage mb` additionally reads ACL/CORS/ownership, +download enumerates objects then uses a range downloader, and `storage rb` +lists multipart uploads before deletion. Fixing only region discovery cannot +make its lifecycle work. + +The prefixed door remains proved with AWS CLI and `scw`, **not exo**. +The endpoint exports do not change this client's measured addressing behavior; +they are not certified Exoscale CLI configuration. The addressing decision +must be revisited before that lifecycle can be certified. Bare root requests +must not be guessed into a namespace: different providers/regions can +legitimately contain the same bucket name. No such fallback was added. diff --git a/internal/cli/exo_endpoint_conformance_test.go b/internal/cli/exo_endpoint_conformance_test.go new file mode 100644 index 00000000..59eed5f9 --- /dev/null +++ b/internal/cli/exo_endpoint_conformance_test.go @@ -0,0 +1,95 @@ +//go:build conformance + +package cli + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stephrobert/feint/internal/conformancetest" + "github.com/stephrobert/feint/internal/s3" +) + +func TestExoDropsConfiguredS3EndpointPrefix(t *testing.T) { + var guardHits atomic.Int64 + guard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + guardHits.Add(1) + http.Error(w, "never forwarded", http.StatusBadGateway) + })) + defer guard.Close() + if err := conformancetest.ProbeProxyGuard(t.Context(), guard.URL); err != nil || guardHits.Load() != 1 { + t.Fatal("proxy guard did not observe its positive CONNECT refusal:", err) + } + aws := conformancetest.AWS(t, guard.URL, "") + type exchange struct{ method, path string } + observed := make(chan exchange, 8) + probe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case observed <- exchange{r.Method, r.URL.EscapedPath()}: + default: + t.Error("too many requests for the bounded endpoint measurement") + } + if r.Method == http.MethodHead { + w.Header().Set("X-Amz-Bucket-Region", "ch-gva-2") + w.WriteHeader(http.StatusOK) + return + } + http.Error(w, "measurement only", http.StatusNotImplemented) + })) + defer probe.Close() + srv, _, err := newServer(nil) + if err != nil { + t.Fatal(err) + } + for _, p := range srv.Packs() { + if p.Name() != "exoscale" { + continue + } + pack, ok := p.(packEnv) + if !ok { + t.Fatal("pack environment missing") + } + vars := pack.Env(probe.URL).Vars + vars["EXOSCALE_SOS_ENDPOINT"] = probe.URL + s3.DoorPath("exoscale", "ch-gva-2") + vars["EXOSCALE_STORAGE_API_ENDPOINT"] = vars["EXOSCALE_SOS_ENDPOINT"] + exo := conformancetest.EXO(t, aws, vars) + file := filepath.Join(aws.Home, "input") + if err := os.WriteFile(file, []byte("fake measurement"), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, exo.Path, "storage", "upload", file, "sos://feint-bucket/exo-byte-proof") + cmd.Env = exo.Env + out, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "S3: PutObject") || !strings.Contains(string(out), "501") { + t.Fatalf("measurement did not reach its explicit PUT refusal: %v %q", err, out) + } + want := []exchange{{"HEAD", "/feint-bucket"}, {"PUT", "/feint-bucket/exo-byte-proof"}} + for _, expected := range want { + select { + case got := <-observed: + if got != expected { + t.Fatalf("pinned client's endpoint behavior changed: got=%+v want=%+v", got, expected) + } + t.Logf("measured method=%s path=%s; configured prefix absent", got.method, got.path) + default: + t.Fatal("missing measured request:", expected) + } + } + if len(observed) != 0 || guardHits.Load() != 1 { + t.Fatalf("unexpected requests: local=%d refused external delta=%d", len(observed), guardHits.Load()-1) + } + t.Log("intentional measurement-only PUT refusal; no object lifecycle is certified") + return + } + t.Fatal("Exoscale pack absent") +} diff --git a/internal/conformancetest/exo.go b/internal/conformancetest/exo.go new file mode 100644 index 00000000..0ca87820 --- /dev/null +++ b/internal/conformancetest/exo.go @@ -0,0 +1,31 @@ +//go:build conformance + +package conformancetest + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// EXO uses only the witness HOME, refusing proxy and the pack's public fake +// environment. No account configuration from the host is read. +func EXO(t *testing.T, base Client, vars map[string]string) Client { + t.Helper() + binary, err := exec.LookPath("exo") + if err != nil { + t.Fatal("install Exoscale CLI 1.95.1 for this explicit byte proof:", err) + } + client := Client{Path: binary, Home: base.Home, Env: append([]string{}, base.Env...)} + client.Env = append(client.Env, "PATH="+filepath.Dir(binary)+":/usr/bin:/bin") + for key, value := range vars { + client.Env = append(client.Env, key+"="+value) + } + version := string(client.Run(t, "version")) + if !strings.HasPrefix(version, "exo 1.95.1 7aaf76d0 ") { + t.Fatal("proof requires the measured Exoscale CLI 1.95.1, got:", version) + } + t.Log("Exoscale CLI 1.95.1; isolated pack-owned fake credentials") + return client +} diff --git a/mise.toml b/mise.toml index b01b7ad4..f830fba5 100644 --- a/mise.toml +++ b/mise.toml @@ -58,6 +58,10 @@ run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBy description = "Prove prefixed bytes and private Scaleway buckets through real start/stop/restart with AWS CLI 2.32.17 and scw 2.62.0" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" +[tasks."conformance:s3:exo:endpoint"] +description = "Measure exo 1.95.1 dropping configured S3 URL prefixes; not a successful byte lifecycle" +run = "go test -race -tags=conformance ./internal/cli -run '^TestExoDropsConfiguredS3EndpointPrefix$' -count=1 -v" + [tasks.fuzz] description = "Fuzz the request decoders. Longer run: FUZZTIME=5m mise run fuzz" run = "go test ./internal/providers/scaleway/ -run '^$' -fuzz 'FuzzCreateServer' -fuzztime=\"${FUZZTIME:-30s}\"" diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 4be64841..32ccac40 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -78,6 +78,12 @@ type rule struct { const prepushIsTheWholeGate = "" var rules = []rule{ + { + Path: "internal/cli/exo_endpoint_conformance_test.go", + Why: "pinned Exoscale S3 addressing measurement, not a successful object lifecycle", + Runs: []string{"conformance:s3:exo:endpoint"}, + Unproven: "Exoscale byte lifecycles, mounted S3 host dispatch, multipart, native bucket usage and interception", + }, // ---------------------------------------------------------------- packs { Path: "internal/providers/scaleway/object_storage.go", From 09159515de72a2966d051dad31c8745841adb094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 10:42:38 +0100 Subject: [PATCH 07/28] feat(s3): route Exo uploads through explicit same-port hosts Add opt-in synthetic provider/region HTTP proxy authorities without DNS, TLS, forwarding or another listener. Preserve prefixed AWS/scw addressing and select the Exoscale host through the pack's client declaration.\n\nProve real exo uploads, AWS byte/checksum retrieval and actual restart; retain the negative prefix measurement, browser/peer guards, quotas and control-plane routing. Version the S3 report at 2 and CLI surface at 27. Complete Exoscale lifecycles remain deferred. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 12 + CHANGELOG.md | 12 + docs/confidence.md | 2 +- docs/limits.md | 6 + docs/object-storage-routing.md | 76 +++++- internal/cli/cli.go | 9 +- internal/cli/doctor.go | 5 + internal/cli/env.go | 8 + internal/cli/object_storage.go | 35 ++- .../cli/object_storage_conformance_test.go | 112 ++++++++- internal/cli/object_storage_test.go | 88 ++++++- internal/cli/status.go | 3 + internal/cli/testdata/frozen/cli.json | 232 ++++++++++++++++++ internal/cli/testdata/frozen/s3.json | 59 +++++ internal/providers/exoscale/object_storage.go | 2 +- internal/s3/doors.go | 102 ++++++-- internal/s3/host_doors_test.go | 157 ++++++++++++ internal/s3/server.go | 5 +- mise.toml | 2 +- .../falsify/specs/mounted-object-storage.json | 8 +- .../specs/s3-bucket-configuration.json | 4 +- tools/falsify/specs/s3-gateway.json | 4 +- tools/falsify/specs/s3-host-doors.json | 95 +++++++ tools/testplan/rules.go | 4 +- 24 files changed, 980 insertions(+), 62 deletions(-) create mode 100644 internal/s3/host_doors_test.go create mode 100644 tools/falsify/specs/s3-host-doors.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index b72894ee..bbf00cb7 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,18 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **Des hôtes S3 synthétiques monoport, prouvés par les uploads Exo** (#850) : + `serve/start --s3-host-doors`, opt-in, conserve les préfixes et sert les + autorités HTTP provider/région par un proxy local qui ne transfère rien. + Exo 1.95.1 téléverse un binaire ; AWS vérifie octets/SHA-256 avant et après + redémarrage réel. Routes natives, gardes navigateur/peer, quotas et espaces + restent intacts. `env exoscale --s3` remplace les proxies d'un shell dédié ; + `env --s3 --unset` les retire sans restaurer les anciennes valeurs. + Statut/doctor décrivent les hôtes. Le rapport S3 passe à **2**, la surface + CLI à **27**. Cycles Exoscale complets, virtual hosts de buckets cloud et + interception TLS/Terraform restent hors périmètre. + Scaleway conserve son environnement préfixé mesuré. + - **L'incompatibilité Exoscale avec les préfixes S3, mesurée** (#850) : `conformance:s3:exo:endpoint` prouve qu'exo 1.95.1 retire le préfixe configuré de la découverte du bucket comme de l'upload. Ce listener refuse diff --git a/CHANGELOG.md b/CHANGELOG.md index e46999ce..a8d401b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,18 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Same-port synthetic S3 hosts, proved with Exo uploads** (#850): + opt-in `serve/start --s3-host-doors` retains the prefix doors and serves + declared provider/region HTTP authorities through a local non-forwarding + proxy. Exo 1.95.1 uploads binary bytes; AWS verifies bytes/SHA-256 before + and after actual restart. Native routes, browser/peer guards, quotas and + namespace separation remain intact. Exoscale `env --s3` sets dedicated-shell + proxy variables; explicit `env --s3 --unset` removes them without restoring + prior values. Scaleway retains its measured prefix environment. + Status/doctor describe the hosts. S3 report schema is now **2** and the + CLI surface **27**. Full Exoscale workflows, cloud bucket virtual hosts + and TLS/Terraform interception remain unsupported. + - **Measured Exoscale S3 prefix incompatibility** (#850): `conformance:s3:exo:endpoint` proves that exo 1.95.1 removes the configured URL prefix from both bucket discovery and object upload. This listener diff --git a/docs/confidence.md b/docs/confidence.md index 7c3ff52e..2b193926 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -44,7 +44,7 @@ prints the reason under the pack that owns it. | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17 and scw 2.62.0 | `mise run conformance:s3:mounted` drives bytes and private non-versioned Scaleway buckets before and after actual process restart; complete Exoscale lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#measured-scaleway-bucket-workflow) | +| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets and host-routed Exo uploads with byte/checksum proof across actual restart; complete Exoscale lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-upload-through-a-synthetic-host) | | Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | diff --git a/docs/limits.md b/docs/limits.md index d43a122b..53eec117 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -11,6 +11,12 @@ declared `/_feint/s3///` endpoints, cumulative byte budgets and an AWS CLI 2.32.17 / scw 2.62.0 restart proof. The Scaleway private, non-versioned bucket `create/list/get/delete` workflow is served. See the [door ADR and commands](object-storage-routing.md). +Opt-in `--s3-host-doors` also permits exo 1.95.1 simple uploads to an existing +bucket through a synthetic scoped authority and the same-port non-forwarding +HTTP proxy. AWS retrieval proves the uploaded binary bytes and SHA-256 survive +actual restart. This does not implement bucket virtual hosts, DNS/TLS +interception or a complete Exoscale lifecycle; use a dedicated shell because +its S3 environment replaces proxy settings. Complete Exoscale workflows, object listing, writable tags/public ACLs, version histories, multipart, ETags, checkpoint/reset and hardcoded Terraform redirection remain unsupported. diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 528b202e..41857682 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -31,6 +31,12 @@ Unknown doors are refused, not treated as a default provider or region. The outer Host/Origin browser guard stays in place. S3 remains loopback-only; network exposure is not implied by the control-plane exposure option. +**Amended 2026-11-07:** the measured Exoscale client discards endpoint paths. +An additional opt-in `--s3-host-doors` declares synthetic provider/region +authorities, served through absolute-form HTTP proxy requests on the same +listener. Prefix doors remain available. This advances local host dispatch, +not TLS interception or hardcoded cloud-host routing. + Use one private neutral ObjectStore. Provider/region separation is a declared gateway namespace, not a provider name embedded in `internal/core`, and never a bucket/key-derived filesystem path. Provider-specific XML belongs at the @@ -86,10 +92,9 @@ The configured-client door must be proved against clients that actually support the chosen endpoint prefix. If a client rejects it, revisit this decision rather than quietly add a second port or change its workload. -Virtual hosts, trusted intercepted-host handoff, hardcoded Scaleway Terraform -endpoints, checkpoint/reset, garbage collection and public-stack certification -remain later work. Existing Object Storage support limits stay unchanged until -the mounted-client proofs exist. +Cloud virtual hosts, TLS interception, hardcoded Scaleway Terraform endpoints, +checkpoint/reset, garbage collection and public-stack certification remain +later work. The synthetic HTTP host slice is described below. ## Implemented mounted slice @@ -125,8 +130,8 @@ control-plane-only. `env --unset` also removes S3 endpoint variables, without requiring a running instance. All three shell renderers use this same selection. These are provider CLI variables, not an AWS CLI credential configuration. -The separate `GET /_feint/s3` report (schema version 1) publishes actual -activation, declared paths, the store's configured blob-byte limit, the +The separate `GET /_feint/s3` report (now schema version 2, previously 1) publishes +actual activation, declared paths and optional synthetic hosts, the store's configured blob-byte limit, the per-object limit and completed HTTP-response counts per declared door/status. It publishes no credentials, keys, payloads or storage paths. These counts are **not official-client proof** and do not enter control-plane SDK coverage, @@ -238,8 +243,61 @@ lists multipart uploads before deletion. Fixing only region discovery cannot make its lifecycle work. The prefixed door remains proved with AWS CLI and `scw`, **not exo**. -The endpoint exports do not change this client's measured addressing behavior; -they are not certified Exoscale CLI configuration. The addressing decision -must be revisited before that lifecycle can be certified. Bare root requests +Prefix-only endpoint exports do not change this client's measured addressing +behavior; they are not certified Exoscale CLI configuration. This measurement +prompted the host-door amendment above. Bare root requests must not be guessed into a namespace: different providers/regions can legitimately contain the same bucket name. No such fallback was added. + +## Measured Exoscale upload through a synthetic host + +Start with `--s3-host-doors` in addition to `--objectstore`. Each declared scope +then has an authority such as `exoscale.ch-gva-2.s3.feint.invalid`. This is +**path-style S3 at a scoped authority**, not bucket virtual-host addressing. +The `.invalid` name is not resolved: the client sends an absolute HTTP URL to +the emulator as its HTTP proxy, on the existing port. + +```bash +feint start --objectstore "$PWD/object-storage/data" --s3-host-doors +# Use a dedicated shell with fake credentials, not an operator account. +eval "$(feint env exoscale --s3)" +exo storage upload input.bin sos://existing-bucket/key +eval "$(feint env exoscale --s3 --unset)" +``` + +The bucket must already exist. Normal Exoscale bucket creation, download and +cleanup workflows are **not** certified by this upload proof. + +For Exoscale, `env --s3` selects the live declared authority and exports upper/lower-case +HTTP/HTTPS/ALL proxy variables pointing at the literal loopback endpoint, with +NO_PROXY restricted to loopback control-plane names. This replaces the dedicated +shell's previous proxy settings. Apply `env --s3 --unset` to remove these exports; +it does not restore earlier values. Ordinary `env --unset` retains its previous +behavior and does not clear unrelated proxy variables. Discovery refuses +foreign or mismatched reported hosts instead of exporting them. +The pack declares whether its client needs host addressing: Scaleway keeps +its measured prefix endpoint and does not gain shell proxy exports when host +doors are enabled. The mounted witness also exercises this exported Scaleway +configuration, so enabling Exoscale does not silently change the other client. + +Only exact declared authorities with matching absolute URL/Host and scheme +`http` are served. Undeclared targets, CONNECT, URL credentials, aliases and +authority mismatches are refused; nothing is forwarded. Host S3 cannot reach +health, state, reports or native routes. Prefix/control-plane requests retain +the rebinding guard; host S3 independently rejects every browser Origin and any +non-loopback peer, even under `--expose`. No host-header rewriting, global DNS, +certificate installation, extra listener or external S3 service is used. + +The existing `conformance:s3:mounted` witness now also runs **exo 1.95.1** +`storage upload`, using the exact exported environment in its private HOME. +It proves actual HEAD/PUT host exchanges, binary identity and SHA-256 by AWS +retrieval through the corresponding prefix, persistence after real restart, +another Exo upload after restart, and a different same-name object in the other +provider. Unit tests additionally hold region separation, literal encoded keys, +browser/peer refusals and read deadlines. The declared guards are falsified in +`tools/falsify/specs/s3-host-doors.json`. + +The witness uses pack-owned public fake credentials. Its refusing proxy is not +a network namespace and does not constrain unrelated clients that ignore proxy +settings. Host response counters are separate from prefix counters and start +again at zero on restart; neither is official-client or SDK-coverage evidence. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 7283d33e..9483c544 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -243,12 +243,13 @@ const ( // // Version 26 adds opt-in Object Storage flags to serve/start and live S3 // discovery to env/doctor (#850); existing flags and exit codes stay unchanged. +// Version 27 adds --s3-host-doors to serve/start, retaining prefix addressing. // // The surface itself is frozen in testdata/frozen/cli.json, compared by // TestTheFrozenSurfacesStillMatchTheirFixture, and a fixture regenerated // without bumping this constant fails TestASurfaceChangeDemandsItsVersionBump. // The procedure for a deliberate change is in RELEASING.md ("Frozen surfaces"). -const cliSurfaceVersion = 26 +const cliSurfaceVersion = 27 // Run executes one command and returns the process exit code. func Run(args []string, stdout, stderr io.Writer) int { @@ -350,7 +351,7 @@ Usage: [--projects [,...]] [--consistency immediate|eventual] [--strict-catalog ] [--objectstore ] [--s3-doors ] - [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] + [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] [--s3-host-doors] Serve the three emulated clouds on one port, in the foreground. --expose-to-network is the only way off loopback, and it disarms the anti-rebinding guard: this @@ -390,7 +391,7 @@ Usage: [--projects [,...]] [--strict-catalog ] [--timeout 30s] [--detach] [--foreground] [--objectstore ] [--s3-doors ] - [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] + [--objectstore-budget-mib 1024] [--s3-object-limit-mib 256] [--s3-host-doors] Same, detached: records the instance, waits until it answers, prints where the log is. Refuses to adopt an instance already running on that address. The serve flags @@ -1184,7 +1185,7 @@ func serve(args []string, stdout io.Writer) (result error) { // Wrapped here rather than in the server: this is where the listen // address is known, and the guard's whole job is to compare what the // caller asked for against what this process is bound to. - Handler: emulator.GuardRebinding(doors, *addr), + Handler: doors.Handler(*addr), ReadHeaderTimeout: 10 * time.Second, WriteTimeout: writeTimeoutFor(rt), } diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index fb52e1b3..1eff87fe 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -112,6 +112,11 @@ func checkObjectStorage(addr string) check { c.state = verdictOK c.detail = fmt.Sprintf("%d declared door(s); %d blob bytes maximum; %d bytes per simple PUT; ancillary client lifecycles are not certified", len(report.Doors), report.MaxBlobBytes, report.MaxObjectBytes) + for _, door := range report.Doors { + if door.Host != "" { + c.detail += "; host " + door.Host + " requires the local non-forwarding HTTP proxy" + } + } } return c } diff --git a/internal/cli/env.go b/internal/cli/env.go index c41eb9de..7fc12c62 100644 --- a/internal/cli/env.go +++ b/internal/cli/env.go @@ -123,8 +123,16 @@ func envCommand(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "feint: %v\n", err) return exitError } + if storage, ok := pack.(packObjectStorage); *unset && *withS3 && ok && storage.ObjectStorage(*s3Region).HostEndpoint { + for _, key := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { + env.Vars[key] = "" + } + } if !*unset { fmt.Fprintln(stderr, "note: S3 supports simple bytes and private non-versioned Scaleway buckets; Exoscale ancillary workflows and hardcoded Terraform endpoints are not certified.") + if env.Vars["HTTP_PROXY"] != "" { + fmt.Fprintln(stderr, "note: S3 host doors replace this shell's proxy settings with the local non-forwarding emulator. Use a dedicated shell; env --s3 --unset removes them, it does not restore previous values.") + } } } diff --git a/internal/cli/object_storage.go b/internal/cli/object_storage.go index b518790e..e5799fa6 100644 --- a/internal/cli/object_storage.go +++ b/internal/cli/object_storage.go @@ -27,18 +27,23 @@ type objectStorageFlags struct { doors string budgetMiB int64 objectMiB int64 + hosts bool } func bindObjectStorageFlags(fs *flag.FlagSet) *objectStorageFlags { f := &objectStorageFlags{} fs.StringVar(&f.root, "objectstore", "", "opt in to persistent S3 bytes in this private directory (independent of --state)") fs.StringVar(&f.doors, "s3-doors", defaultS3Doors, "comma-separated provider/region S3 doors; only these prefixes are served") + fs.BoolVar(&f.hosts, "s3-host-doors", false, "also accept declared synthetic S3 hosts through local HTTP proxy requests on this same port") fs.Int64Var(&f.budgetMiB, "objectstore-budget-mib", 1024, "cumulative retained and reserved blob budget in MiB; deletion does not free blobs") fs.Int64Var(&f.objectMiB, "s3-object-limit-mib", 256, "maximum decoded simple PUT size in MiB") return f } func (f *objectStorageFlags) validate() error { + if f.hosts && f.root == "" { + return errors.New("--s3-host-doors requires --objectstore") + } if f.budgetMiB <= 0 || f.budgetMiB > math.MaxInt64>>20 || f.objectMiB <= 0 || f.objectMiB > (math.MaxInt64-(64<<10))/2>>20 || f.objectMiB > f.budgetMiB { @@ -58,8 +63,12 @@ func (f *objectStorageFlags) args() []string { if f.root == "" { return nil } - return []string{"--objectstore", f.root, "--s3-doors", f.doors, + args := []string{"--objectstore", f.root, "--s3-doors", f.doors, "--objectstore-budget-mib", fmt.Sprint(f.budgetMiB), "--s3-object-limit-mib", fmt.Sprint(f.objectMiB)} + if f.hosts { + args = append(args, "--s3-host-doors") + } + return args } type packObjectStorage interface { @@ -105,7 +114,11 @@ func mountObjectStorage(ctx context.Context, srv *emulator.Server, flags *object if err != nil { return nil, nil, fmt.Errorf("open object storage: %w", err) } - doors, err := s3.Mount(srv.Handler(), store, profiles, flags.objectMiB<<20, log) + mount := s3.Mount + if flags.hosts { + mount = s3.MountHosts + } + doors, err := mount(srv.Handler(), store, profiles, flags.objectMiB<<20, log) if err != nil { return nil, nil, errors.Join(err, store.Close()) } @@ -186,7 +199,8 @@ func fetchS3(endpoint string) (report s3.Report, result error) { seen := make(map[string]bool) for _, door := range report.Doors { if !s3.ValidToken(door.Provider) || !s3.ValidToken(door.Region) || - door.Path != s3.DoorPath(door.Provider, door.Region) || seen[door.Path] { + door.Path != s3.DoorPath(door.Provider, door.Region) || seen[door.Path] || + (door.Host != "" && door.Host != s3.DoorHost(door.Provider, door.Region)) { return report, errors.New("S3 report declares a noncanonical or duplicate endpoint") } seen[door.Path] = true @@ -227,8 +241,21 @@ func storageEnvironment(pack packEnv, endpoint, region string, unset bool, vars return fmt.Errorf("no enabled S3 door for provider %q and selected region %q", pack.Name(), region) } profile := storage.ObjectStorage(selected.Region) + target := strings.TrimSuffix(endpoint, "/") + strings.TrimSuffix(selected.Path, "/") + if profile.HostEndpoint && selected.Host != "" { + if !strings.HasPrefix(endpoint, "http://") { + return errors.New("S3 host doors require a local HTTP proxy endpoint, not HTTPS") + } + target = "http://" + selected.Host + for _, key := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"} { + vars[key] = strings.TrimSuffix(endpoint, "/") + } + for _, key := range []string{"NO_PROXY", "no_proxy"} { + vars[key] = "127.0.0.1,localhost,::1" + } + } for _, key := range profile.Variables { - vars[key] = strings.TrimSuffix(endpoint, "/") + strings.TrimSuffix(selected.Path, "/") + vars[key] = target } vars[profile.RegionVariable] = selected.Region return nil diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index 1b0bbbfe..9b53ebe6 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -12,6 +12,7 @@ import ( "net" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" @@ -79,7 +80,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } }) run("start", "--addr", addr, "--vm", "off", "--objectstore", "bytes", - "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1") + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1", "--s3-host-doors") var first statusReport if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &first); err != nil || !first.Running || first.PID <= 0 || first.ObjectStorage == nil || @@ -89,12 +90,33 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if len(first.ObjectStorage.Doors) != 2 { t.Fatal("actual serve did not mount both declared doors") } + localProxy, err := url.Parse(endpoint) + if err != nil { + t.Fatal(err) + } + proxyClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(localProxy)}, Timeout: 5 * time.Second} + defer proxyClient.CloseIdleConnections() + control, err := http.NewRequestWithContext(t.Context(), "GET", "http://must-not-forward.invalid/_feint/health", nil) + if err != nil { + t.Fatal(err) + } + refused, err := proxyClient.Do(control) + if err != nil { + t.Fatal("actual emulator proxy control failed:", err) + } + if err := refused.Body.Close(); err != nil || refused.StatusCode != http.StatusForbidden { + t.Fatal("actual emulator did not refuse an undeclared proxy authority:", refused.StatusCode, err) + } for _, provider := range []string{"scaleway", "exoscale"} { out := run("env", provider, "--s3", "--endpoint", endpoint) found := false for _, door := range first.ObjectStorage.Doors { - if door.Provider == provider && bytes.Contains(out, []byte(endpoint+strings.TrimSuffix(door.Path, "/"))) && - !bytes.Contains(out, []byte(endpoint+door.Path)) { + target := "http://" + door.Host + if provider == "scaleway" { + target = endpoint + strings.TrimSuffix(door.Path, "/") + } + if door.Provider == provider && door.Host == s3.DoorHost(door.Provider, door.Region) && + bytes.Contains(out, []byte(target)) && bytes.Contains(out, []byte(endpoint)) { found = true } } @@ -133,9 +155,9 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("actual persisted versioning status was lost or invented: %v %q", err, out) } } - get := func(door s3.Door, expected []byte) { + getKey := func(door s3.Door, objectKey string, expected []byte) { t.Helper() - result := call(door, "get-object", "--bucket", bucket, "--key", key, "--checksum-mode", "ENABLED", output) + result := call(door, "get-object", "--bucket", bucket, "--key", objectKey, "--checksum-mode", "ENABLED", output) got, err := os.ReadFile(output) var meta struct{ ChecksumSHA256 string } hash := sha256.Sum256(expected) @@ -144,6 +166,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("official client did not return identical verified bytes: read=%v result=%q", err, result) } } + get := func(door s3.Door, expected []byte) { getKey(door, key, expected) } for i, door := range first.ObjectStorage.Doors { if door.Provider == "scaleway" { bucketCall("create", true) @@ -160,6 +183,62 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } get(first.ObjectStorage.Doors[0], payload) get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + // Use each client's exact exported environment, without evaluating shell text. + var exoVars map[string]string + hostSCW := scw + hostSCW.Env = append([]string{}, scw.Env...) + server, _, err := newServer(nil) + if err != nil { + t.Fatal(err) + } + for _, p := range server.Packs() { + if p.Name() == "scaleway" { + pack, ok := p.(packEnv) + if !ok { + t.Fatal("Scaleway pack environment missing") + } + vars := pack.Env(endpoint).Vars + if err := storageEnvironment(pack, endpoint, "fr-par", false, vars); err != nil { + t.Fatal(err) + } + for key, value := range vars { + hostSCW.Env = append(hostSCW.Env, key+"="+value) + } + } + if p.Name() == "exoscale" { + pack, ok := p.(packEnv) + if !ok { + t.Fatal("Exoscale pack environment missing") + } + exoVars = pack.Env(endpoint).Vars + if err := storageEnvironment(pack, endpoint, "ch-gva-2", false, exoVars); err != nil { + t.Fatal(err) + } + } + } + if exoVars == nil { + t.Fatal("Exoscale pack missing") + } + hostSCW.Run(t, "-o", "json", "object", "bucket", "create", "feint-host-scw", "region=fr-par") + hostSCW.Run(t, "-o", "json", "object", "bucket", "get", "feint-host-scw", "region=fr-par") + if out := hostSCW.Run(t, "-o", "json", "object", "bucket", "list", "region=fr-par"); !bytes.Contains(out, []byte("feint-host-scw")) { + t.Fatal("exported SCW inventory omitted its actual created bucket:", string(out)) + } + hostSCW.Run(t, "-o", "json", "object", "bucket", "delete", "feint-host-scw", "region=fr-par") + exo := conformancetest.EXO(t, aws, exoVars) + if err := os.WriteFile(input, payload, 0600); err != nil { + t.Fatal(err) + } + exoKey := "host-byte-proof" + exo.Run(t, "storage", "upload", input, "sos://"+bucket+"/"+exoKey) + getKey(first.ObjectStorage.Doors[1], exoKey, payload) + // A same-name object in the other provider is a positive isolation control. + if err := os.WriteFile(input, []byte("other-provider-host-key"), 0600); err != nil { + t.Fatal(err) + } + call(first.ObjectStorage.Doors[0], "put-object", "--bucket", bucket, "--key", exoKey, "--body", input, + "--checksum-algorithm", "CRC32") + getKey(first.ObjectStorage.Doors[0], exoKey, []byte("other-provider-host-key")) beforeBucket := bucketCall("get", true) checkVersioning() beforeList := bucketCall("list", false) @@ -177,9 +256,26 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } get(first.ObjectStorage.Doors[0], payload) get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) + getKey(first.ObjectStorage.Doors[1], exoKey, payload) + getKey(first.ObjectStorage.Doors[0], exoKey, []byte("other-provider-host-key")) + if restarted.ObjectStorage.Doors[1].Host != first.ObjectStorage.Doors[1].Host || + restarted.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 0 { + t.Fatal("restart lost host configuration or invented persisted HTTP evidence") + } + exo.Run(t, "storage", "upload", input, "sos://"+bucket+"/after-restart") + getKey(first.ObjectStorage.Doors[1], "after-restart", []byte("other-provider-host-key")) + var live statusReport + if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &live); err != nil || + live.ObjectStorage == nil || live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 2 { + t.Fatalf("actual Exo HEAD/PUT after restart did not traverse its declared host: %+v %v", live.ObjectStorage, err) + } + run("doctor", "--addr", addr, "--s3", "--vm", "off") if after := bucketCall("get", true); !bytes.Equal(after, beforeBucket) { t.Fatalf("Scaleway bucket configuration changed across restart: before=%q after=%q", beforeBucket, after) } + if after := hostSCW.Run(t, "-o", "json", "object", "bucket", "get", bucket, "region=fr-par"); !bytes.Equal(after, beforeBucket) { + t.Fatalf("exported SCW read diverged after restart: before=%q after=%q", beforeBucket, after) + } checkVersioning() if after := bucketCall("list", false); !bytes.Equal(after, beforeList) { t.Fatalf("Scaleway bucket identity/creation time changed across restart: before=%q after=%q", beforeList, after) @@ -201,10 +297,14 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatal(err) } run("start", "--addr", addr, "--vm", "off", "--objectstore", root, - "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1") + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1", "--s3-host-doors") for _, door := range first.ObjectStorage.Doors { call(door, "head-object", "--bucket", bucket, "--key", key) call(door, "delete-object", "--bucket", bucket, "--key", key) + call(door, "delete-object", "--bucket", bucket, "--key", exoKey) + if door.Provider == "exoscale" { + call(door, "delete-object", "--bucket", bucket, "--key", "after-restart") + } if door.Provider == "scaleway" { bucketCall("delete", true) } else { diff --git a/internal/cli/object_storage_test.go b/internal/cli/object_storage_test.go index a8e0f843..70ec15f6 100644 --- a/internal/cli/object_storage_test.go +++ b/internal/cli/object_storage_test.go @@ -26,7 +26,7 @@ func TestStartPreservesObjectStorageFlagsAndAbsoluteRoot(t *testing.T) { fs := newFlagSet("start") flags := bindServeFlags(fs) if err := fs.Parse([]string{"--objectstore", "relative-bytes", "--s3-doors", "scaleway/fr-par", - "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1"}); err != nil { + "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1", "--s3-host-doors"}); err != nil { t.Fatal(err) } if err := flags.storage.validate(); err != nil { @@ -60,6 +60,10 @@ func TestObjectStorageValidatesDefaultsAndLimitsWithoutStartingAnything(t *testi if len(defaults.args()) != 0 { t.Fatal("disabled byte storage rendered activation flags") } + defaults.hosts = true + if err := defaults.validate(); err == nil { + t.Fatal("host doors silently activated without byte storage") + } } func TestS3ListenConfigurationAlwaysProducesUsableLoopbackEndpoints(t *testing.T) { @@ -221,6 +225,8 @@ func TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints(t *testing func(r *s3.Report) { r.MaxObjectBytes = 4 }, func(r *s3.Report) { r.Enabled = false }, func(r *s3.Report) { r.Doors[0].Path = "/foreign" }, + func(r *s3.Report) { r.Doors[0].Host = "cloud.example" }, + func(r *s3.Report) { r.Doors[0].Host = s3.DoorHost("scaleway", "nl-ams") }, func(r *s3.Report) { r.Doors = append(r.Doors, r.Doors[0]) }, } { report := valid @@ -236,3 +242,83 @@ func TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints(t *testing server.Close() } } + +func TestS3HostEnvironmentUsesOnlyDeclaredAuthorityAndLocalProxy(t *testing.T) { + srv, env, err := newServer(nil) + if err != nil { + t.Fatal(err) + } + flags := &objectStorageFlags{root: filepath.Join(privateStorageDir(t), "bytes"), + doors: defaultS3Doors, budgetMiB: 3, objectMiB: 1, hosts: true} + doors, store, err := mountObjectStorage(t.Context(), srv, flags, env.Log) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }() + ts := httptest.NewServer(doors.Handler("127.0.0.1:4599")) + defer ts.Close() + for _, p := range srv.Packs() { + if p.Name() != "exoscale" { + continue + } + pack, ok := p.(packEnv) + if !ok { + t.Fatal("Exoscale environment missing") + } + vars := pack.Env(ts.URL).Vars + if err := storageEnvironment(pack, ts.URL, "ch-gva-2", false, vars); err != nil { + t.Fatal(err) + } + for _, key := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"} { + if vars[key] != ts.URL { + t.Fatalf("%s did not select the literal local proxy: %q", key, vars[key]) + } + } + for _, key := range []string{"NO_PROXY", "no_proxy"} { + if vars[key] != "127.0.0.1,localhost,::1" { + t.Fatalf("%s could bypass proxy interception: %q", key, vars[key]) + } + } + } + for _, shell := range []string{"bash", "fish", "powershell"} { + var out, note strings.Builder + code := envCommand([]string{"exoscale", "--s3", "--endpoint", ts.URL, "--shell", shell}, &out, ¬e) + if code != 0 || !strings.Contains(out.String(), "http://"+s3.DoorHost("exoscale", "ch-gva-2")) || + !strings.Contains(out.String(), "HTTP_PROXY") || !strings.Contains(out.String(), "http_proxy") || + !strings.Contains(out.String(), ts.URL) || !strings.Contains(out.String(), "127.0.0.1,localhost,::1") || + !strings.Contains(note.String(), "proxy settings") { + t.Fatalf("host endpoint/proxy environment missing: %d %q %q", code, out.String(), note.String()) + } + } + report, err := fetchS3(ts.URL) + if err != nil || report.Doors[1].Host != s3.DoorHost("exoscale", "ch-gva-2") { + t.Fatal("live discovery lost the declared host:", report, err) + } + var scaleway, scalewayNote strings.Builder + if code := envCommand([]string{"scaleway", "--s3", "--endpoint", ts.URL}, &scaleway, &scalewayNote); code != 0 || + !strings.Contains(scaleway.String(), ts.URL+strings.TrimSuffix(s3.DoorPath("scaleway", "fr-par"), "/")) || + strings.Contains(scaleway.String(), "HTTP_PROXY") { + t.Fatal("host activation broke the measured Scaleway prefix environment:", code, scaleway.String()) + } + if c := checkObjectStorage(strings.TrimPrefix(ts.URL, "http://")); c.state != verdictOK || + !strings.Contains(c.detail, report.Doors[1].Host) { + t.Fatal("doctor did not describe the required local proxy:", c) + } + var out, note strings.Builder + if code := envCommand([]string{"exoscale", "--s3", "--unset"}, &out, ¬e); code != 0 || + !strings.Contains(out.String(), "unset HTTP_PROXY") || !strings.Contains(out.String(), "unset no_proxy") { + t.Fatal("explicit S3 unset did not remove local proxy settings:", code, out.String()) + } + out.Reset() + if code := envCommand([]string{"exoscale", "--unset"}, &out, ¬e); code != 0 || strings.Contains(out.String(), "HTTP_PROXY") { + t.Fatal("ordinary unset changed unrelated proxy settings:", code, out.String()) + } + out.Reset() + if code := envCommand([]string{"scaleway", "--s3", "--unset"}, &out, ¬e); code != 0 || strings.Contains(out.String(), "HTTP_PROXY") { + t.Fatal("prefix-only Scaleway unset changed unrelated proxy settings:", code, out.String()) + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 220abad4..a12b1863 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -183,6 +183,9 @@ func writeStatusText(w io.Writer, r statusReport, health healthResponse) { len(r.ObjectStorage.Doors), r.ObjectStorage.MaxBlobBytes, r.ObjectStorage.MaxObjectBytes) for _, door := range r.ObjectStorage.Doors { fmt.Fprintf(w, " %s\n", door.Path) + if door.Host != "" { + fmt.Fprintf(w, " http://%s (local HTTP proxy, never forwarded)\n", door.Host) + } } } } diff --git a/internal/cli/testdata/frozen/cli.json b/internal/cli/testdata/frozen/cli.json index 5782d1a3..ae60d5df 100644 --- a/internal/cli/testdata/frozen/cli.json +++ b/internal/cli/testdata/frozen/cli.json @@ -4687,6 +4687,238 @@ ] } } + }, + { + "schema_version": 27, + "content": { + "exit_codes": { + "drift": 2, + "error": 1, + "ok": 0 + }, + "verbs": { + "catalog": [ + "--format" + ], + "clean": [ + "--check", + "--closing", + "--doorstep", + "--force", + "--format", + "--vm" + ], + "corpus": [ + "--accepted", + "--against-cloud", + "--bind", + "--check", + "--credential", + "--dir", + "--dry-run", + "--endpoint", + "--file", + "--format", + "--mark-stale", + "--timeout" + ], + "coverage": [ + "--artefact", + "--axis", + "--baseline", + "--contract", + "--contract-only", + "--document", + "--evidence", + "--fail-on-unknown", + "--format", + "--gaps", + "--observed", + "--products", + "--provider", + "--sdk", + "--write-baseline" + ], + "docs": [ + "--check", + "--client-pins", + "--confidence", + "--contracts", + "--coverage", + "--file", + "--install", + "--limits", + "--proved", + "--routes", + "--screenshots", + "--ui-manifest", + "--workflow" + ], + "doctor": [ + "--addr", + "--s3", + "--vm" + ], + "down": [ + "--file", + "--keep-emulator" + ], + "env": [ + "--client", + "--endpoint", + "--s3", + "--s3-region", + "--shell", + "--unset" + ], + "evidence": [ + "--allow-narrowing", + "--contracts", + "--endpoint", + "--join", + "--out", + "--reshape", + "--shapes", + "--suites" + ], + "evidence baseline": [ + "--axes", + "--evidence", + "--out" + ], + "evidence verify": [ + "--accepted", + "--baseline", + "--evidence" + ], + "images": [ + "--check", + "--only", + "--vm" + ], + "logs": [ + "--addr", + "-f", + "-n" + ], + "probe": [ + "--contracts", + "--endpoint", + "--provider" + ], + "proxy": [ + "--addr", + "--expose-to-network", + "--forward", + "--intercept", + "--max-body", + "--provider", + "--queue", + "--record", + "--upstream" + ], + "replay": [ + "--endpoint", + "--format", + "--refusals-only", + "--timeout" + ], + "restart": [ + "--addr", + "--timeout" + ], + "serve": [ + "--addr", + "--cleanup", + "--consistency", + "--contracts", + "--coverage", + "--expose-to-network", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-host-doors", + "--s3-object-limit-mib", + "--shapes", + "--state", + "--strict-catalog", + "--vm" + ], + "shapes": [ + "--check", + "--dir", + "--dry-run", + "--profile", + "--provider", + "--record" + ], + "snapshot list": [ + "--format" + ], + "snapshot load": [ + "--addr" + ], + "snapshot save": [ + "--addr", + "--force" + ], + "start": [ + "--addr", + "--cleanup", + "--contracts", + "--detach", + "--foreground", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-host-doors", + "--s3-object-limit-mib", + "--state", + "--strict-catalog", + "--timeout", + "--vm" + ], + "status": [ + "--addr", + "--coverage", + "--format" + ], + "stop": [ + "--addr", + "--timeout" + ], + "transcript": [ + "--against", + "--contract", + "--format", + "--sanitise", + "--shape" + ], + "ui": [ + "--addr", + "--print" + ], + "up": [ + "--file", + "--no-iac", + "--runtime", + "--timeout" + ], + "version": [ + "--check" + ], + "wait": [ + "--addr", + "--timeout" + ] + } + } } ] } diff --git a/internal/cli/testdata/frozen/s3.json b/internal/cli/testdata/frozen/s3.json index b78dfc62..fc67dce7 100644 --- a/internal/cli/testdata/frozen/s3.json +++ b/internal/cli/testdata/frozen/s3.json @@ -54,6 +54,65 @@ } ] } + }, + { + "schema_version": 2, + "content": { + "fields": [ + { + "path": "doors", + "type": "array" + }, + { + "path": "doors[]", + "type": "object" + }, + { + "path": "doors[].host", + "type": "string" + }, + { + "path": "doors[].path", + "type": "string" + }, + { + "path": "doors[].provider", + "type": "string" + }, + { + "path": "doors[].region", + "type": "string" + }, + { + "path": "enabled", + "type": "bool" + }, + { + "path": "http_requests", + "type": "object" + }, + { + "path": "http_requests.*", + "type": "array" + }, + { + "path": "http_requests.*[]", + "type": "number" + }, + { + "path": "max_blob_bytes", + "type": "number" + }, + { + "path": "max_object_bytes", + "type": "number" + }, + { + "path": "schema_version", + "type": "number" + } + ] + } } ] } diff --git a/internal/providers/exoscale/object_storage.go b/internal/providers/exoscale/object_storage.go index a6247682..0de6b312 100644 --- a/internal/providers/exoscale/object_storage.go +++ b/internal/providers/exoscale/object_storage.go @@ -5,6 +5,6 @@ import "github.com/stephrobert/feint/internal/s3" // ObjectStorage declares the measured SOS dialect and configurable CLI endpoints. // It does not advertise control-plane bucket usage or multipart support. func (p *Pack) ObjectStorage(region string) s3.Profile { - return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ExoscaleMissingBucket, + return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ExoscaleMissingBucket, HostEndpoint: true, Variables: []string{"EXOSCALE_SOS_ENDPOINT", "EXOSCALE_STORAGE_API_ENDPOINT"}, RegionVariable: "EXOSCALE_ZONE"} } diff --git a/internal/s3/doors.go b/internal/s3/doors.go index 923452df..04a993c7 100644 --- a/internal/s3/doors.go +++ b/internal/s3/doors.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/stephrobert/feint/internal/core/emulator" "github.com/stephrobert/feint/internal/core/objectstore" ) @@ -19,7 +20,7 @@ import ( const ReportPath = "/_feint/s3" // ReportVersion versions the S3 report independently of control-plane schemas. -const ReportVersion = 1 +const ReportVersion = 2 // UploadReadTimeout bounds a simple PUT body without changing the runtime's // independent write deadline or the control-plane request-body policy. @@ -34,13 +35,16 @@ type Profile struct { Variables []string RegionVariable string BucketConfiguration bool + // HostEndpoint selects the host door for clients that discard path prefixes. + HostEndpoint bool } -// Door describes an enabled path-style endpoint, without filesystem paths. +// Door describes an enabled prefix and optional local HTTP proxy authority. type Door struct { Provider string `json:"provider"` Region string `json:"region"` Path string `json:"path"` + Host string `json:"host"` } // Report owns the wire shape consumed by env/status/doctor and conformance. @@ -59,6 +63,12 @@ func DoorPath(provider, region string) string { return ReportPath + "/" + provider + "/" + region + "/" } +// DoorHost is a synthetic authority, never a cloud hostname or a DNS requirement. +// Like DoorPath, it requires validated provider and region tokens. +func DoorHost(provider, region string) string { + return provider + "." + region + ".s3.feint.invalid" +} + // ValidToken limits namespace components to short lower-case ASCII tokens. func ValidToken(value string) bool { if value == "" || len(value) > 32 { @@ -78,6 +88,7 @@ type Doors struct { next http.Handler log *slog.Logger handlers map[string]*Server + hosts map[string]*Server mu sync.Mutex report Report } @@ -85,10 +96,21 @@ type Doors struct { // Mount composes a control-plane handler and explicitly selected S3 doors. // A nil store and no profiles publishes disabled S3 without activating storage. func Mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxObjectBytes int64, log *slog.Logger) (*Doors, error) { + return mount(next, store, profiles, maxObjectBytes, log, false) +} + +// MountHosts additionally declares synthetic authorities for the same scopes. +// Handler must be used to receive local absolute-form HTTP proxy requests. +// No request is forwarded and no DNS or extra listener is involved. +func MountHosts(next http.Handler, store *objectstore.Store, profiles []Profile, maxObjectBytes int64, log *slog.Logger) (*Doors, error) { + return mount(next, store, profiles, maxObjectBytes, log, true) +} + +func mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxObjectBytes int64, log *slog.Logger, hosts bool) (*Doors, error) { if next == nil || log == nil || (store == nil) != (len(profiles) == 0) { return nil, errors.New("s3: handler, logger and consistent explicit storage activation are required") } - d := &Doors{next: next, log: log, handlers: make(map[string]*Server), report: Report{ + d := &Doors{next: next, log: log, handlers: make(map[string]*Server), hosts: make(map[string]*Server), report: Report{ SchemaVersion: ReportVersion, Enabled: store != nil, Doors: []Door{}, HTTPRequests: make(map[string]uint64), }} if store == nil { @@ -114,11 +136,39 @@ func Mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxO return nil, err } d.handlers[path] = server - d.report.Doors = append(d.report.Doors, Door{Provider: p.Provider, Region: p.Region, Path: path}) + door := Door{Provider: p.Provider, Region: p.Region, Path: path} + if hosts { + door.Host = DoorHost(p.Provider, p.Region) + cfg := server.cfg + cfg.endpointHost = door.Host + d.hosts[door.Host] = &Server{store: store, cfg: cfg} + } + d.report.Doors = append(d.report.Doors, door) } return d, nil } +// Handler keeps the control-plane rebinding guard, admitting only declared +// absolute-form HTTP authorities to S3's own loopback/Origin boundary. +func (d *Doors) Handler(addr string) http.Handler { + guarded := emulator.GuardRebinding(d, addr) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler := d.hosts[r.Host] + if handler == nil && !r.URL.IsAbs() && r.Method != http.MethodConnect { + guarded.ServeHTTP(w, r) + return + } + // TestHostDoorsRefuseAmbiguousProxyTargets holds the authority boundary. + if handler == nil || r.Method == http.MethodConnect || r.URL.Scheme != "http" || + r.URL.Host != r.Host || r.URL.User != nil || r.URL.Fragment != "" || r.URL.Opaque != "" { + d.log.WarnContext(r.Context(), "S3 proxy target refused") + http.Error(w, "feint: only declared absolute-form HTTP S3 targets are supported; never forwarded", http.StatusForbidden) + return + } + d.serveDoor(w, r, handler, "http://"+r.Host) + }) +} + // Snapshot returns detached report data so readers cannot mutate live evidence. func (d *Doors) Snapshot() Report { d.mu.Lock() @@ -165,32 +215,36 @@ func (d *Doors) ServeHTTP(w http.ResponseWriter, r *http.Request) { } request := r.Clone(r.Context()) request.URL.Path, request.URL.RawPath = path, "/"+remainder - observed := &doorWriter{ResponseWriter: w} - defer func() { - // TestAnInterruptedDoorDoesNotInventResponseEvidence holds this. - if observed.status == 0 { - d.log.ErrorContext(r.Context(), "S3 request ended without response headers") - return - } - d.mu.Lock() - key := prefix + " " + fmt.Sprint(observed.status) - d.report.HTTPRequests[key]++ - d.mu.Unlock() - }() - if r.Method == http.MethodPut { - if err := http.NewResponseController(w).SetReadDeadline(time.Now().Add(UploadReadTimeout)); err != nil { - d.log.ErrorContext(r.Context(), "S3 upload read deadline unavailable", "error", err) - http.Error(observed, "feint: S3 upload read deadline unavailable", http.StatusServiceUnavailable) - return - } - } - handler.ServeHTTP(observed, request) + d.serveDoor(w, request, handler, prefix) return } } d.refuse(w, r, "S3 door is not enabled or its prefix is not canonical") } +func (d *Doors) serveDoor(w http.ResponseWriter, r *http.Request, handler *Server, endpoint string) { + observed := &doorWriter{ResponseWriter: w} + defer func() { + // TestAnInterruptedDoorDoesNotInventResponseEvidence holds this. + if observed.status == 0 { + d.log.ErrorContext(r.Context(), "S3 request ended without response headers") + return + } + d.mu.Lock() + key := endpoint + " " + fmt.Sprint(observed.status) + d.report.HTTPRequests[key]++ + d.mu.Unlock() + }() + if r.Method == http.MethodPut { + if err := http.NewResponseController(w).SetReadDeadline(time.Now().Add(UploadReadTimeout)); err != nil { + d.log.ErrorContext(r.Context(), "S3 upload read deadline unavailable", "error", err) + http.Error(observed, "feint: S3 upload read deadline unavailable", http.StatusServiceUnavailable) + return + } + } + handler.ServeHTTP(observed, r) +} + func (d *Doors) refuse(w http.ResponseWriter, r *http.Request, reason string) { d.log.WarnContext(r.Context(), "S3 dispatch refused", "reason", reason) http.Error(w, "feint: "+reason, http.StatusNotFound) diff --git a/internal/s3/host_doors_test.go b/internal/s3/host_doors_test.go new file mode 100644 index 00000000..a8566355 --- /dev/null +++ b/internal/s3/host_doors_test.go @@ -0,0 +1,157 @@ +package s3 + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func hostFixture(t *testing.T) *Doors { + t.Helper() + _, store := fixture(t) + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("control-witness")) + }) + doors, err := MountHosts(next, store, []Profile{ + {Provider: "scaleway", Region: "fr-par", MissingBucket: ScalewayMissingBucket}, + {Provider: "exoscale", Region: "ch-gva-2", MissingBucket: ExoscaleMissingBucket}, + {Provider: "exoscale", Region: "ch-dk-2", MissingBucket: ExoscaleMissingBucket}, + }, 512, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + return doors +} + +func TestHostDoorsSharePrefixBytesWithoutCollapsingNamespaces(t *testing.T) { + doors := hostFixture(t) + ts := httptest.NewServer(doors.Handler("127.0.0.1:4599")) + defer ts.Close() + proxy, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxy)}} + defer client.CloseIdleConnections() + wireKey := "folder//a/../%2F%252F%3F%23" + for i, door := range doors.Snapshot().Doors { + if door.Host != DoorHost(door.Provider, door.Region) { + t.Fatal("host report lost its explicit namespace:", door) + } + target := "http://" + door.Host + "/bucket" + doorRequest(t, client, "PUT", target, "", 200) + payload := strings.Repeat("host", i+1) + doorRequest(t, client, "PUT", target+"/"+wireKey, payload, 200) + got := doorRequest(t, ts.Client(), "GET", ts.URL+door.Path+"bucket/"+wireKey, "", 200) + if string(got) != payload { + t.Fatalf("prefix did not expose the same host bytes: %q", got) + } + } + for i, door := range doors.Snapshot().Doors { + got := doorRequest(t, client, "GET", "http://"+door.Host+"/bucket/"+wireKey, "", 200) + if string(got) != strings.Repeat("host", i+1) { + t.Fatal("a later host write contaminated another provider or region:", door) + } + if doors.Snapshot().HTTPRequests["http://"+door.Host+" 200"] != 3 { + t.Fatal("host evidence did not count the actual completed exchanges") + } + } + if got := doorRequest(t, ts.Client(), "GET", ts.URL+"/native", "", 200); string(got) != "control-witness" { + t.Fatal("host mounting displaced the control plane") + } +} + +func TestHostDoorsRefuseAmbiguousProxyTargets(t *testing.T) { + doors := hostFixture(t) + handler := doors.Handler("127.0.0.1:4599") + host := DoorHost("exoscale", "ch-gva-2") + for _, tc := range []struct { + name, target, host, method string + }{ + {"unknown", "http://unknown.s3.feint.invalid/native", "unknown.s3.feint.invalid", "GET"}, + {"undeclared region", "http://exoscale.foreign.s3.feint.invalid/native", "exoscale.foreign.s3.feint.invalid", "GET"}, + {"authority mismatch", "http://foreign.example/native", host, "GET"}, + {"loopback Host forgery", "http://foreign.example/native", "localhost", "GET"}, + {"origin form", "/native", host, "GET"}, + {"https", "https://" + host + "/bucket", host, "HEAD"}, + {"credentials", "http://user@" + host + "/bucket", host, "HEAD"}, + {"connect", "http://" + host + "/bucket", host, "CONNECT"}, + {"explicit port alias", "http://" + host + ":80/bucket", host + ":80", "HEAD"}, + {"case alias", "http://" + strings.ToUpper(host) + "/bucket", strings.ToUpper(host), "HEAD"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(tc.method, tc.target, nil) + r.Host, r.RemoteAddr = tc.host, "127.0.0.1:1234" + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != 403 || bytes.Contains(w.Body.Bytes(), []byte("control-witness")) { + t.Fatalf("ambiguous target reached a handler: status=%d body=%q", w.Code, w.Body) + } + }) + } + // A known authority cannot reach reports, state, health or native routes. + for _, path := range []string{ReportPath, "/_feint/state", "/_feint/health", "/native"} { + r := httptest.NewRequest("GET", "http://"+host+path, nil) + r.RemoteAddr = "127.0.0.1:1234" + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code == 200 || bytes.Contains(w.Body.Bytes(), []byte("control-witness")) { + t.Fatal("S3 host escaped into the control plane:", path, w.Code, w.Body) + } + } +} + +func TestHostDoorsRetainOriginPeerAndUploadDeadlineGuards(t *testing.T) { + doors := hostFixture(t) + handler := doors.Handler("0.0.0.0:4599") + target := "http://" + DoorHost("exoscale", "ch-gva-2") + "/bucket" + for _, tc := range []struct{ peer, origin string }{ + {"192.0.2.10:1234", ""}, + {"127.0.0.1:1234", "https://foreign.example"}, + {"127.0.0.1:1234", "http://localhost"}, + } { + r := httptest.NewRequest("HEAD", target, nil) + r.RemoteAddr = tc.peer + if tc.origin != "" { + r.Header.Set("Origin", tc.origin) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != 403 { + t.Fatal("host mode weakened the S3 peer/Origin boundary:", tc, w.Code) + } + } + r := httptest.NewRequest("PUT", target, nil) + r.RemoteAddr = "127.0.0.1:1234" + unsupported := httptest.NewRecorder() + handler.ServeHTTP(unsupported, r) + if unsupported.Code != 503 { + t.Fatal("host upload bypassed its read deadline:", unsupported.Code) + } + w := &deadlineWriter{ResponseRecorder: httptest.NewRecorder()} + handler.ServeHTTP(w, r) + if w.Code != 200 || w.deadline.IsZero() { + t.Fatal("valid loopback host PUT was refused or unbounded:", w.Code, w.deadline) + } +} + +func TestHostDoorsRequireExplicitActivation(t *testing.T) { + doors, _ := mountedFixture(t) + r := httptest.NewRequest("HEAD", "http://"+DoorHost("exoscale", "ch-gva-2")+"/bucket", nil) + r.RemoteAddr = "127.0.0.1:1234" + w := httptest.NewRecorder() + doors.Handler("127.0.0.1:4599").ServeHTTP(w, r) + if w.Code != 403 { + t.Fatal("prefixed-only deployment accepted a synthetic host:", w.Code) + } + for _, door := range doors.Snapshot().Doors { + if door.Host != "" { + t.Fatal("prefixed-only deployment advertised host activation") + } + } +} diff --git a/internal/s3/server.go b/internal/s3/server.go index 4f6fe916..b583f79c 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -52,6 +52,8 @@ type Config struct { // BucketConfiguration enables the measured private, non-versioned bucket // workflow, independently of the missing-bucket error dialect. BucketConfiguration bool + // Set only by the explicit host-door composition, not by direct callers. + endpointHost string } // Server is a direct HTTP handler. Do not put it behind a path-cleaning mux: @@ -94,7 +96,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { host = h } ip, ipErr := netip.ParseAddr(strings.Trim(host, "[]")) - if host != "localhost" && (ipErr != nil || !ip.Unmap().IsLoopback()) { + if host != "localhost" && (ipErr != nil || !ip.Unmap().IsLoopback()) && + (s.cfg.endpointHost == "" || r.Host != s.cfg.endpointHost) { s.refuse(w, r, http.StatusNotImplemented, "only explicit loopback path-style endpoints are supported", nil) return } diff --git a/mise.toml b/mise.toml index f830fba5..cae9b5d2 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove prefixed bytes and private Scaleway buckets through real start/stop/restart with AWS CLI 2.32.17 and scw 2.62.0" +description = "Prove prefixed AWS/scw bytes and host-routed exo 1.95.1 upload through real start/stop/restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:exo:endpoint"] diff --git a/tools/falsify/specs/mounted-object-storage.json b/tools/falsify/specs/mounted-object-storage.json index 3f7f8b6e..3360fafd 100644 --- a/tools/falsify/specs/mounted-object-storage.json +++ b/tools/falsify/specs/mounted-object-storage.json @@ -20,8 +20,8 @@ { "label": "interrupted handlers invent completed HTTP response evidence", "file": "internal/s3/doors.go", - "find": "if observed.status == 0 {\n\t\t\t\t\td.log.ErrorContext", - "replace": "if observed.status == 0 && false {\n\t\t\t\t\td.log.ErrorContext", + "find": "if observed.status == 0 {\n\t\t\td.log.ErrorContext", + "replace": "if observed.status == 0 && false {\n\t\t\td.log.ErrorContext", "test": "TestAnInterruptedDoorDoesNotInventResponseEvidence" }, { @@ -83,8 +83,8 @@ { "label": "PUT proceeds without read-deadline support", "file": "internal/s3/doors.go", - "find": "err != nil {\n\t\t\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", - "replace": "err != nil && false {\n\t\t\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", + "find": "err != nil {\n\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", + "replace": "err != nil && false {\n\t\t\td.log.ErrorContext(r.Context(), \"S3 upload read deadline unavailable\"", "test": "TestMountedPutRequiresAndInstallsBodyReadDeadline" }, { diff --git a/tools/falsify/specs/s3-bucket-configuration.json b/tools/falsify/specs/s3-bucket-configuration.json index 01918cd9..344ea884 100644 --- a/tools/falsify/specs/s3-bucket-configuration.json +++ b/tools/falsify/specs/s3-bucket-configuration.json @@ -145,8 +145,8 @@ "label": "SDK endpoint exports retain the refused terminal slash", "package": "./internal/cli/", "file": "internal/cli/object_storage.go", - "find": "vars[key] = strings.TrimSuffix(endpoint, \"/\") + strings.TrimSuffix(selected.Path, \"/\")", - "replace": "vars[key] = strings.TrimSuffix(endpoint, \"/\") + selected.Path", + "find": "target := strings.TrimSuffix(endpoint, \"/\") + strings.TrimSuffix(selected.Path, \"/\")", + "replace": "target := strings.TrimSuffix(endpoint, \"/\") + selected.Path", "test": "TestS3EnvOnlyExportsLiveSelectedDoorsAndUnsetWorksOffline" } ] diff --git a/tools/falsify/specs/s3-gateway.json b/tools/falsify/specs/s3-gateway.json index 84002d23..386dda2b 100644 --- a/tools/falsify/specs/s3-gateway.json +++ b/tools/falsify/specs/s3-gateway.json @@ -46,8 +46,8 @@ { "label": "virtual cloud hostnames are treated as path-style endpoints", "file": "internal/s3/server.go", - "find": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) {", - "replace": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) && false {", + "find": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) &&\n\t\t(s.cfg.endpointHost == \"\" || r.Host != s.cfg.endpointHost) {", + "replace": "if host != \"localhost\" && (ipErr != nil || !ip.Unmap().IsLoopback()) &&\n\t\t(s.cfg.endpointHost == \"\" || r.Host != s.cfg.endpointHost) && false {", "test": "TestLoopbackAndPathStyleBoundary" }, { diff --git a/tools/falsify/specs/s3-host-doors.json b/tools/falsify/specs/s3-host-doors.json new file mode 100644 index 00000000..1de16686 --- /dev/null +++ b/tools/falsify/specs/s3-host-doors.json @@ -0,0 +1,95 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "absolute proxy requests can enter the native control plane", + "file": "internal/s3/doors.go", + "find": "handler == nil && !r.URL.IsAbs() && r.Method != http.MethodConnect", + "replace": "handler == nil && (!r.URL.IsAbs() || true) && r.Method != http.MethodConnect", + "test": "TestHostDoorsRefuseAmbiguousProxyTargets" + }, + { + "label": "proxy URL and Host need not agree", + "file": "internal/s3/doors.go", + "find": "r.URL.Host != r.Host || r.URL.User != nil", + "replace": "(r.URL.Host != r.Host && false) || r.URL.User != nil", + "test": "TestHostDoorsRefuseAmbiguousProxyTargets" + }, + { + "label": "HTTPS and origin-form requests masquerade as HTTP proxy requests", + "file": "internal/s3/doors.go", + "find": "r.URL.Scheme != \"http\"", + "replace": "r.URL.Scheme == \"\\x00never\"", + "test": "TestHostDoorsRefuseAmbiguousProxyTargets" + }, + { + "label": "proxy targets can carry URL credentials", + "file": "internal/s3/doors.go", + "find": "r.URL.User != nil || r.URL.Fragment", + "replace": "(r.URL.User != nil && false) || r.URL.Fragment", + "test": "TestHostDoorsRefuseAmbiguousProxyTargets" + }, + { + "label": "S3 host peer validation is disabled", + "file": "internal/s3/server.go", + "find": "if err != nil || !peer.Addr().Unmap().IsLoopback() {", + "replace": "if (err != nil || !peer.Addr().Unmap().IsLoopback()) && false {", + "test": "TestHostDoorsRetainOriginPeerAndUploadDeadlineGuards" + }, + { + "label": "S3 host browser Origins are admitted", + "file": "internal/s3/server.go", + "find": "if len(r.Header.Values(\"Origin\")) != 0 {", + "replace": "if len(r.Header.Values(\"Origin\")) != 0 && false {", + "test": "TestHostDoorsRetainOriginPeerAndUploadDeadlineGuards" + }, + { + "label": "live host reports can export a foreign authority", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "(door.Host != \"\" && door.Host != s3.DoorHost(door.Provider, door.Region))", + "replace": "(door.Host != \"\" && door.Host != s3.DoorHost(door.Provider, door.Region) && false)", + "test": "TestS3DiscoveryRefusesRedirectsInvalidReportsAndForeignEndpoints" + }, + { + "label": "host activation without storage succeeds silently", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if f.hosts && f.root == \"\" {", + "replace": "if f.hosts && f.root == \"\" && false {", + "test": "TestObjectStorageValidatesDefaultsAndLimitsWithoutStartingAnything" + }, + { + "label": "host mode replaces the measured prefix of every client", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if profile.HostEndpoint && selected.Host != \"\" {", + "replace": "if (profile.HostEndpoint || true) && selected.Host != \"\" {", + "test": "TestS3HostEnvironmentUsesOnlyDeclaredAuthorityAndLocalProxy" + }, + { + "label": "host proxy exports can point away from the local emulator", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "vars[key] = strings.TrimSuffix(endpoint, \"/\")", + "replace": "vars[key] = strings.TrimSuffix(\"http://foreign.example/\"+endpoint, \"/\")", + "test": "TestS3HostEnvironmentUsesOnlyDeclaredAuthorityAndLocalProxy" + }, + { + "label": "NO_PROXY can bypass the synthetic host interception", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "vars[key] = \"127.0.0.1,localhost,::1\"", + "replace": "vars[key] = \"*\"", + "test": "TestS3HostEnvironmentUsesOnlyDeclaredAuthorityAndLocalProxy" + }, + { + "label": "host mode is lost on restart", + "package": "./internal/cli/", + "file": "internal/cli/object_storage.go", + "find": "if f.hosts {\n\t\targs = append(args, \"--s3-host-doors\")", + "replace": "if f.hosts && false {\n\t\targs = append(args, \"--s3-host-doors\")", + "test": "TestStartPreservesObjectStorageFlagsAndAbsoluteRoot" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 32ccac40..29362fb1 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -260,9 +260,9 @@ var rules = []rule{ }, { Path: "internal/s3/", - Why: "bounded path-style S3 protocol, private Scaleway buckets and real-byte official-client proof", + Why: "bounded S3 prefix/host dispatch, private Scaleway buckets and real-byte official-client proof", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, - Unproven: "one AWS CLI and one scw version, not complete Exoscale lifecycles, public ACLs, version histories, virtual hosts, interception, ETags, multipart or public stacks", + Unproven: "one AWS/scw/exo version, not complete Exoscale lifecycles, public ACLs, version histories, TLS interception, cloud virtual hosts, ETags, multipart or public stacks", }, { Path: "internal/conformancetest/", From ff2df80c20dceb433a33b1051b0cfa3eed8cc619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 13:42:50 +0100 Subject: [PATCH 08/28] feat(s3): prove Exo listing and multi-range downloads Serve bounded ListObjectsV2 and single-byte ranges through the existing scoped doors. Persist publication dates in neutral metadata v3 while retaining read-only v1/v2 compatibility and explicitly refusing undated legacy listings. Real Exo 1.95.1 downloads 73,728-byte and 6,340,608-byte objects before and after actual restart; AWS independently decodes pagination. All 25 new and 96 tightly coupled inherited falsifications compile, bite and restore green. Deterministic checks, tagged lint, generated docs, official-client witnesses and the partial probe leg pass. Independent drift remains code 2 for the two untriaged Scaleway LB operations. The limits freshness signal also reports ten unchanged sections with stale acknowledgements; their dates were not refreshed without an audit. Refs #850 Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 10 + CHANGELOG.md | 10 + docs/confidence.md | 2 +- docs/limits.md | 11 +- docs/object-storage-routing.md | 42 ++++ .../cli/object_storage_conformance_test.go | 94 ++++++- .../core/objectstore/bucket_metadata_test.go | 7 +- internal/core/objectstore/files.go | 9 +- internal/core/objectstore/manifest.go | 9 +- internal/core/objectstore/modified_test.go | 112 +++++++++ internal/core/objectstore/store.go | 10 +- internal/core/objectstore/store_test.go | 4 +- internal/s3/list.go | 232 ++++++++++++++++++ internal/s3/list_test.go | 224 +++++++++++++++++ internal/s3/range.go | 58 +++++ internal/s3/range_test.go | 105 ++++++++ internal/s3/server.go | 60 ++++- internal/s3/server_test.go | 4 +- mise.toml | 2 +- tools/falsify/specs/neutral-object-bytes.json | 8 +- tools/falsify/specs/s3-download.json | 184 ++++++++++++++ tools/falsify/specs/s3-gateway.json | 6 +- tools/testplan/rules.go | 4 +- 23 files changed, 1170 insertions(+), 37 deletions(-) create mode 100644 internal/core/objectstore/modified_test.go create mode 100644 internal/s3/list.go create mode 100644 internal/s3/list_test.go create mode 100644 internal/s3/range.go create mode 100644 internal/s3/range_test.go create mode 100644 tools/falsify/specs/s3-download.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index bbf00cb7..88a25387 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,16 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **Le listing Exo et les téléchargements multi-plages, mesurés** (#850) : + exo 1.95.1 retrouve deux binaires de 73 728 et 6 340 608 octets, octets/SHA-256 + et dates de listing identiques avant/après redémarrage réel. AWS décode aussi + deux pages ListObjectsV2 d'une entrée. Les GET par plage servent les vrais + offsets/longueurs, sans checksum du fichier entier sur une réponse partielle. + Le format privé v3 persiste les dates de publication, lit v1/v2 sans les réécrire + et migre à la prochaine mutation ; les anciens binaires ne lisent pas v3. + Une page contenant un ancien objet sans date est refusée jusqu'à sa réécriture. + ETag, multipart et cycles complets de buckets Exoscale restent hors périmètre. + - **Des hôtes S3 synthétiques monoport, prouvés par les uploads Exo** (#850) : `serve/start --s3-host-doors`, opt-in, conserve les préfixes et sert les autorités HTTP provider/région par un proxy local qui ne transfère rien. diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d401b9..51ee40ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,16 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Measured Exo object listing and multi-range downloads** (#850): + exo 1.95.1 downloads 73,728-byte and 6,340,608-byte binary objects with exact + byte/SHA-256 and listing-date proof before/after actual restart. AWS independently + decodes two one-entry ListObjectsV2 pages. Single-byte-range GETs serve actual + offsets/lengths without whole-object checksums on partial bytes. + Private metadata v3 persists publication dates, reads v1/v2 without rewriting + them and upgrades on mutation; older binaries cannot read v3. Listing pages + containing legacy undated objects are explicitly refused until overwrite. + ETags, multipart and complete Exoscale bucket workflows remain unsupported. + - **Same-port synthetic S3 hosts, proved with Exo uploads** (#850): opt-in `serve/start --s3-host-doors` retains the prefix doors and serves declared provider/region HTTP authorities through a local non-forwarding diff --git a/docs/confidence.md b/docs/confidence.md index 2b193926..9af568ac 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -44,7 +44,7 @@ prints the reason under the pack that owns it. | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets and host-routed Exo uploads with byte/checksum proof across actual restart; complete Exoscale lifecycles are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-upload-through-a-synthetic-host) | +| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets, AWS paginated listing and host-routed Exo uploads/downloads (including multi-range 6,340,608 bytes), with byte/checksum and persisted-date proof across actual restart; complete Exoscale bucket workflows are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-listing-and-multi-range-downloads) | | Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | diff --git a/docs/limits.md b/docs/limits.md index 53eec117..adbf7759 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -11,13 +11,18 @@ declared `/_feint/s3///` endpoints, cumulative byte budgets and an AWS CLI 2.32.17 / scw 2.62.0 restart proof. The Scaleway private, non-versioned bucket `create/list/get/delete` workflow is served. See the [door ADR and commands](object-storage-routing.md). -Opt-in `--s3-host-doors` also permits exo 1.95.1 simple uploads to an existing +Opt-in `--s3-host-doors` also permits exo 1.95.1 uploads, object listing and downloads from an existing bucket through a synthetic scoped authority and the same-port non-forwarding HTTP proxy. AWS retrieval proves the uploaded binary bytes and SHA-256 survive -actual restart. This does not implement bucket virtual hosts, DNS/TLS +actual restart. Real Exo downloads also preserve a 6,340,608-byte object across +multiple HTTP ranges. ListObjectsV2 exposes actual persisted modification dates; +legacy undated objects must be overwritten before their listing can be served. +Private byte metadata v3 reads v1/v2 without inventing dates; successful mutations +upgrade it, and older binaries cannot read v3. +This does not implement bucket virtual hosts, DNS/TLS interception or a complete Exoscale lifecycle; use a dedicated shell because its S3 environment replaces proxy settings. -Complete Exoscale workflows, object listing, writable tags/public ACLs, +Complete Exoscale bucket workflows, ListObjects v1, owner requests, writable tags/public ACLs, version histories, multipart, ETags, checkpoint/reset and hardcoded Terraform redirection remain unsupported. No MinIO service or dependency is used by this implementation. diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 41857682..acda7415 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -301,3 +301,45 @@ The witness uses pack-owned public fake credentials. Its refusing proxy is not a network namespace and does not constrain unrelated clients that ignore proxy settings. Host response counters are separate from prefix counters and start again at zero on restart; neither is official-client or SDK-coverage evidence. + +## Measured Exoscale listing and multi-range downloads + +`conformance:s3:mounted` also drives exo 1.95.1 object listing and downloads +through the same scoped host. It checks actual listing keys, sizes and dates +and compares downloaded binary bytes and SHA-256 before and after a real process +restart. The small object is 73,728 bytes. A 6,340,608-byte object exceeds the +client's 5 MiB download part size and produces multiple `206` responses. +That object is seeded with a **simple AWS PUT**, not an Exo multipart upload. +AWS additionally decodes two actual one-entry listing pages, including a literal +key with escaped characters. + +The gateway implements **ListObjectsV2 only**, with bounded prefix, delimiter, +start-after and max-keys (0..1000), deduplicated common prefixes and scoped +continuation tokens. An empty optional token means the first page, as the real +Exo downloader requests it. Cursors are bounded, closed JSON bound to the +provider/region/bucket/prefix/delimiter; they are neither signed nor IAM and do +not freeze an inventory against concurrent mutations. `encoding-type=url` +preserves keys unrepresentable in XML; without it, such keys are explicitly +refused, never silently replaced. Owner requests and v1 listing remain refused. + +Single `bytes` ranges support closed, open-ended and suffix forms. GET returns +only the selected bytes with actual Content-Length and Content-Range. +Unsatisfiable ranges return a plain Feint `416` with `bytes */`. +Multiple ranges, conditional reads and Range on other methods remain refused. +Every range still verifies the **entire stored blob** before seeking. +Partial responses deliberately omit whole-object checksum headers; full reads +retain the verified SHA-256. ETags remain absent. + +Private byte metadata is now **version 3**. New stores and successful mutations +write v3; v1/v2 are still readable and reads do not rewrite them. PUT persists +its publication date, reused by GET/HEAD and listings across restart. Old +undated object references retain no Last-Modified header, and pages containing +them refuse listing until those objects are overwritten. Neither blob mtime nor +restart time is substituted. **Older binaries cannot read an upgraded v3 index**: +keep a pre-upgrade copy of the whole owned store if rollback is required. +Resource snapshots still do not own these bytes. + +The date, listing and range properties are replayed by +`tools/falsify/specs/s3-download.json`. This milestone does **not** certify normal +Exo bucket creation/cleanup, ACL/CORS/ownership discovery, native bucket usage, +multipart, version histories, bulk deletion, TLS or Terraform redirection. diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index 9b53ebe6..1c245227 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -79,12 +79,22 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Errorf("stop the owned S3 witness: %v\n%s", err, out) } }) + t.Cleanup(func() { + if t.Failed() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, binary, "logs", "--addr", addr) + cmd.Env = aws.Env + out, err := cmd.CombinedOutput() + t.Logf("owned witness logs (read error=%v): %s", err, out) + } + }) run("start", "--addr", addr, "--vm", "off", "--objectstore", "bytes", - "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1", "--s3-host-doors") + "--objectstore-budget-mib", "20", "--s3-object-limit-mib", "8", "--s3-host-doors") var first statusReport if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &first); err != nil || !first.Running || first.PID <= 0 || first.ObjectStorage == nil || - first.ObjectStorage.MaxBlobBytes != 3<<20 || first.ObjectStorage.MaxObjectBytes != 1<<20 { + first.ObjectStorage.MaxBlobBytes != 20<<20 || first.ObjectStorage.MaxObjectBytes != 8<<20 { t.Fatalf("real process/configuration did not become ready: %+v %v", first, err) } if len(first.ObjectStorage.Doors) != 2 { @@ -232,6 +242,66 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { exoKey := "host-byte-proof" exo.Run(t, "storage", "upload", input, "sos://"+bucket+"/"+exoKey) getKey(first.ObjectStorage.Doors[1], exoKey, payload) + largePayload := bytes.Repeat(payload, 86) + largeInput := filepath.Join(aws.Home, "large-input") + if err := os.WriteFile(largeInput, largePayload, 0600); err != nil { + t.Fatal(err) + } + largeKey := "large-byte-proof" + // Seed a simple PUT, not an unsupported multipart upload. The Exo downloader + // must then read more than its 5 MiB part size through multiple byte ranges. + call(first.ObjectStorage.Doors[1], "put-object", "--bucket", bucket, "--key", largeKey, + "--body", largeInput, "--checksum-algorithm", "CRC32") + download := func(objectKey string, expected []byte) { + t.Helper() + destination := filepath.Join(exo.Home, "download-"+objectKey) + out := exo.Run(t, "storage", "download", "--force", "sos://"+bucket+"/"+objectKey, destination) + got, err := os.ReadFile(destination) + if err != nil || !bytes.Equal(got, expected) || sha256.Sum256(got) != sha256.Sum256(expected) { + t.Fatalf("Exo's exit code is not byte proof: read=%v bytes=%d want=%d output=%q", err, len(got), len(expected), out) + } + } + download(exoKey, payload) + download(largeKey, largePayload) + var page struct { + IsTruncated bool + NextContinuationToken string + KeyCount int + Contents []struct { + Key, LastModified string + Size int64 + } + } + firstPage := call(first.ObjectStorage.Doors[1], "list-objects-v2", "--bucket", bucket, + "--max-keys", "1", "--no-paginate") + if err := json.Unmarshal(firstPage, &page); err != nil || !page.IsTruncated || + page.KeyCount != 1 || len(page.Contents) != 1 || page.Contents[0].Key != key || page.NextContinuationToken == "" { + t.Fatalf("real AWS decoder did not receive the literal paginated key: %v %q", err, firstPage) + } + secondPage := call(first.ObjectStorage.Doors[1], "list-objects-v2", "--bucket", bucket, + "--max-keys", "1", "--no-paginate", "--continuation-token", page.NextContinuationToken) + if err := json.Unmarshal(secondPage, &page); err != nil || page.KeyCount != 1 || + len(page.Contents) != 1 || page.Contents[0].Key != exoKey || page.Contents[0].Size != int64(len(payload)) { + t.Fatalf("real AWS pagination repeated or invented an entry: %v %q", err, secondPage) + } + beforeObjects := exo.Run(t, "--output-format", "json", "storage", "list", "sos://"+bucket+"/") + var listed []struct { + Name string `json:"name"` + Size int64 `json:"size"` + LastModified string `json:"last_modified"` + } + if err := json.Unmarshal(beforeObjects, &listed); err != nil { + t.Fatal("decode real Exo listing:", err, string(beforeObjects)) + } + foundLarge := false + for _, obj := range listed { + if obj.Name == largeKey && obj.Size == int64(len(largePayload)) && obj.LastModified != "" { + foundLarge = true + } + } + if !foundLarge { + t.Fatal("actual Exo listing omitted size/date of the uploaded object:", string(beforeObjects)) + } // A same-name object in the other provider is a positive isolation control. if err := os.WriteFile(input, []byte("other-provider-host-key"), 0600); err != nil { t.Fatal(err) @@ -251,7 +321,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { var restarted statusReport if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &restarted); err != nil || !restarted.Running || restarted.PID == first.PID || restarted.ObjectStorage == nil || - restarted.ObjectStorage.MaxBlobBytes != 3<<20 || restarted.ObjectStorage.MaxObjectBytes != 1<<20 { + restarted.ObjectStorage.MaxBlobBytes != 20<<20 || restarted.ObjectStorage.MaxObjectBytes != 8<<20 { t.Fatalf("restart lost the process or persisted configuration: %+v %v", restarted, err) } get(first.ObjectStorage.Doors[0], payload) @@ -262,13 +332,22 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { restarted.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 0 { t.Fatal("restart lost host configuration or invented persisted HTTP evidence") } + if after := exo.Run(t, "--output-format", "json", "storage", "list", "sos://"+bucket+"/"); !bytes.Equal(after, beforeObjects) { + t.Fatalf("actual Exo object inventory changed across restart: before=%q after=%q", beforeObjects, after) + } exo.Run(t, "storage", "upload", input, "sos://"+bucket+"/after-restart") getKey(first.ObjectStorage.Doors[1], "after-restart", []byte("other-provider-host-key")) var live statusReport if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &live); err != nil || - live.ObjectStorage == nil || live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 2 { + live.ObjectStorage == nil || live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 4 { t.Fatalf("actual Exo HEAD/PUT after restart did not traverse its declared host: %+v %v", live.ObjectStorage, err) } + download(exoKey, payload) + download(largeKey, largePayload) + if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &live); err != nil || + live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 206"] < 3 { + t.Fatal("actual Exo small/multi-part downloads did not produce partial HTTP responses:", live.ObjectStorage, err) + } run("doctor", "--addr", addr, "--s3", "--vm", "off") if after := bucketCall("get", true); !bytes.Equal(after, beforeBucket) { t.Fatalf("Scaleway bucket configuration changed across restart: before=%q after=%q", beforeBucket, after) @@ -283,7 +362,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { // Inspect only our private store while the owner is stopped: GET of the // same cleaned key twice would otherwise masquerade as key preservation. run("stop", "--addr", addr) - store, err := objectstore.OpenWithLimit(t.Context(), root, 3<<20) + store, err := objectstore.OpenWithLimit(t.Context(), root, 20<<20) if err != nil { t.Fatal(err) } @@ -297,13 +376,14 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatal(err) } run("start", "--addr", addr, "--vm", "off", "--objectstore", root, - "--objectstore-budget-mib", "3", "--s3-object-limit-mib", "1", "--s3-host-doors") + "--objectstore-budget-mib", "20", "--s3-object-limit-mib", "8", "--s3-host-doors") for _, door := range first.ObjectStorage.Doors { call(door, "head-object", "--bucket", bucket, "--key", key) call(door, "delete-object", "--bucket", bucket, "--key", key) call(door, "delete-object", "--bucket", bucket, "--key", exoKey) if door.Provider == "exoscale" { call(door, "delete-object", "--bucket", bucket, "--key", "after-restart") + call(door, "delete-object", "--bucket", bucket, "--key", largeKey) } if door.Provider == "scaleway" { bucketCall("delete", true) @@ -314,6 +394,8 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if guardHits.Load() != 1 { t.Fatalf("official client reached the refusing egress proxy: delta=%d", guardHits.Load()-1) } + t.Logf("Exo downloads before/after actual restart: small=%d bytes; multi-range=%d bytes sha256=%x", + len(payload), len(largePayload), sha256.Sum256(largePayload)) t.Logf("actual feint pid %d -> %d; two same-port namespaces; preserved key; bytes=%d sha256=%s; stop/reopen/restart; proxy delta=0 (not a network namespace)", first.PID, restarted.PID, len(payload), hex.EncodeToString(sum[:])) } diff --git a/internal/core/objectstore/bucket_metadata_test.go b/internal/core/objectstore/bucket_metadata_test.go index a2eea681..240bfc14 100644 --- a/internal/core/objectstore/bucket_metadata_test.go +++ b/internal/core/objectstore/bucket_metadata_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync" "testing" + "time" ) func TestBucketMetadataPersistsAndDoesNotAlias(t *testing.T) { @@ -99,6 +100,10 @@ func TestLegacyBucketMetadataUpgradeAndValidation(t *testing.T) { mustPut(t, s, "bucket", "key", "legacy") legacy := s.clone() legacy.Version = 1 + for key, obj := range legacy.Buckets["bucket"].Objects { + obj.Modified = time.Time{} + legacy.Buckets["bucket"].Objects[key] = obj + } body, err := json.Marshal(legacy) if err != nil { t.Fatal(err) @@ -130,7 +135,7 @@ func TestLegacyBucketMetadataUpgradeAndValidation(t *testing.T) { if err := reopened.SetBucketMetadata(t.Context(), "bucket", "key", "value"); err != nil { t.Fatal(err) } - if reopened.state.Version != 2 { + if reopened.state.Version != 3 { t.Fatal("write did not upgrade metadata version") } if _, err := reopened.HeadObject(t.Context(), "bucket", "key"); err != nil { diff --git a/internal/core/objectstore/files.go b/internal/core/objectstore/files.go index 68d3fa04..5c149957 100644 --- a/internal/core/objectstore/files.go +++ b/internal/core/objectstore/files.go @@ -99,7 +99,7 @@ func Open(ctx context.Context, path string) (result *Store, err error) { if err := root.Mkdir("blobs", 0o700); err != nil { return nil, fmt.Errorf("create blob directory: %w", err) } - s.state = manifest{Format: storeFormat, Version: 2, Buckets: make(map[string]*bucket)} + s.state = manifest{Format: storeFormat, Version: storeVersion, Buckets: make(map[string]*bucket)} if err := s.commit(ctx, s.state); err != nil { return nil, err } @@ -359,3 +359,10 @@ func (r *blobReader) Read(p []byte) (int, error) { func (r *blobReader) Close() error { return r.file.Close() } + +func (r *blobReader) Seek(offset int64, whence int) (int64, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.file.Seek(offset, whence) +} diff --git a/internal/core/objectstore/manifest.go b/internal/core/objectstore/manifest.go index 8762a8e1..dcd0a7b8 100644 --- a/internal/core/objectstore/manifest.go +++ b/internal/core/objectstore/manifest.go @@ -14,6 +14,7 @@ import ( const ( storeFormat = "feint-objectstore" + storeVersion = 3 maxManifestBytes = 64 << 20 ) @@ -47,7 +48,7 @@ func decodeManifest(body []byte) (manifest, error) { if state.Format != storeFormat { return manifest{}, ErrUnowned } - if (state.Version != 1 && state.Version != 2) || state.Buckets == nil { + if state.Version < 1 || state.Version > storeVersion || state.Buckets == nil { return manifest{}, fmt.Errorf("%w: unsupported version or missing buckets", ErrCorrupt) } for name, b := range state.Buckets { @@ -64,6 +65,12 @@ func decodeManifest(body []byte) (manifest, error) { return manifest{}, errors.Join(ErrCorrupt, err) } for key, obj := range b.Objects { + if state.Version < 3 && !obj.Modified.IsZero() { + return manifest{}, fmt.Errorf("%w: object modification dates require metadata version 3", ErrCorrupt) + } + if !obj.Modified.IsZero() && (obj.Modified.Year() < 1 || obj.Modified.Year() > 9999) { + return manifest{}, fmt.Errorf("%w: invalid object modification date", ErrCorrupt) + } if err := validKey(key); err != nil { return manifest{}, errors.Join(ErrCorrupt, err) } diff --git a/internal/core/objectstore/modified_test.go b/internal/core/objectstore/modified_test.go new file mode 100644 index 00000000..4481bb87 --- /dev/null +++ b/internal/core/objectstore/modified_test.go @@ -0,0 +1,112 @@ +package objectstore + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" +) + +func TestObjectModificationDatePersistsAndLegacyDatesAreNotInvented(t *testing.T) { + s, path := newStore(t) + if err := s.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + before := time.Now() + obj := mustPut(t, s, "bucket", "key", "bytes") + if obj.Modified.IsZero() || obj.Modified.Before(before) || obj.Modified.After(time.Now()) { + t.Fatal("new object lacks its actual commit time:", obj.Modified) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + got, err := reopened.HeadObject(t.Context(), "bucket", "key") + if err != nil || !got.Modified.Equal(obj.Modified) { + t.Fatal("restart changed the object's modification date:", got, err) + } + legacy := reopened.clone() + legacy.Version = 2 + incompatible, err := json.Marshal(legacy) + if err != nil { + t.Fatal(err) + } + if _, err := decodeManifest(incompatible); !errors.Is(err, ErrCorrupt) { + t.Fatal("version 2 admitted a version 3 object date:", err) + } + yearZero := reopened.clone() + obj.Modified = time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) + yearZero.Buckets["bucket"].Objects["key"] = obj + invalidDate, err := json.Marshal(yearZero) + if err != nil { + t.Fatal(err) + } + if _, err := decodeManifest(invalidDate); !errors.Is(err, ErrCorrupt) { + t.Fatal("restored modification date was not validated:", err) + } + got.Modified = time.Time{} + legacy.Buckets["bucket"].Objects["key"] = got + raw, err := json.Marshal(legacy) + if err != nil { + t.Fatal(err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "index.json"), raw, 0600); err != nil { + t.Fatal(err) + } + older, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := older.Close(); err != nil { + t.Error(err) + } + }() + legacyObject, err := older.HeadObject(t.Context(), "bucket", "key") + if err != nil || !legacyObject.Modified.IsZero() { + t.Fatal("legacy object was refused or assigned a fabricated date:", legacyObject, err) + } + unchanged, err := os.ReadFile(filepath.Join(path, "index.json")) + if err != nil || string(unchanged) != string(raw) { + t.Fatal("reading version 2 rewrote the index:", err) + } +} + +func TestVerifiedObjectReaderSeeksAndCancelsWithoutExposingFileMethods(t *testing.T) { + s, _ := newStore(t) + if err := s.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + mustPut(t, s, "bucket", "key", "abcdef") + ctx, cancel := context.WithCancel(t.Context()) + _, body, err := s.GetObject(ctx, "bucket", "key") + if err != nil { + t.Fatal(err) + } + defer func() { + if err := body.Close(); err != nil { + t.Error(err) + } + }() + if _, err := body.Seek(3, io.SeekStart); err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(body) + if err != nil || string(raw) != "def" { + t.Fatal(string(raw), err) + } + cancel() + if _, err := body.Seek(0, io.SeekStart); !errors.Is(err, context.Canceled) { + t.Fatal("seek ignored cancellation:", err) + } +} diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index b2401f1d..1863bf17 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -17,6 +17,7 @@ import ( "os" "sort" "sync" + "time" "unicode/utf8" "github.com/stephrobert/feint/internal/core/serialise" @@ -42,6 +43,8 @@ type Object struct { Size int64 `json:"size"` ContentType string `json:"content_type"` Metadata map[string]string `json:"metadata"` + // Modified is the persisted publication time, zero for legacy references. + Modified time.Time `json:"modified,omitzero"` } // PutOptions declares the exact decoded byte count and application metadata. @@ -242,6 +245,7 @@ func (s *Store) PutObject(ctx context.Context, name, key string, body io.Reader, if b.Generation != generation { return Object{}, fmt.Errorf("%w: bucket was recreated during upload", ErrBucketNotFound) } + obj.Modified = time.Now().UTC() next := s.clone() next.Buckets[name].Objects[key] = obj if err := s.commit(ctx, next); err != nil { @@ -251,9 +255,9 @@ func (s *Store) PutObject(ctx context.Context, name, key string, body io.Reader, } // GetObject verifies the complete blob's size and digest before returning a -// reader positioned at byte zero. It never returns missing or corrupt bytes +// seekable reader positioned at byte zero. It never returns missing or corrupt bytes // as a successful empty object. The caller must close the reader. -func (s *Store) GetObject(ctx context.Context, name, key string) (Object, io.ReadCloser, error) { +func (s *Store) GetObject(ctx context.Context, name, key string) (Object, io.ReadSeekCloser, error) { if err := validKey(key); err != nil { return Object{}, nil, err } @@ -358,7 +362,7 @@ func (s *Store) lookup(name string) (*bucket, error) { } func (s *Store) clone() manifest { - next := manifest{Format: s.state.Format, Version: 2, Buckets: make(map[string]*bucket)} + next := manifest{Format: s.state.Format, Version: storeVersion, Buckets: make(map[string]*bucket)} for name, b := range s.state.Buckets { next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects), Metadata: maps.Clone(b.Metadata)} } diff --git a/internal/core/objectstore/store_test.go b/internal/core/objectstore/store_test.go index fcdfffd7..53b66f9b 100644 --- a/internal/core/objectstore/store_test.go +++ b/internal/core/objectstore/store_test.go @@ -753,7 +753,7 @@ func TestManifestIsClosedVersionedAndValidated(t *testing.T) { } for _, body := range []string{ `null`, `{}`, `{"format":"operator","version":1,"buckets":{}}`, - `{"format":"feint-objectstore","version":3,"buckets":{}}`, + `{"format":"feint-objectstore","version":4,"buckets":{}}`, `{"format":"feint-objectstore","version":1,"buckets":null}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":null}}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"","objects":{}}}}`, @@ -820,7 +820,7 @@ func TestFailedOpenReleasesOwnershipLock(t *testing.T) { if err != nil { t.Fatal(err) } - if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":3,"buckets":{}}`), 0o600); err != nil { + if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":4,"buckets":{}}`), 0o600); err != nil { t.Fatal(err) } if broken, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || broken != nil { diff --git a/internal/s3/list.go b/internal/s3/list.go new file mode 100644 index 00000000..209591d1 --- /dev/null +++ b/internal/s3/list.go @@ -0,0 +1,232 @@ +package s3 + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type listOptions struct { + prefix, delimiter, encoding, token, after string + max int +} + +func listQueryKey(key string) bool { + switch key { + case "list-type", "prefix", "delimiter", "max-keys", "continuation-token", "start-after", "encoding-type", "fetch-owner", "x-id": + return true + } + return false +} + +func parseListOptions(q url.Values) (listOptions, error) { + options := listOptions{prefix: q.Get("prefix"), delimiter: q.Get("delimiter"), + encoding: q.Get("encoding-type"), token: q.Get("continuation-token"), after: q.Get("start-after"), max: 1000} + for key, values := range q { + if !listQueryKey(key) || len(values) != 1 { + return options, fmt.Errorf("unknown or duplicate listing parameter %q", key) + } + } + if q.Get("list-type") != "2" || (options.encoding != "" && options.encoding != "url") || + (q.Has("fetch-owner") && q.Get("fetch-owner") != "false") || (options.token != "" && options.after != "") { + return options, errors.New("only ListObjectsV2 without owner requests is supported") + } + for _, value := range []string{options.prefix, options.delimiter, options.after} { + if len(value) > 1024 || !utf8.ValidString(value) { + return options, errors.New("listing key parameters must be bounded UTF-8") + } + } + if q.Has("max-keys") { + n, err := strconv.Atoi(q.Get("max-keys")) + if err != nil || n < 0 || n > 1000 { + return options, errors.New("max-keys must be between 0 and 1000") + } + options.max = n + } + if len(options.token) > 4096 { + return options, errors.New("invalid continuation token length") + } + return options, nil +} + +type listCursor struct { + Scope string `json:"scope"` + After string `json:"after"` +} + +func (s *Server) listScope(bucket string, options listOptions) string { + // A cursor is scope-bound, not signed or authenticated. It is not IAM. + sum := sha256.Sum256(fmt.Appendf(nil, "%q/%q/%q", s.name(bucket), options.prefix, options.delimiter)) + return hex.EncodeToString(sum[:]) +} + +func (s *Server) listAfter(bucket string, options listOptions) (string, error) { + if options.token == "" { + return options.after, nil + } + raw, err := base64.RawURLEncoding.DecodeString(options.token) + if err != nil || !utf8.Valid(raw) { + return "", fmt.Errorf("decode listing cursor: %w", errors.Join(err, errors.New("cursor must be base64url UTF-8 JSON"))) + } + var cursor listCursor + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cursor); err != nil { + return "", fmt.Errorf("decode listing cursor: %w", err) + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return "", errors.New("trailing listing cursor data") + } + if cursor.Scope != s.listScope(bucket, options) || cursor.After == "" || + len(cursor.After) > 1024 || !utf8.ValidString(cursor.After) { + return "", errors.New("listing cursor does not match the declared scope and prefix") + } + return cursor.After, nil +} + +type listedObject struct { + Key string `xml:"Key"` + Modified string `xml:"LastModified"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type listedPrefix struct { + Prefix string `xml:"Prefix"` +} + +type listResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Namespace string `xml:"xmlns,attr"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Delimiter string `xml:"Delimiter,omitempty"` + MaxKeys int `xml:"MaxKeys"` + KeyCount int `xml:"KeyCount"` + Truncated bool `xml:"IsTruncated"` + Encoding string `xml:"EncodingType,omitempty"` + Token string `xml:"ContinuationToken,omitempty"` + Next string `xml:"NextContinuationToken,omitempty"` + After string `xml:"StartAfter,omitempty"` + Objects []listedObject `xml:"Contents"` + Prefixes []listedPrefix `xml:"CommonPrefixes"` +} + +func listText(value, encoding string) (string, error) { + if encoding == "url" { + return strings.ReplaceAll(url.QueryEscape(value), "+", "%20"), nil + } + for _, c := range value { + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') || c == 0xfffe || c == 0xffff { + return "", errors.New("key needs encoding-type=url to be represented in XML") + } + } + return value, nil +} + +func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, bucket string, q url.Values) { + options, err := parseListOptions(q) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid S3 listing", err) + return + } + after, err := s.listAfter(bucket, options) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid S3 listing cursor", err) + return + } + objects, err := s.store.ListObjects(r.Context(), s.name(bucket)) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + entries := make(map[string]bool) + for key := range objects { + if !strings.HasPrefix(key, options.prefix) { + continue + } + name, prefix := key, false + if options.delimiter != "" { + if i := strings.Index(strings.TrimPrefix(key, options.prefix), options.delimiter); i >= 0 { + name, prefix = key[:len(options.prefix)+i+len(options.delimiter)], true + } + } + if name > after { + entries[name] = prefix + } + } + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + result := listResult{Namespace: s3XMLNamespace, Name: bucket, MaxKeys: options.max, + Encoding: options.encoding, Token: options.token, After: options.after, + Truncated: options.max > 0 && len(names) > options.max} + result.Prefix, err = listText(options.prefix, options.encoding) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "S3 listing prefix cannot be encoded", err) + return + } + result.Delimiter, err = listText(options.delimiter, options.encoding) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "S3 listing delimiter cannot be encoded", err) + return + } + result.After, err = listText(options.after, options.encoding) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "S3 listing start-after cannot be encoded", err) + return + } + for _, name := range names[:min(len(names), options.max)] { + text, err := listText(name, options.encoding) + if err != nil { + s.refuse(w, r, http.StatusNotImplemented, "S3 listing key cannot be encoded without encoding-type=url", err) + return + } + result.KeyCount++ + if entries[name] { + result.Prefixes = append(result.Prefixes, listedPrefix{Prefix: text}) + continue + } + obj := objects[name] + if obj.Modified.IsZero() { + s.refuse(w, r, http.StatusNotImplemented, "legacy object has no persisted modification date; overwrite it before listing", nil) + return + } + result.Objects = append(result.Objects, listedObject{Key: text, + Modified: obj.Modified.UTC().Format(time.RFC3339Nano), Size: obj.Size, StorageClass: "STANDARD"}) + } + if result.Truncated { + raw, err := json.Marshal(listCursor{Scope: s.listScope(bucket, options), After: names[options.max-1]}) + if err != nil { + s.refuse(w, r, http.StatusInternalServerError, "S3 listing cursor encoding failed", err) + return + } + result.Next = base64.RawURLEncoding.EncodeToString(raw) + } + raw, err := xml.Marshal(result) + if err != nil { + s.refuse(w, r, http.StatusInternalServerError, "S3 listing encoding failed", err) + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(raw); err != nil { + s.cfg.Log.ErrorContext(r.Context(), "S3 listing write failed", "error", err) + } +} diff --git a/internal/s3/list_test.go b/internal/s3/list_test.go new file mode 100644 index 00000000..f6f815d0 --- /dev/null +++ b/internal/s3/list_test.go @@ -0,0 +1,224 @@ +package s3 + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "encoding/xml" + "io" + "log/slog" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +func decodeListing(t *testing.T, s *Server, query string) listResult { + t.Helper() + w := request(t, s, "GET", "/bucket?"+query, nil, nil) + var result listResult + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &result) != nil { + t.Fatalf("listing failed: status=%d body=%q", w.Code, w.Body) + } + if bytes.Contains(w.Body.Bytes(), []byte("")) { + t.Fatal("listing invented an ETag") + } + return result +} + +func TestListingPagesContainActualSortedObjectsAndPrefixes(t *testing.T) { + s, store := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + for _, key := range []string{"z", "dir/two", "a", "dir/one"} { + if w := request(t, s, "PUT", "/bucket/"+key, []byte(key), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body) + } + } + first := decodeListing(t, s, "list-type=2&delimiter=/&max-keys=1") + if first.KeyCount != 1 || len(first.Objects) != 1 || first.Objects[0].Key != "a" || + first.Objects[0].Size != 1 || !first.Truncated || first.Next == "" { + t.Fatal("first page did not reflect actual objects:", first) + } + modified, err := time.Parse(time.RFC3339Nano, first.Objects[0].Modified) + obj, readErr := store.HeadObject(t.Context(), "bucket", "a") + if err != nil || readErr != nil || !modified.Equal(obj.Modified) { + t.Fatal("listing invented or lost its persisted modification time:", err, readErr) + } + second := decodeListing(t, s, "list-type=2&delimiter=/&max-keys=1&continuation-token="+url.QueryEscape(first.Next)) + if second.KeyCount != 1 || len(second.Objects) != 0 || len(second.Prefixes) != 1 || + second.Prefixes[0].Prefix != "dir/" || !second.Truncated || second.Next == "" || second.Next == first.Next { + t.Fatal("prefix grouping consumed the wrong page or cursor:", second) + } + third := decodeListing(t, s, "list-type=2&delimiter=/&max-keys=1&continuation-token="+url.QueryEscape(second.Next)) + if third.KeyCount != 1 || len(third.Objects) != 1 || third.Objects[0].Key != "z" || third.Truncated || third.Next != "" { + t.Fatal("final page repeated or fabricated an entry:", third) + } + filtered := decodeListing(t, s, "list-type=2&prefix=dir/&start-after=dir/one") + if len(filtered.Objects) != 1 || filtered.Objects[0].Key != "dir/two" { + t.Fatal("prefix and start-after were not applied literally:", filtered) + } + zero := decodeListing(t, s, "list-type=2&max-keys=0") + if zero.KeyCount != 0 || zero.Truncated || zero.Next != "" || len(zero.Objects) != 0 || len(zero.Prefixes) != 0 { + t.Fatal("zero limit fabricated a page or a non-progressing cursor:", zero) + } +} + +func TestListingRejectsInvalidQueriesAndCrossScopeCursors(t *testing.T) { + s, _ := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + request(t, s, "PUT", "/bucket/a", nil, nil) + request(t, s, "PUT", "/bucket/b", nil, nil) + first := decodeListing(t, s, "list-type=2&max-keys=1") + for _, query := range []string{ + "list-type=2&max-keys=-1", "list-type=2&max-keys=1001", "list-type=2&max-keys=no", + "list-type=2&prefix=a&prefix=b", "list-type=2&fetch-owner=true", + "list-type=2&encoding-type=unknown", + "list-type=2&continuation-token=invalid", + "list-type=2&prefix=a&continuation-token=" + url.QueryEscape(first.Next), + "list-type=2&delimiter=/&continuation-token=" + url.QueryEscape(first.Next), + "list-type=2&start-after=a&continuation-token=" + url.QueryEscape(first.Next), + "list-type=2&prefix=" + url.QueryEscape(strings.Repeat("a", 1025)), + "list-type=2&prefix=" + url.QueryEscape(string([]byte{255})), + "list-type=2&continuation-token=" + strings.Repeat("a", 4097), + } { + if w := request(t, s, "GET", "/bucket?"+query, nil, nil); w.Code != 400 { + t.Fatal("invalid listing silently fell back to another query:", query, w.Code, w.Body) + } + } + raw, err := base64.RawURLEncoding.DecodeString(first.Next) + if err != nil { + t.Fatal(err) + } + for _, body := range []string{string(raw[:len(raw)-1]) + `,"unknown":true}`, string(raw) + ` {}`, + string(bytes.Replace(raw, []byte(`"after":"a"`), []byte("\"after\":\"\xff\""), 1))} { + token := base64.RawURLEncoding.EncodeToString([]byte(body)) + if w := request(t, s, "GET", "/bucket?list-type=2&continuation-token="+token, nil, nil); w.Code != 400 { + t.Fatal("non-closed cursor admitted:", w.Code, w.Body) + } + } + var cursor listCursor + if err := json.Unmarshal(raw, &cursor); err != nil { + t.Fatal(err) + } + for _, after := range []string{"", strings.Repeat("a", 1025)} { + cursor.After = after + body, err := json.Marshal(cursor) + if err != nil { + t.Fatal(err) + } + token := base64.RawURLEncoding.EncodeToString(body) + if w := request(t, s, "GET", "/bucket?list-type=2&continuation-token="+token, nil, nil); w.Code != 400 { + t.Fatal("unbounded or empty cursor boundary admitted:", w.Code) + } + } + if w := request(t, s, "GET", "/other?list-type=2&continuation-token="+url.QueryEscape(first.Next), nil, nil); w.Code != 400 { + t.Fatal("cursor crossed a bucket boundary:", w.Code) + } + s.cfg.Namespace = "other-region" + if w := request(t, s, "GET", "/bucket?list-type=2&continuation-token="+url.QueryEscape(first.Next), nil, nil); w.Code != 400 { + t.Fatal("cursor crossed a deployment namespace:", w.Code) + } +} + +func TestListingRefusesUndatedLegacyObjectsWithoutBreakingTheirBytes(t *testing.T) { + parent := t.TempDir() + if err := os.Chmod(parent, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(parent, "objects") + store, err := objectstore.Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + obj, err := store.PutObject(t.Context(), "bucket", "key", strings.NewReader("bytes"), objectstore.PutOptions{Size: 5}) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(path, "index.json")) + if err != nil { + t.Fatal(err) + } + date, err := json.Marshal(obj.Modified) + if err != nil { + t.Fatal(err) + } + field := append([]byte(`,"modified":`), date...) + if bytes.Count(raw, field) != 1 { + t.Fatal("the actual producer did not persist the expected modification field") + } + if err := os.WriteFile(filepath.Join(path, "index.json"), bytes.Replace(raw, field, nil, 1), 0600); err != nil { + t.Fatal(err) + } + reopened, err := objectstore.Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }) + s, err := New(reopened, Config{Region: "fr-par", MaxObjectBytes: 1 << 20, + MissingBucket: ExoscaleMissingBucket, Log: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatal(err) + } + if w := request(t, s, "GET", "/bucket?list-type=2", nil, nil); w.Code != 501 || + !strings.Contains(w.Body.String(), "no persisted modification date") { + t.Fatal("legacy listing fabricated a modification date:", w.Code, w.Body) + } + if w := request(t, s, "GET", "/bucket/key", nil, nil); w.Code != 200 || + w.Body.String() != "bytes" || w.Header().Get("Last-Modified") != "" { + t.Fatal("legacy byte reads were refused or assigned an invented date:", w.Code, w.Body) + } + if w := request(t, s, "PUT", "/bucket/key", []byte("new"), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body) + } + result := decodeListing(t, s, "list-type=2") + if len(result.Objects) != 1 || result.Objects[0].Size != 3 || result.Objects[0].Modified == "" { + t.Fatal("overwriting a legacy object did not restore truthful listing:", result) + } +} + +func TestListingEmptyContinuationTokenStartsAtActualInventory(t *testing.T) { + s, _ := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + request(t, s, "PUT", "/bucket/key", []byte("content"), nil) + result := decodeListing(t, s, "list-type=2&continuation-token=") + if len(result.Objects) != 1 || result.Objects[0].Key != "key" || result.Objects[0].Size != 7 || result.Truncated { + t.Fatal("empty optional token did not yield the actual first page:", result) + } +} + +func TestListingPreservesEncodedKeysAndRefusesLossyXML(t *testing.T) { + s, _ := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + for _, key := range []string{"folder//a/../%2F +?#", "control\x00key"} { + if w := request(t, s, "PUT", "/bucket/"+url.PathEscape(key), []byte("bytes"), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body) + } + } + if w := request(t, s, "GET", "/bucket?list-type=2", nil, nil); w.Code != 501 { + t.Fatal("XML silently replaced a logical control character:", w.Code, w.Body) + } + result := decodeListing(t, s, "list-type=2&encoding-type=url") + if len(result.Objects) != 2 || result.Encoding != "url" { + t.Fatal("encoded listing lost actual keys:", result) + } + for i, expected := range []string{"control\x00key", "folder//a/../%2F +?#"} { + key, err := url.PathUnescape(result.Objects[i].Key) + if err != nil || key != expected { + t.Fatal("key was cleaned or decoded twice:", key, expected, err) + } + } +} diff --git a/internal/s3/range.go b/internal/s3/range.go new file mode 100644 index 00000000..2aab7fcd --- /dev/null +++ b/internal/s3/range.go @@ -0,0 +1,58 @@ +package s3 + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +var ( + errRangeUnsatisfiable = errors.New("range does not overlap the object") + errRangeUnsupported = errors.New("only one bytes range is supported") +) + +func byteRange(value string, size int64) (offset, length int64, err error) { + if !strings.HasPrefix(value, "bytes=") || strings.Contains(value, ",") { + return 0, 0, errRangeUnsupported + } + first, last, ok := strings.Cut(strings.TrimPrefix(value, "bytes="), "-") + if !ok || (first == "" && last == "") { + return 0, 0, errors.New("invalid byte range") + } + number := func(s string) (int64, error) { + for _, c := range s { + if c < '0' || c > '9' { + return 0, errors.New("range bounds must be decimal digits") + } + } + return strconv.ParseInt(s, 10, 64) + } + if first == "" { + n, err := number(last) + if err != nil { + return 0, 0, fmt.Errorf("suffix range: %w", err) + } + if n == 0 || size == 0 { + return 0, 0, errRangeUnsatisfiable + } + n = min(n, size) + return size - n, n, nil + } + start, err := number(first) + if err != nil { + return 0, 0, fmt.Errorf("range start: %w", err) + } + end := size - 1 + if last != "" { + end, err = number(last) + if err != nil || end < start { + return 0, 0, fmt.Errorf("invalid range end: %w", errors.Join(err, errors.New("end must not precede start"))) + } + } + if start >= size { + return 0, 0, errRangeUnsatisfiable + } + end = min(end, size-1) + return start, end - start + 1, nil +} diff --git a/internal/s3/range_test.go b/internal/s3/range_test.go new file mode 100644 index 00000000..f0cceef0 --- /dev/null +++ b/internal/s3/range_test.go @@ -0,0 +1,105 @@ +package s3 + +import ( + "bytes" + "net/http/httptest" + "strconv" + "testing" +) + +func TestSingleByteRangesReturnOnlyTheirVerifiedBytes(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + payload := []byte{0, 255, 13, 10, 1, 128} + if w := request(t, s, "PUT", "/bucket/key", payload, nil); w.Code != 200 { + t.Fatal(w.Code, w.Body) + } + for _, tc := range []struct { + value, contentRange string + start, end int + }{ + {"bytes=1-3", "bytes 1-3/6", 1, 4}, + {"bytes=4-", "bytes 4-5/6", 4, 6}, + {"bytes=-2", "bytes 4-5/6", 4, 6}, + {"bytes=0-99", "bytes 0-5/6", 0, 6}, + {"bytes=-99", "bytes 0-5/6", 0, 6}, + } { + w := request(t, s, "GET", "/bucket/key", nil, map[string]string{"Range": tc.value, "X-Amz-Checksum-Mode": "ENABLED"}) + if w.Code != 206 || !bytes.Equal(w.Body.Bytes(), payload[tc.start:tc.end]) || + w.Header().Get("Content-Range") != tc.contentRange || w.Header().Get("X-Amz-Checksum-Sha256") != "" || + w.Header().Get("Content-Length") != strconv.Itoa(tc.end-tc.start) || w.Header().Get("Last-Modified") == "" { + t.Fatalf("range %s: status=%d body=%v headers=%v", tc.value, w.Code, w.Body.Bytes(), w.Header()) + } + } + if w := request(t, s, "GET", "/bucket/key", nil, nil); w.Code != 200 || + !bytes.Equal(w.Body.Bytes(), payload) || w.Header().Get("X-Amz-Checksum-Sha256") == "" { + t.Fatal("range support weakened the full-object checksum proof:", w.Code, w.Header()) + } +} + +func TestRangeRefusalsAreExplicitAndDoNotMasqueradeAsFullReads(t *testing.T) { + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + request(t, s, "PUT", "/bucket/key", []byte("123456"), nil) + request(t, s, "PUT", "/bucket/empty", nil, nil) + for _, tc := range []struct { + key, value string + status int + }{ + {"key", "bytes=6-", 416}, + {"empty", "bytes=0-1", 416}, + {"key", "bytes=-0", 416}, + {"key", "bytes=3-2", 400}, + {"key", "bytes=+1-2", 400}, + {"key", "bytes=1", 400}, + {"key", "bytes=999999999999999999999-", 400}, + {"key", "items=0-1", 501}, + {"key", "bytes=0-1,3-4", 501}, + } { + w := request(t, s, "GET", "/bucket/"+tc.key, nil, map[string]string{"Range": tc.value}) + if w.Code != tc.status || w.Header().Get("X-Amz-Checksum-Sha256") != "" { + t.Fatalf("range refusal %s %s: status=%d want=%d headers=%v", tc.key, tc.value, w.Code, tc.status, w.Header()) + } + if tc.status == 416 { + size := "6" + if tc.key == "empty" { + size = "0" + } + if w.Header().Get("Content-Range") != "bytes */"+size { + t.Fatal("unsatisfiable range did not state its actual size") + } + } + } + for _, method := range []string{"HEAD", "PUT", "DELETE"} { + if w := request(t, s, method, "/bucket/key", nil, map[string]string{"Range": "bytes=0-1"}); w.Code != 501 { + t.Fatal("Range leaked into another operation:", method, w.Code) + } + } + r := httptest.NewRequest("GET", "http://localhost/bucket/key", nil) + r.RemoteAddr = "127.0.0.1:1" + r.Header["Range"] = []string{"bytes=0-1", "bytes=2-3"} + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if w.Code != 400 { + t.Fatal("ambiguous Range headers were accepted:", w.Code) + } +} + +func FuzzByteRangeIsBounded(f *testing.F) { + for _, value := range []string{"bytes=0-1", "bytes=-3", "bytes=8-", "bytes=0-9999999999999999999", "items=0-1", "bytes=2-1"} { + f.Add(value, int64(9)) + } + f.Fuzz(func(t *testing.T, value string, size int64) { + if size < 0 { + return + } + start, length, err := byteRange(value, size) + if err == nil && (start < 0 || length <= 0 || start >= size || length > size-start) { + t.Fatalf("unbounded range: %q size=%d start=%d length=%d", value, size, start, length) + } + }) +} diff --git a/internal/s3/server.go b/internal/s3/server.go index b583f79c..542f3697 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -78,7 +78,8 @@ func New(store *objectstore.Store, cfg Config) (*Server, error) { return &Server{store: store, cfg: cfg}, nil } -// ServeHTTP implements bucket PUT/HEAD/DELETE and object PUT/GET/HEAD/DELETE. +// ServeHTTP implements bucket PUT/HEAD/DELETE, ListObjectsV2 and object +// PUT/GET/HEAD/DELETE, including single-byte-range GETs. // Other features get an explicit Feint refusal, never a successful empty stub // or an invented cloud XML error. Signatures and credentials are never logged. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -106,13 +107,17 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.refuse(w, r, http.StatusBadRequest, "invalid query", err) return } + rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") + listing := r.Method == http.MethodGet && rawKey == "" && query.Get("list-type") == "2" for key := range query { + if listing && listQueryKey(key) { + continue + } if key != "x-id" && (!s.cfg.BucketConfiguration || (key != "versioning" && key != "acl" && key != "tagging")) { s.refuse(w, r, http.StatusNotImplemented, "S3 query features are not implemented", nil) return } } - rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") bucket, bucketErr := url.PathUnescape(rawBucket) key, keyErr := url.PathUnescape(rawKey) checked, err := s.bucketHeaders(r, query, rawKey == "") @@ -133,6 +138,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.refuse(w, r, http.StatusBadRequest, "invalid bucket or object key", errors.Join(bucketErr, keyErr)) return } + if rawKey == "" && len(r.Header.Values("Range")) != 0 { + s.refuse(w, r, http.StatusNotImplemented, "Range requires an object GET", nil) + return + } + if listing { + s.listObjects(w, r, bucket, query) + return + } if key == "" { if len(query) != 0 && (len(query) != 1 || !query.Has("x-id")) { s.bucketConfiguration(w, r, bucket, query) @@ -264,6 +277,31 @@ func (s *Server) get(w http.ResponseWriter, r *http.Request, bucket, key string) s.refuse(w, r, http.StatusInternalServerError, "stored metadata cannot be represented as HTTP headers", nil) return } + status, length := http.StatusOK, obj.Size + if values := r.Header.Values("Range"); len(values) != 0 { + if len(values) != 1 { + s.refuse(w, r, http.StatusBadRequest, "ambiguous Range header", nil) + return + } + offset, count, rangeErr := byteRange(values[0], obj.Size) + if rangeErr != nil { + code := http.StatusBadRequest + if errors.Is(rangeErr, errRangeUnsatisfiable) { + code = http.StatusRequestedRangeNotSatisfiable + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", obj.Size)) + } else if errors.Is(rangeErr, errRangeUnsupported) { + code = http.StatusNotImplemented + } + s.refuse(w, r, code, "S3 byte range refused", rangeErr) + return + } + if _, err := body.Seek(offset, io.SeekStart); err != nil { + s.refuse(w, r, http.StatusInternalServerError, "verified object range seek failed", err) + return + } + status, length = http.StatusPartialContent, count + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, offset+count-1, obj.Size)) + } seen := make(map[string]bool) for key, value := range obj.Metadata { canonical := strings.ToLower(key) @@ -273,17 +311,21 @@ func (s *Server) get(w http.ResponseWriter, r *http.Request, bucket, key string) } seen[canonical] = true } - if !s.digestHeader(w, r, obj) { + if status == http.StatusOK && !s.digestHeader(w, r, obj) { return } - w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) + w.Header().Set("Accept-Ranges", "bytes") + if !obj.Modified.IsZero() { + w.Header().Set("Last-Modified", obj.Modified.UTC().Format(http.TimeFormat)) + } + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) w.Header().Set("Content-Type", obj.ContentType) for key, value := range obj.Metadata { w.Header().Set("X-Amz-Meta-"+key, value) } - w.WriteHeader(http.StatusOK) + w.WriteHeader(status) if r.Method != http.MethodHead { - if _, err := io.Copy(w, body); err != nil { + if _, err := io.CopyN(w, body, length); err != nil { s.cfg.Log.ErrorContext(r.Context(), "S3 download failed after headers", "error", err) } } @@ -420,7 +462,11 @@ func supportedHeaders(r *http.Request) error { for name := range r.Header { lower := strings.ToLower(name) switch lower { - case "range", "if-match", "if-none-match", "if-modified-since", "if-unmodified-since", "if-range", "content-md5": + case "range": + if r.Method != http.MethodGet { + return fmt.Errorf("range requires GET") + } + case "if-match", "if-none-match", "if-modified-since", "if-unmodified-since", "if-range", "content-md5": return fmt.Errorf("unsupported header %s", name) } if !strings.HasPrefix(lower, "x-amz-") || strings.HasPrefix(lower, "x-amz-meta-") { diff --git a/internal/s3/server_test.go b/internal/s3/server_test.go index 4cfeb497..8cf7667e 100644 --- a/internal/s3/server_test.go +++ b/internal/s3/server_test.go @@ -190,12 +190,12 @@ func TestRefusalsDoNotMasqueradeAsCloudSupport(t *testing.T) { method, path string headers map[string]string }{ - {"GET", "/bucket?list-type=2", nil}, + {"GET", "/bucket?list-type=1", nil}, {"GET", "/bucket/missing", nil}, {"PUT", "/bucket", nil}, {"PUT", "/bucket/key?uploads", nil}, {"PUT", "/bucket/key", map[string]string{"X-Amz-Acl": "public-read"}}, - {"GET", "/bucket/key", map[string]string{"Range": "bytes=0-1"}}, + {"GET", "/bucket/key", map[string]string{"Range": "items=0-1"}}, {"PUT", "/bucket/key", map[string]string{"If-None-Match": "*"}}, } { w := request(t, s, tc.method, tc.path, nil, tc.headers) diff --git a/mise.toml b/mise.toml index cae9b5d2..a6d04f3b 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove prefixed AWS/scw bytes and host-routed exo 1.95.1 upload through real start/stop/restart" +description = "Prove AWS/scw bytes, paginated listing and host-routed exo 1.95.1 upload/range download across actual restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:exo:endpoint"] diff --git a/tools/falsify/specs/neutral-object-bytes.json b/tools/falsify/specs/neutral-object-bytes.json index ed4bac0d..66fb9eca 100644 --- a/tools/falsify/specs/neutral-object-bytes.json +++ b/tools/falsify/specs/neutral-object-bytes.json @@ -4,8 +4,8 @@ { "label": "a returned reader lets io.Copy continue after cancellation", "file": "internal/core/objectstore/files.go", - "find": "if err := r.ctx.Err(); err != nil {", - "replace": "if err := r.ctx.Err(); err != nil && false {", + "find": "func (r *contextReader) Read(p []byte) (int, error) {\n\tif err := r.ctx.Err(); err != nil {", + "replace": "func (r *contextReader) Read(p []byte) (int, error) {\n\tif err := r.ctx.Err(); err != nil && false {", "test": "TestReturnedReaderHonorsCancellationThroughCopy" }, { @@ -88,8 +88,8 @@ { "label": "a future metadata version is silently accepted", "file": "internal/core/objectstore/manifest.go", - "find": "if (state.Version != 1 && state.Version != 2) || state.Buckets == nil {", - "replace": "if ((state.Version != 1 && state.Version != 2) && false) || state.Buckets == nil {", + "find": "if state.Version < 1 || state.Version > storeVersion || state.Buckets == nil {", + "replace": "if ((state.Version < 1 || state.Version > storeVersion) && false) || state.Buckets == nil {", "test": "TestManifestIsClosedVersionedAndValidated" }, { diff --git a/tools/falsify/specs/s3-download.json b/tools/falsify/specs/s3-download.json new file mode 100644 index 00000000..8009f65f --- /dev/null +++ b/tools/falsify/specs/s3-download.json @@ -0,0 +1,184 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "empty first-page tokens from the real Exo downloader are refused", + "file": "internal/s3/list.go", + "find": "if len(options.token) > 4096 {", + "replace": "if len(options.token) > 4096 || (q.Has(\"continuation-token\") && options.token == \"\") {", + "test": "TestListingEmptyContinuationTokenStartsAtActualInventory" + }, + { + "label": "listing options silently accept duplicates", + "file": "internal/s3/list.go", + "find": "!listQueryKey(key) || len(values) != 1", + "replace": "!listQueryKey(key) || (len(values) != 1 && false)", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "unimplemented owner and encoding options masquerade as supported", + "file": "internal/s3/list.go", + "find": "(q.Has(\"fetch-owner\") && q.Get(\"fetch-owner\") != \"false\")", + "replace": "(q.Has(\"fetch-owner\") && q.Get(\"fetch-owner\") != \"false\" && false)", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "listing key parameters exceed their UTF-8 byte bound", + "file": "internal/s3/list.go", + "find": "if len(value) > 1024 || !utf8.ValidString(value) {", + "replace": "if (len(value) > 1024 || !utf8.ValidString(value)) && false {", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "listing exceeds its maximum page bound", + "file": "internal/s3/list.go", + "find": "n > 1000", + "replace": "(n > 1000 && false)", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "listing cursors cross scope or prefix boundaries", + "file": "internal/s3/list.go", + "find": "cursor.Scope != s.listScope(bucket, options)", + "replace": "(cursor.Scope != s.listScope(bucket, options) && false)", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "cursor JSON silently accepts unknown fields", + "file": "internal/s3/list.go", + "find": "decoder.DisallowUnknownFields()", + "replace": "decoder.InputOffset()", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "cursor JSON accepts trailing documents", + "file": "internal/s3/list.go", + "find": "if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {", + "replace": "if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) && false {", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "cursor decoding silently replaces invalid UTF-8", + "file": "internal/s3/list.go", + "find": "if err != nil || !utf8.Valid(raw) {", + "replace": "if err != nil || (!utf8.Valid(raw) && false) {", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "cursor boundaries need not be nonempty and bounded", + "file": "internal/s3/list.go", + "find": "cursor.After == \"\" ||\n\t\tlen(cursor.After) > 1024", + "replace": "(cursor.After == \"\" && false) ||\n\t\t(len(cursor.After) > 1024 && false)", + "test": "TestListingRejectsInvalidQueriesAndCrossScopeCursors" + }, + { + "label": "listing ignores its literal prefix", + "file": "internal/s3/list.go", + "find": "if !strings.HasPrefix(key, options.prefix) {", + "replace": "if !strings.HasPrefix(key, options.prefix) && false {", + "test": "TestListingPagesContainActualSortedObjectsAndPrefixes" + }, + { + "label": "listing cursor fails to advance", + "file": "internal/s3/list.go", + "find": "if name > after {", + "replace": "if name > after || true {", + "test": "TestListingPagesContainActualSortedObjectsAndPrefixes" + }, + { + "label": "delimiter grouping is lost", + "file": "internal/s3/list.go", + "find": "if options.delimiter != \"\" {", + "replace": "if options.delimiter != \"\" && false {", + "test": "TestListingPagesContainActualSortedObjectsAndPrefixes" + }, + { + "label": "literal percent escapes are decoded twice", + "file": "internal/s3/list.go", + "find": "url.QueryEscape(value)", + "replace": "url.QueryEscape(strings.ReplaceAll(value, \"%2F\", \"/\"))", + "test": "TestListingPreservesEncodedKeysAndRefusesLossyXML" + }, + { + "label": "XML silently replaces unsupported key characters", + "file": "internal/s3/list.go", + "find": "(c < 0x20 && c != '\\t' && c != '\\n' && c != '\\r') || c == 0xfffe || c == 0xffff", + "replace": "((c < 0x20 && c != '\\t' && c != '\\n' && c != '\\r') || c == 0xfffe || c == 0xffff) && false", + "test": "TestListingPreservesEncodedKeysAndRefusesLossyXML" + }, + { + "label": "undated legacy objects are listed with invented dates", + "file": "internal/s3/list.go", + "find": "if obj.Modified.IsZero() {", + "replace": "if obj.Modified.IsZero() && false {", + "test": "TestListingRefusesUndatedLegacyObjectsWithoutBreakingTheirBytes" + }, + { + "label": "range bytes are read from the beginning rather than the actual offset", + "file": "internal/s3/server.go", + "find": "body.Seek(offset, io.SeekStart)", + "replace": "body.Seek(offset*0, io.SeekStart)", + "test": "TestSingleByteRangesReturnOnlyTheirVerifiedBytes" + }, + { + "label": "range Content-Length describes bytes not sent", + "file": "internal/s3/server.go", + "find": "strconv.FormatInt(length, 10)", + "replace": "strconv.FormatInt(length+1, 10)", + "test": "TestSingleByteRangesReturnOnlyTheirVerifiedBytes" + }, + { + "label": "whole-object checksums are attached to partial bytes", + "file": "internal/s3/server.go", + "find": "status == http.StatusOK && !s.digestHeader(w, r, obj)", + "replace": "(status == http.StatusOK || true) && !s.digestHeader(w, r, obj)", + "test": "TestSingleByteRangesReturnOnlyTheirVerifiedBytes" + }, + { + "label": "range ends exceed the actual blob size", + "file": "internal/s3/range.go", + "find": "end = min(end, size-1)", + "replace": "end = min(end, size+100)", + "test": "TestSingleByteRangesReturnOnlyTheirVerifiedBytes" + }, + { + "label": "multiple ranges and unsupported units become other responses", + "file": "internal/s3/range.go", + "find": "if !strings.HasPrefix(value, \"bytes=\") || strings.Contains(value, \",\") {", + "replace": "if (!strings.HasPrefix(value, \"bytes=\") || strings.Contains(value, \",\")) && false {", + "test": "TestRangeRefusalsAreExplicitAndDoNotMasqueradeAsFullReads" + }, + { + "label": "restored legacy formats admit version 3 dates", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/manifest.go", + "find": "if state.Version < 3 && !obj.Modified.IsZero() {", + "replace": "if state.Version < 3 && !obj.Modified.IsZero() && false {", + "test": "TestObjectModificationDatePersistsAndLegacyDatesAreNotInvented" + }, + { + "label": "restored dates admit year zero", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/manifest.go", + "find": "if !obj.Modified.IsZero() && (obj.Modified.Year() < 1 || obj.Modified.Year() > 9999) {", + "replace": "if !obj.Modified.IsZero() && (obj.Modified.Year() < 1 || obj.Modified.Year() > 9999) && false {", + "test": "TestObjectModificationDatePersistsAndLegacyDatesAreNotInvented" + }, + { + "label": "publication dates are returned but not persisted", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/store.go", + "find": "next.Buckets[name].Objects[key] = obj", + "replace": "next.Buckets[name].Objects[key] = obj\n\tundated := obj\n\tundated.Modified = time.Time{}\n\tnext.Buckets[name].Objects[key] = undated", + "test": "TestObjectModificationDatePersistsAndLegacyDatesAreNotInvented" + }, + { + "label": "seeking verified readers ignores cancellation", + "package": "./internal/core/objectstore/", + "file": "internal/core/objectstore/files.go", + "find": "func (r *blobReader) Seek(offset int64, whence int) (int64, error) {\n\tif err := r.ctx.Err(); err != nil {", + "replace": "func (r *blobReader) Seek(offset int64, whence int) (int64, error) {\n\tif err := r.ctx.Err(); err != nil && false {", + "test": "TestVerifiedObjectReaderSeeksAndCancelsWithoutExposingFileMethods" + } + ] +} diff --git a/tools/falsify/specs/s3-gateway.json b/tools/falsify/specs/s3-gateway.json index 386dda2b..a279de2e 100644 --- a/tools/falsify/specs/s3-gateway.json +++ b/tools/falsify/specs/s3-gateway.json @@ -60,9 +60,9 @@ { "label": "range requests silently return full bytes", "file": "internal/s3/server.go", - "find": "case \"range\", \"if-match\", \"if-none-match\", \"if-modified-since\", \"if-unmodified-since\", \"if-range\", \"content-md5\":", - "replace": "case \"\\x00never\", \"if-match\", \"if-none-match\", \"if-modified-since\", \"if-unmodified-since\", \"if-range\", \"content-md5\":", - "test": "TestRefusalsDoNotMasqueradeAsCloudSupport" + "find": "if values := r.Header.Values(\"Range\"); len(values) != 0 {", + "replace": "if values := r.Header.Values(\"Range\"); len(values) != 0 && false {", + "test": "TestSingleByteRangesReturnOnlyTheirVerifiedBytes" }, { "label": "unsafe restored metadata is emitted as HTTP headers", diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 29362fb1..90ff91d0 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -254,13 +254,13 @@ var rules = []rule{ }, { Path: "internal/core/objectstore/", - Why: "neutral persisted bytes used by the S3 component and the real emulator process", + Why: "neutral persisted bytes and modification dates used by the S3 component and real emulator process", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "complete provider-client lifecycles, checkpoint/reset, power-loss recovery and network filesystem locking", }, { Path: "internal/s3/", - Why: "bounded S3 prefix/host dispatch, private Scaleway buckets and real-byte official-client proof", + Why: "bounded S3 prefix/host dispatch, listing/ranges, private Scaleway buckets and real-byte official-client proof", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "one AWS/scw/exo version, not complete Exoscale lifecycles, public ACLs, version histories, TLS interception, cloud virtual hosts, ETags, multipart or public stacks", }, From fc9258d8bb89a66f96f60ae85abc61329cfa55b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 14:21:10 +0100 Subject: [PATCH 09/28] feat(s3): prove Exo transactional bulk deletion Verify checksums and closed XML before deleting unversioned references in one metadata transaction. Real Exo removes four keys; AWS verifies empty inventory and Quiet/idempotent results across restart while Scaleway bytes remain intact. Multipart and complete bucket workflows remain open in #856 and #858. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 8 + CHANGELOG.md | 8 + docs/confidence.md | 2 +- docs/limits.md | 5 + docs/object-storage-routing.md | 34 +++- .../cli/object_storage_conformance_test.go | 65 ++++++- internal/core/objectstore/delete_test.go | 95 ++++++++++ internal/core/objectstore/store.go | 37 ++++ internal/s3/bucket_configuration.go | 13 +- internal/s3/delete.go | 161 +++++++++++++++++ internal/s3/delete_test.go | 168 ++++++++++++++++++ internal/s3/server.go | 14 +- mise.toml | 2 +- tools/falsify/specs/s3-delete.json | 141 +++++++++++++++ tools/testplan/rules.go | 4 +- 15 files changed, 738 insertions(+), 19 deletions(-) create mode 100644 internal/core/objectstore/delete_test.go create mode 100644 internal/s3/delete.go create mode 100644 internal/s3/delete_test.go create mode 100644 tools/falsify/specs/s3-delete.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index 88a25387..f8284d5b 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,14 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **La suppression en lot Exo, mesurée** (#856) : exo 1.95.1 supprime quatre + vraies clés ; AWS vérifie l'inventaire vide après redémarrage et les octets + Scaleway intacts. DeleteObjects vérifie checksum et XML fermé avant une seule + transaction de métadonnées (1 à 1000 clés sans version, XML de 8 Mio maximum). + Les réponses verbose/quiet décrivent des suppressions réelles. Les blobs restent + conservés ; multipart, erreurs cloud mixtes par clé et cycles complets de + buckets restent hors de cette tranche mesurée. + - **Le listing Exo et les téléchargements multi-plages, mesurés** (#850) : exo 1.95.1 retrouve deux binaires de 73 728 et 6 340 608 octets, octets/SHA-256 et dates de listing identiques avant/après redémarrage réel. AWS décode aussi diff --git a/CHANGELOG.md b/CHANGELOG.md index 51ee40ef..9b6ee3e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Measured Exo bulk deletion** (#856): exo 1.95.1 deletes four actual keys; + AWS independently checks the empty inventory across restart and untouched + Scaleway bytes. DeleteObjects verifies transport checksums and closed XML + before one metadata transaction (1..1000 unversioned keys, 8 MiB XML). + Verbose/quiet results describe actual deletions. Blobs remain retained; + multipart, mixed per-key cloud errors and complete bucket workflows remain + outside this measured slice. + - **Measured Exo object listing and multi-range downloads** (#850): exo 1.95.1 downloads 73,728-byte and 6,340,608-byte binary objects with exact byte/SHA-256 and listing-date proof before/after actual restart. AWS independently diff --git a/docs/confidence.md b/docs/confidence.md index 9af568ac..9954127e 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -44,7 +44,7 @@ prints the reason under the pack that owns it. | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets, AWS paginated listing and host-routed Exo uploads/downloads (including multi-range 6,340,608 bytes), with byte/checksum and persisted-date proof across actual restart; complete Exoscale bucket workflows are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-listing-and-multi-range-downloads) | +| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download/delete | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets, AWS pagination and host-routed Exo uploads/downloads (including multi-range 6,340,608 bytes) and bulk deletion, with actual bytes, dates and empty-inventory proof across restart; complete Exoscale bucket workflows are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-bulk-deletion) | | Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | diff --git a/docs/limits.md b/docs/limits.md index adbf7759..b5090ff2 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -19,6 +19,11 @@ multiple HTTP ranges. ListObjectsV2 exposes actual persisted modification dates; legacy undated objects must be overwritten before their listing can be served. Private byte metadata v3 reads v1/v2 without inventing dates; successful mutations upgrade it, and older binaries cannot read v3. +Exo also bulk-deletes actual unversioned keys; AWS independently confirms the +empty inventory survives restart while the other provider's bytes remain. +DeleteObjects requires a verified transport checksum and closed XML, bounded to +1000 keys and 8 MiB. Store failures return an explicit whole-request Feint refusal, +not fabricated per-key cloud errors; immutable blobs remain retained. This does not implement bucket virtual hosts, DNS/TLS interception or a complete Exoscale lifecycle; use a dedicated shell because its S3 environment replaces proxy settings. diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index acda7415..8a10b75f 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -218,7 +218,7 @@ index. Existing bytes, generations and quotas remain unchanged. Legacy buckets without a stored creation date receive an explicit listing refusal rather than an invented date. Resource snapshots still do not own this metadata. -Exoscale upload/download/delete, its mandatory ListMultipartUploads during +Complete Exoscale bucket workflows, their mandatory ListMultipartUploads during bucket removal, native bucket-usage API, and full Object Storage 1.0 remain separate follow-up work. No empty multipart success stub was added. @@ -342,4 +342,34 @@ Resource snapshots still do not own these bytes. The date, listing and range properties are replayed by `tools/falsify/specs/s3-download.json`. This milestone does **not** certify normal Exo bucket creation/cleanup, ACL/CORS/ownership discovery, native bucket usage, -multipart, version histories, bulk deletion, TLS or Terraform redirection. +multipart, version histories, TLS or Terraform redirection. + +## Measured Exoscale bulk deletion + +The pinned exo 1.95.1 `storage delete --force --recursive --verbose sos://BUCKET/` +enumerates keys with ListObjectsV2, then sends DeleteObjects. The mounted witness +failed with an actual 501 before this support. It now deletes four real keys, +including a literal escaped key, and checks every reported key. An independent +AWS CLI inventory is empty before and after another actual process restart; +same-name objects in the Scaleway namespace still return their original bytes. +AWS also decodes the idempotent deletion of a missing key using CRC32. + +The request/response members and XML namespace follow the S3 v1.2.0 serializer +and deserializer used by that Exo release. The bounded subset accepts 1..1000 +unversioned keys in at most 8 MiB of decoded XML. Content-MD5, CRC32 or SHA-256 +transport validation finishes before any deletion. Unknown XML fields, versions, +attributes, duplicate singleton fields, invalid sizes/checksums and unrelated +query features are refused before changing any reference. + +Verbose responses identify every requested deleted key, including missing keys. +Quiet responses omit successful keys only after actual deletion. The neutral +store publishes one metadata transaction and retains immutable blobs and their +quota charge: this is not GC. A store error yields an explicit logged +whole-request Feint refusal, not invented mixed per-key cloud errors. An +indeterminate directory-sync failure still faults the store until reopen. +The index remains v3. + +`tools/falsify/specs/s3-delete.json` holds the destructive-input checks and +publication properties. This closes only the bulk-deletion slice of #856: +multipart upload/list/abort/complete and ordinary `exo storage rb` remain +unsupported, and the complete bucket workflow remains #858. diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index 1c245227..916a2c5e 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -377,25 +377,72 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } run("start", "--addr", addr, "--vm", "off", "--objectstore", root, "--objectstore-budget-mib", "20", "--s3-object-limit-mib", "8", "--s3-host-doors") + checkInventory := func(want int) { + t.Helper() + var inventory struct { + Name string + KeyCount *int + Contents []json.RawMessage + } + out := call(first.ObjectStorage.Doors[1], "list-objects-v2", "--bucket", bucket, "--no-paginate") + if err := json.Unmarshal(out, &inventory); err != nil || inventory.Name != bucket || + inventory.KeyCount == nil || *inventory.KeyCount != want || len(inventory.Contents) != want { + t.Fatalf("independent inventory did not witness %d actual objects: %v %q", want, err, out) + } + } + checkInventory(4) + deleted := exo.Run(t, "storage", "delete", "--force", "--recursive", "--verbose", "sos://"+bucket+"/") + for _, objectKey := range []string{key, exoKey, largeKey, "after-restart"} { + if !bytes.Contains(deleted, []byte(objectKey+"\n")) { + t.Fatalf("Exo did not report the actual deleted key %q: %q", objectKey, deleted) + } + } + var absentDeletion struct { + Deleted []struct{ Key string } + Errors []json.RawMessage + } + absent := call(first.ObjectStorage.Doors[1], "delete-objects", "--bucket", bucket, + "--delete", `{"Objects":[{"Key":"absent-witness"}]}`, "--checksum-algorithm", "CRC32") + if err := json.Unmarshal(absent, &absentDeletion); err != nil || len(absentDeletion.Errors) != 0 || + len(absentDeletion.Deleted) != 1 || absentDeletion.Deleted[0].Key != "absent-witness" { + t.Fatalf("actual AWS decoder lost idempotent bulk deletion: %v %q", err, absent) + } + checkEmpty := func() { + t.Helper() + checkInventory(0) + getKey(first.ObjectStorage.Doors[0], exoKey, []byte("other-provider-host-key")) + } + checkEmpty() + call(first.ObjectStorage.Doors[1], "put-object", "--bucket", bucket, "--key", "quiet-witness", + "--body", input, "--checksum-algorithm", "CRC32") + checkInventory(1) + quiet := call(first.ObjectStorage.Doors[1], "delete-objects", "--bucket", bucket, + "--delete", `{"Objects":[{"Key":"quiet-witness"}],"Quiet":true}`, "--checksum-algorithm", "CRC32") + if len(bytes.TrimSpace(quiet)) != 0 { + var response map[string]json.RawMessage + if err := json.Unmarshal(quiet, &response); err != nil || len(response) != 0 { + t.Fatalf("actual AWS Quiet result advertised successful keys or errors: %v %q", err, quiet) + } + } + checkEmpty() + run("restart", "--addr", addr) + checkEmpty() for _, door := range first.ObjectStorage.Doors { - call(door, "head-object", "--bucket", bucket, "--key", key) - call(door, "delete-object", "--bucket", bucket, "--key", key) - call(door, "delete-object", "--bucket", bucket, "--key", exoKey) if door.Provider == "exoscale" { - call(door, "delete-object", "--bucket", bucket, "--key", "after-restart") - call(door, "delete-object", "--bucket", bucket, "--key", largeKey) - } - if door.Provider == "scaleway" { - bucketCall("delete", true) - } else { call(door, "delete-bucket", "--bucket", bucket) + continue } + call(door, "head-object", "--bucket", bucket, "--key", key) + call(door, "delete-object", "--bucket", bucket, "--key", key) + call(door, "delete-object", "--bucket", bucket, "--key", exoKey) + bucketCall("delete", true) } if guardHits.Load() != 1 { t.Fatalf("official client reached the refusing egress proxy: delta=%d", guardHits.Load()-1) } t.Logf("Exo downloads before/after actual restart: small=%d bytes; multi-range=%d bytes sha256=%x", len(payload), len(largePayload), sha256.Sum256(largePayload)) + t.Log("Exo bulk-deleted four actual keys; independent AWS inventory empty across restart; other provider bytes retained") t.Logf("actual feint pid %d -> %d; two same-port namespaces; preserved key; bytes=%d sha256=%s; stop/reopen/restart; proxy delta=0 (not a network namespace)", first.PID, restarted.PID, len(payload), hex.EncodeToString(sum[:])) } diff --git a/internal/core/objectstore/delete_test.go b/internal/core/objectstore/delete_test.go new file mode 100644 index 00000000..3839f029 --- /dev/null +++ b/internal/core/objectstore/delete_test.go @@ -0,0 +1,95 @@ +package objectstore + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestBulkDeletePersistsReferencesAndRetainsBytes(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + object := mustPut(t, s, "bucket", "a", "retained bytes") + mustPut(t, s, "bucket", "b", "other bytes") + mustPut(t, s, "bucket", "keep", "untouched") + keys := []string{"b", "a", "a", "missing"} + if err := s.DeleteObjects(t.Context(), "bucket", keys); err != nil { + t.Fatal(err) + } + if !slices.Equal(keys, []string{"b", "a", "a", "missing"}) { + t.Fatal("caller-owned key order changed:", keys) + } + if _, err := os.Stat(filepath.Join(path, "blobs", object.Digest)); err != nil { + t.Fatal("bulk deletion reclaimed immutable bytes:", err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Error(err) + } + }() + inventory, err := reopened.ListObjects(t.Context(), "bucket") + if err != nil || len(inventory) != 1 || inventory["keep"].Size != 9 { + t.Fatal("persisted bulk deletion lost other references:", inventory, err) + } +} + +func TestBulkDeleteRefusalsPreserveEveryReference(t *testing.T) { + for _, name := range []string{"invalid-key", "empty", "canceled", "commit-failure"} { + t.Run(name, func(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + mustPut(t, s, "bucket", "a", "a") + mustPut(t, s, "bucket", "b", "b") + ctx := t.Context() + keys := []string{"a", "b"} + switch name { + case "invalid-key": + keys[1] = "" + case "empty": + keys = nil + case "canceled": + canceled, cancel := context.WithCancel(ctx) + cancel() + ctx = canceled + case "commit-failure": + index := filepath.Join(path, "index.json") + saved := filepath.Join(path, "saved-index") + if err := os.Rename(index, saved); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(index, 0700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Remove(index); err != nil { + t.Error(err) + } + if err := os.Rename(saved, index); err != nil { + t.Error(err) + } + }) + } + if err := s.DeleteObjects(ctx, "bucket", keys); err == nil { + t.Fatal("invalid or failed batch returned success") + } + inventory, err := s.ListObjects(t.Context(), "bucket") + if err != nil || len(inventory) != 2 { + t.Fatal("failed batch partially deleted references:", inventory, err) + } + }) + } + s, _ := newStore(t) + if err := s.DeleteObjects(t.Context(), "missing", []string{"key"}); !errors.Is(err, ErrBucketNotFound) { + t.Fatal("missing bucket did not preserve its typed error:", err) + } +} diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index 1863bf17..43f2b936 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -15,6 +15,7 @@ import ( "maps" "math" "os" + "slices" "sort" "sync" "time" @@ -347,6 +348,42 @@ func (s *Store) DeleteObject(ctx context.Context, name, key string) error { return s.commit(ctx, next) } +// DeleteObjects removes references in one metadata transaction, retaining blobs. +// Missing keys are harmless. Validation or a failed commit never publishes a +// partially deleted inventory; indeterminate durability still faults the store. +func (s *Store) DeleteObjects(ctx context.Context, name string, keys []string) error { + if len(keys) == 0 { + return fmt.Errorf("%w: deletion requires at least one key", ErrInvalid) + } + keys = slices.Clone(keys) + for _, key := range keys { + if err := validKey(key); err != nil { + return err + } + } + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + slices.Sort(keys) + keys = slices.Compact(keys) + for _, key := range keys { + release := serialise.Lock(s.domain + "/" + name + "\x00" + key) + defer release() + } + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.lookup(name); err != nil { + return err + } + next := s.clone() + for _, key := range keys { + delete(next.Buckets[name].Objects, key) + } + return s.commit(ctx, next) +} + func (s *Store) lookup(name string) (*bucket, error) { if err := validName(name); err != nil { return nil, err diff --git a/internal/s3/bucket_configuration.go b/internal/s3/bucket_configuration.go index 122f3bbd..d334e442 100644 --- a/internal/s3/bucket_configuration.go +++ b/internal/s3/bucket_configuration.go @@ -52,18 +52,25 @@ func (s *Server) bucketHeaders(r *http.Request, query url.Values, bucket bool) ( } func configurationBody(w http.ResponseWriter, r *http.Request) ([]byte, error) { - upload, _, err := newUpload(w, r, 64<<10) + return xmlBody(w, r, 64<<10) +} + +func xmlBody(w http.ResponseWriter, r *http.Request, max int64) ([]byte, error) { + upload, size, err := newUpload(w, r, max) if err != nil { return nil, err } - body, err := io.ReadAll(upload) + body, err := io.ReadAll(io.LimitReader(upload, max+1)) if err != nil { return nil, err } + if int64(len(body)) != size { + return nil, errors.New("S3 XML body does not match its bounded decoded size") + } if values := r.Header.Values("Content-MD5"); len(values) != 0 { digest := md5.Sum(body) //nolint:gosec // Verify the legacy S3 transport checksum; never use it as identity. if len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:]) { - return nil, errors.New("bucket configuration Content-MD5 mismatch") + return nil, errors.New("S3 XML Content-MD5 mismatch") } } return body, nil diff --git a/internal/s3/delete.go b/internal/s3/delete.go new file mode 100644 index 00000000..bd764916 --- /dev/null +++ b/internal/s3/delete.go @@ -0,0 +1,161 @@ +package s3 + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "unicode/utf8" +) + +const maxDeleteXMLBytes = 8 << 20 + +type deleteRequest struct { + keys []string + quiet bool +} + +// Names and namespace follow the S3 v1.2.0 serializer/deserializer used by +// exo 1.95.1. TestMountedS3OfficialAWSCLIBytes holds the real client's decoding. +type deletedObject struct { + Key string +} + +type deleteResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult"` + Deleted []deletedObject `xml:"Deleted,omitempty"` +} + +func (s *Server) deleteObjects(w http.ResponseWriter, r *http.Request, bucket string, query url.Values) { + for key, values := range query { + if len(values) != 1 || (key != "delete" && key != "x-id") || + (key == "delete" && values[0] != "") || (key == "x-id" && values[0] != "DeleteObjects") { + s.refuse(w, r, http.StatusBadRequest, "DeleteObjects requires one empty delete query", nil) + return + } + } + if r.Header.Get("Content-MD5") == "" && r.Header.Get("X-Amz-Checksum-Crc32") == "" && + r.Header.Get("X-Amz-Checksum-Sha256") == "" && r.Header.Get("X-Amz-Trailer") == "" { + s.refuse(w, r, http.StatusBadRequest, "DeleteObjects requires a verified transport checksum", nil) + return + } + body, err := xmlBody(w, r, maxDeleteXMLBytes) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid DeleteObjects body", err) + return + } + request, err := parseDelete(body) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid or unsupported DeleteObjects XML", err) + return + } + result := deleteResult{} + if !request.quiet { + for _, key := range request.keys { + result.Deleted = append(result.Deleted, deletedObject{Key: key}) + } + } + if err := s.store.DeleteObjects(r.Context(), s.name(bucket), request.keys); err != nil { + s.storeError(w, r, bucket, err) + return + } + s.writeXML(w, r, http.StatusOK, result) +} + +func parseDelete(body []byte) (deleteRequest, error) { + decoder := xml.NewDecoder(bytes.NewReader(body)) //nolint:gosec // Bounded body; directives and unknown elements are refused. + result := deleteRequest{} + var stack []string + var text strings.Builder + roots, keyCount, quietCount := 0, 0, 0 + declaration := false + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return result, err + } + switch t := token.(type) { + case xml.StartElement: + if t.Name.Space != "" && t.Name.Space != s3XMLNamespace { + return result, errors.New("unsupported DeleteObjects namespace") + } + for _, attr := range t.Attr { + if len(stack) != 0 || attr.Name.Space != "" || attr.Name.Local != "xmlns" || attr.Value != s3XMLNamespace { + return result, errors.New("unsupported DeleteObjects attribute") + } + } + switch len(stack) { + case 0: + if roots != 0 || t.Name.Local != "Delete" { + return result, errors.New("one Delete root is required") + } + roots++ + case 1: + switch t.Name.Local { + case "Object": + keyCount = 0 + case "Quiet": + if quietCount != 0 { + return result, errors.New("duplicate Quiet element") + } + quietCount++ + text.Reset() + default: + return result, errors.New("unsupported Delete member") + } + case 2: + if stack[1] != "Object" || t.Name.Local != "Key" { + return result, errors.New("object requires one Key; versions are not supported") + } + keyCount++ + text.Reset() + default: + return result, errors.New("nested DeleteObjects value") + } + stack = append(stack, t.Name.Local) + case xml.EndElement: + switch t.Name.Local { + case "Object": + key := text.String() + if keyCount != 1 || key == "" || len(key) > 1024 || !utf8.ValidString(key) { + return result, errors.New("invalid DeleteObjects key") + } + result.keys = append(result.keys, key) + if len(result.keys) > 1000 { + return result, errors.New("DeleteObjects exceeds 1000 keys") + } + case "Quiet": + value := strings.TrimSpace(text.String()) + if value != "true" && value != "false" { + return result, errors.New("invalid Quiet boolean") + } + result.quiet = value == "true" + } + stack = stack[:len(stack)-1] + case xml.CharData: + if len(stack) == 3 || (len(stack) == 2 && stack[1] == "Quiet") { + text.Write(t) + } else if strings.TrimSpace(string(t)) != "" { + return result, errors.New("unexpected DeleteObjects text") + } + case xml.ProcInst: + if roots != 0 || t.Target != "xml" || declaration { + return result, errors.New("unexpected DeleteObjects instruction") + } + declaration = true + default: + return result, fmt.Errorf("unsupported DeleteObjects token %T", token) + } + } + if len(stack) != 0 || len(result.keys) == 0 { + return result, errors.New("DeleteObjects requires a complete nonempty request") + } + return result, nil +} diff --git a/internal/s3/delete_test.go b/internal/s3/delete_test.go new file mode 100644 index 00000000..7efb59f5 --- /dev/null +++ b/internal/s3/delete_test.go @@ -0,0 +1,168 @@ +package s3 + +import ( + "crypto/md5" //nolint:gosec // Exercise the official client's legacy transport checksum. + "encoding/base64" + "encoding/binary" + "encoding/xml" + "hash/crc32" + "net/http/httptest" + "strings" + "testing" +) + +func deleteMD5(body []byte) string { + sum := md5.Sum(body) //nolint:gosec // Transport checksum only. + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func TestDeleteObjectsReturnsActualLiteralKeysAndQuietState(t *testing.T) { + for _, mode := range []string{"false", "true"} { + t.Run(mode, func(t *testing.T) { + quiet := mode == "true" + s, store := fixture(t) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + if w := request(t, s, "PUT", "/bucket/a%2F..%2F%252F%3F%26%23", []byte("bytes"), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + body := []byte(`a/../%2F?&#missing` + + mode + ``) + w := request(t, s, "POST", "/bucket?delete=&x-id=DeleteObjects", body, map[string]string{"Content-MD5": deleteMD5(body)}) + var result deleteResult + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &result) != nil { + t.Fatal("invalid DeleteResult:", w.Code, w.Body.String()) + } + if (!quiet && (len(result.Deleted) != 2 || result.Deleted[0].Key != "a/../%2F?&#" || result.Deleted[1].Key != "missing")) || + (quiet && len(result.Deleted) != 0) { + t.Fatal("literal keys or Quiet lost:", w.Body.String()) + } + inventory, err := store.ListObjects(t.Context(), "bucket") + if err != nil || len(inventory) != 0 { + t.Fatal("successful response did not delete actual objects:", inventory, err) + } + }) + } +} + +func TestDeleteObjectsAcceptsExactly1000Keys(t *testing.T) { + body := `` + strings.Repeat(`key`, 1000) + `` + request, err := parseDelete([]byte(body)) + if err != nil || len(request.keys) != 1000 { + t.Fatal("the documented key bound was not accepted:", len(request.keys), err) + } +} + +func TestDeleteObjectsClosedRefusalShapes(t *testing.T) { + for _, body := range []string{ + ``, + `` + strings.Repeat("x", 1025) + ``, + } { + if _, err := parseDelete([]byte(body)); err == nil { + t.Fatal("the wire parser accepted an empty or oversized request") + } + } +} + +func TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix(t *testing.T) { + valid := `keep` + for _, tc := range []struct{ name, body, query string }{ + {"version", `keepotherversion`, "?delete"}, + {"unknown", `keep`, "?delete"}, + {"duplicate-key", `keepother`, "?delete"}, + {"empty-key", `keep`, "?delete"}, + {"oversize-key", `keep` + strings.Repeat("x", 1025) + ``, "?delete"}, + {"over-1000", `` + strings.Repeat(`keep`, 1001) + ``, "?delete"}, + {"quiet-value", `keepmaybe`, "?delete"}, + {"quiet-duplicate", `keeptruefalse`, "?delete"}, + {"namespace", `keep`, "?delete"}, + {"unbound-prefix", `keep`, "?delete"}, + {"attribute", `keep`, "?delete"}, + {"nested", `keep`, "?delete"}, + {"directive", `keep`, "?delete"}, + {"trailing", valid + ``, "?delete"}, + {"empty-request", ``, "?delete"}, + {"query-value", valid, "?delete=true"}, + {"query-duplicate", valid, "?delete=&delete="}, + {"unknown-query", valid, "?delete=&unknown="}, + {"wrong-operation", valid, "?delete=&x-id=PutObject"}, + } { + t.Run(tc.name, func(t *testing.T) { + s, store := fixture(t) + if w := request(t, s, "PUT", "/bucket", nil, nil); w.Code != 200 { + t.Fatal(w.Code) + } + if w := request(t, s, "PUT", "/bucket/keep", []byte("retain"), nil); w.Code != 200 { + t.Fatal(w.Code) + } + body := []byte(tc.body) + w := request(t, s, "POST", "/bucket"+tc.query, body, map[string]string{"Content-MD5": deleteMD5(body)}) + inventory, err := store.ListObjects(t.Context(), "bucket") + if w.Code < 400 || err != nil || len(inventory) != 1 { + t.Fatal("refused batch deleted a validated prefix:", w.Code, inventory, err) + } + }) + } +} + +func TestDeleteObjectsChecksumsAndSizeBeforeDeletion(t *testing.T) { + body := []byte(`keep`) + var crc [4]byte + binary.BigEndian.PutUint32(crc[:], crc32.ChecksumIEEE(body)) + for _, tc := range []struct { + name, header, value string + accept bool + length int64 + }{ + {"md5", "Content-MD5", deleteMD5(body), true, int64(len(body))}, + {"crc32", "X-Amz-Checksum-Crc32", base64.StdEncoding.EncodeToString(crc[:]), true, int64(len(body))}, + {"wrong-md5", "Content-MD5", deleteMD5([]byte("wrong")), false, int64(len(body))}, + {"wrong-crc32", "X-Amz-Checksum-Crc32", "AAAAAA==", false, int64(len(body))}, + {"missing", "", "", false, int64(len(body))}, + {"short-body", "Content-MD5", deleteMD5(body), false, int64(len(body) + 1)}, + {"long-body", "Content-MD5", deleteMD5(body), false, int64(len(body) - 1)}, + {"over-bound", "Content-MD5", deleteMD5(body), false, maxDeleteXMLBytes + 1}, + {"duplicate-md5", "Content-MD5", deleteMD5(body), false, int64(len(body))}, + } { + t.Run(tc.name, func(t *testing.T) { + s, store := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + request(t, s, "PUT", "/bucket/keep", []byte("retain"), nil) + r := httptest.NewRequest("POST", "http://127.0.0.1:4599/bucket?delete", strings.NewReader(string(body))) + r.RemoteAddr = "127.0.0.1:1234" + r.ContentLength = tc.length + if tc.header != "" { + r.Header.Set(tc.header, tc.value) + } + if tc.name == "duplicate-md5" { + r.Header.Add(tc.header, tc.value) + } + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + inventory, err := store.ListObjects(t.Context(), "bucket") + want := 1 + if tc.accept { + want = 0 + } + if (w.Code == 200) != tc.accept || err != nil || len(inventory) != want { + t.Fatal("checksum/size validation did not precede deletion:", w.Code, inventory, err) + } + }) + } +} + +func FuzzDeleteObjectsClosedShape(f *testing.F) { + for _, body := range []string{`key`, ``, ``, `&`} { + f.Add(body) + } + f.Fuzz(func(t *testing.T, body string) { + if len(body) > maxDeleteXMLBytes { + return + } + request, err := parseDelete([]byte(body)) + if err == nil && (len(request.keys) == 0 || len(request.keys) > 1000) { + t.Fatal("accepted unbounded or empty deletion") + } + }) +} diff --git a/internal/s3/server.go b/internal/s3/server.go index 542f3697..2a4b204c 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -78,7 +78,7 @@ func New(store *objectstore.Store, cfg Config) (*Server, error) { return &Server{store: store, cfg: cfg}, nil } -// ServeHTTP implements bucket PUT/HEAD/DELETE, ListObjectsV2 and object +// ServeHTTP implements bucket PUT/HEAD/DELETE, ListObjectsV2, DeleteObjects and object // PUT/GET/HEAD/DELETE, including single-byte-range GETs. // Other features get an explicit Feint refusal, never a successful empty stub // or an invented cloud XML error. Signatures and credentials are never logged. @@ -109,10 +109,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") listing := r.Method == http.MethodGet && rawKey == "" && query.Get("list-type") == "2" + bulkDelete := r.Method == http.MethodPost && rawKey == "" && query.Has("delete") for key := range query { if listing && listQueryKey(key) { continue } + if bulkDelete && key == "delete" { + continue + } if key != "x-id" && (!s.cfg.BucketConfiguration || (key != "versioning" && key != "acl" && key != "tagging")) { s.refuse(w, r, http.StatusNotImplemented, "S3 query features are not implemented", nil) return @@ -125,6 +129,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.refuse(w, r, http.StatusNotImplemented, "S3 bucket header feature is not implemented", err) return } + if bulkDelete { + checked = checked.Clone(r.Context()) + checked.Header.Del("Content-MD5") + } if err := supportedHeaders(checked); err != nil { s.refuse(w, r, http.StatusNotImplemented, "S3 header feature is not implemented", err) return @@ -146,6 +154,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.listObjects(w, r, bucket, query) return } + if bulkDelete { + s.deleteObjects(w, r, bucket, query) + return + } if key == "" { if len(query) != 0 && (len(query) != 1 || !query.Has("x-id")) { s.bucketConfiguration(w, r, bucket, query) diff --git a/mise.toml b/mise.toml index a6d04f3b..b3e873da 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove AWS/scw bytes, paginated listing and host-routed exo 1.95.1 upload/range download across actual restart" +description = "Prove AWS/scw bytes, listing and host-routed exo 1.95.1 upload/range download/bulk deletion across actual restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:exo:endpoint"] diff --git a/tools/falsify/specs/s3-delete.json b/tools/falsify/specs/s3-delete.json new file mode 100644 index 00000000..9ce3ed2e --- /dev/null +++ b/tools/falsify/specs/s3-delete.json @@ -0,0 +1,141 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "bulk deletion silently accepts duplicate query values", + "file": "internal/s3/delete.go", + "find": "if len(values) != 1 || (key != \"delete\" && key != \"x-id\")", + "replace": "if (len(values) != 1 && false) || (key != \"delete\" && key != \"x-id\")", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "bulk deletion ignores a nonempty operation selector", + "file": "internal/s3/delete.go", + "find": "(key == \"delete\" && values[0] != \"\")", + "replace": "(key == \"delete\" && values[0] != \"\" && false)", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "bulk deletion accepts an unrelated SDK operation", + "file": "internal/s3/delete.go", + "find": "(key == \"x-id\" && values[0] != \"DeleteObjects\")", + "replace": "(key == \"x-id\" && values[0] != \"DeleteObjects\" && false)", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "destructive XML need not carry a transport checksum", + "file": "internal/s3/delete.go", + "find": "r.Header.Get(\"X-Amz-Checksum-Sha256\") == \"\" && r.Header.Get(\"X-Amz-Trailer\") == \"\" {", + "replace": "r.Header.Get(\"X-Amz-Checksum-Sha256\") == \"\" && r.Header.Get(\"X-Amz-Trailer\") == \"\" && false {", + "test": "TestDeleteObjectsChecksumsAndSizeBeforeDeletion" + }, + { + "label": "destructive XML Content-MD5 mismatch is ignored", + "file": "internal/s3/bucket_configuration.go", + "find": "if len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:]) {", + "replace": "if (len(values) != 1 || values[0] != base64.StdEncoding.EncodeToString(digest[:])) && false {", + "test": "TestDeleteObjectsChecksumsAndSizeBeforeDeletion" + }, + { + "label": "destructive XML decoded size may differ from its declaration", + "file": "internal/s3/bucket_configuration.go", + "find": "if int64(len(body)) != size {", + "replace": "if int64(len(body)) != size && false {", + "test": "TestDeleteObjectsChecksumsAndSizeBeforeDeletion" + }, + { + "label": "XML from a foreign namespace deletes objects", + "file": "internal/s3/delete.go", + "find": "if t.Name.Space != \"\" && t.Name.Space != s3XMLNamespace {", + "replace": "if t.Name.Space != \"\" && t.Name.Space != s3XMLNamespace && false {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "XML silently drops unknown attributes before deletion", + "file": "internal/s3/delete.go", + "find": "if len(stack) != 0 || attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace {", + "replace": "if (len(stack) != 0 || attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace) && false {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "a trailing root is silently accepted", + "file": "internal/s3/delete.go", + "find": "if roots != 0 || t.Name.Local != \"Delete\" {", + "replace": "if (roots != 0 && false) || t.Name.Local != \"Delete\" {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "duplicate Quiet fields silently change response semantics", + "file": "internal/s3/delete.go", + "find": "if quietCount != 0 {", + "replace": "if quietCount != 0 && false {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "duplicate object Key fields silently choose another object", + "file": "internal/s3/delete.go", + "find": "if keyCount != 1 || key == \"\"", + "replace": "if keyCount < 1 || key == \"\"", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "oversized keys are accepted by the wire parser", + "file": "internal/s3/delete.go", + "find": "key == \"\" || len(key) > 1024 || !utf8.ValidString(key)", + "replace": "key == \"\" || (len(key) > 1024 && false) || !utf8.ValidString(key)", + "test": "TestDeleteObjectsClosedRefusalShapes" + }, + { + "label": "the 1000-key operation bound is ignored", + "file": "internal/s3/delete.go", + "find": "if len(result.keys) > 1000 {", + "replace": "if len(result.keys) > 1000 && false {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "unknown Quiet values are treated as false", + "file": "internal/s3/delete.go", + "find": "if value != \"true\" && value != \"false\" {", + "replace": "if value != \"true\" && value != \"false\" && false {", + "test": "TestDeleteObjectsRefusalsCannotDeleteValidatedPrefix" + }, + { + "label": "empty deletion is advertised as success", + "file": "internal/s3/delete.go", + "find": "len(stack) != 0 || len(result.keys) == 0", + "replace": "len(stack) != 0 || (len(result.keys) == 0 && false)", + "test": "TestDeleteObjectsClosedRefusalShapes" + }, + { + "label": "Quiet requests incorrectly return successful keys", + "file": "internal/s3/delete.go", + "find": "if !request.quiet {", + "replace": "if !request.quiet || true {", + "test": "TestDeleteObjectsReturnsActualLiteralKeysAndQuietState" + }, + { + "label": "validated XML claims deletion without changing references", + "file": "internal/core/objectstore/store.go", + "find": "for _, key := range keys {\n\t\tdelete(next.Buckets[name].Objects, key)\n\t}", + "replace": "for _, key := range keys {\n\t\tif key == \"\\x00never\" { delete(next.Buckets[name].Objects, key) }\n\t}", + "package": "./internal/core/objectstore/", + "test": "TestBulkDeletePersistsReferencesAndRetainsBytes" + }, + { + "label": "caller-owned key slices are reordered", + "file": "internal/core/objectstore/store.go", + "find": "keys = slices.Clone(keys)", + "replace": "keys = slices.Clip(keys)", + "package": "./internal/core/objectstore/", + "test": "TestBulkDeletePersistsReferencesAndRetainsBytes" + }, + { + "label": "a failed batch commit publishes partially deleted metadata", + "file": "internal/core/objectstore/store.go", + "find": "for _, key := range keys {\n\t\tdelete(next.Buckets[name].Objects, key)\n\t}\n\treturn s.commit(ctx, next)", + "replace": "for _, key := range keys {\n\t\tdelete(next.Buckets[name].Objects, key)\n\t}\n\ts.state = next\n\treturn s.commit(ctx, next)", + "package": "./internal/core/objectstore/", + "test": "TestBulkDeleteRefusalsPreserveEveryReference" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 90ff91d0..bb14b5af 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -254,13 +254,13 @@ var rules = []rule{ }, { Path: "internal/core/objectstore/", - Why: "neutral persisted bytes and modification dates used by the S3 component and real emulator process", + Why: "neutral persisted bytes, dates and transactional deletions used by the S3 component and real emulator process", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "complete provider-client lifecycles, checkpoint/reset, power-loss recovery and network filesystem locking", }, { Path: "internal/s3/", - Why: "bounded S3 prefix/host dispatch, listing/ranges, private Scaleway buckets and real-byte official-client proof", + Why: "bounded S3 prefix/host dispatch, listing/ranges/bulk deletion, private Scaleway buckets and official-client proof", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "one AWS/scw/exo version, not complete Exoscale lifecycles, public ACLs, version histories, TLS interception, cloud virtual hosts, ETags, multipart or public stacks", }, From 22dbc193b445d09db50301b40599c6d85b2cc03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 16:16:19 +0100 Subject: [PATCH 10/28] feat(s3): persist and prove multipart upload sessions Resume durable upload parts across restart, stream verified bounded assembly, and abort real dangling sessions through Exo bucket removal. Prove AWS/scw/exo lifecycles and 35 multipart mutations; preserve v1-v3 reads and document v4 rollback, quota and protocol ceilings.\n\nRefs #856. Assisted-by: GitHub Copilot (Copilot SDK in VS Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 15 +- CHANGELOG.md | 14 +- docs/confidence.md | 2 +- docs/limits.md | 15 +- docs/object-storage-routing.md | 80 ++++- .../cli/object_storage_conformance_test.go | 121 ++++++- .../core/objectstore/bucket_metadata_test.go | 2 +- internal/core/objectstore/manifest.go | 10 +- internal/core/objectstore/store.go | 6 +- internal/core/objectstore/store_test.go | 4 +- internal/core/objectstore/upload.go | 322 ++++++++++++++++++ .../core/objectstore/upload_integrity_test.go | 35 ++ internal/core/objectstore/upload_test.go | 251 ++++++++++++++ internal/s3/multipart.go | 220 ++++++++++++ internal/s3/multipart_list.go | 139 ++++++++ internal/s3/multipart_test.go | 200 +++++++++++ internal/s3/multipart_xml.go | 117 +++++++ internal/s3/server.go | 48 ++- internal/s3/server_test.go | 1 + mise.toml | 2 +- tools/falsify/specs/neutral-object-bytes.json | 4 +- tools/falsify/specs/s3-multipart.json | 268 +++++++++++++++ tools/testplan/rules.go | 8 +- 23 files changed, 1822 insertions(+), 62 deletions(-) create mode 100644 internal/core/objectstore/upload.go create mode 100644 internal/core/objectstore/upload_integrity_test.go create mode 100644 internal/core/objectstore/upload_test.go create mode 100644 internal/s3/multipart.go create mode 100644 internal/s3/multipart_list.go create mode 100644 internal/s3/multipart_test.go create mode 100644 internal/s3/multipart_xml.go create mode 100644 tools/falsify/specs/s3-multipart.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index f8284d5b..e5ca9683 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,12 +19,23 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **Le multipart persistant, prouvé avec Exo/AWS réels** (#856) : + exo 1.95.1 téléverse 6 340 608 octets en multipart ; AWS vérifie l'assemblage, + reprend les parties après redémarrage réel et termine un autre objet. + La suppression de bucket Exo abandonne une vraie session restante. + Le format privé v4 lit v1..v3 ; un retour à un ancien binaire demande une copie + complète du store avant migration. L'assemblage garde la limite du PUT simple ; + parties et assemblage partagent le quota, et l'abandon n'est pas du GC. + Les tags SHA-256 opaques ne promettent pas des ETag génériques. Création normale + de bucket Exo, usage natif, erreurs cloud mixtes et Object Storage 1.0 complet + restent hors de cette preuve. + - **La suppression en lot Exo, mesurée** (#856) : exo 1.95.1 supprime quatre vraies clés ; AWS vérifie l'inventaire vide après redémarrage et les octets Scaleway intacts. DeleteObjects vérifie checksum et XML fermé avant une seule transaction de métadonnées (1 à 1000 clés sans version, XML de 8 Mio maximum). Les réponses verbose/quiet décrivent des suppressions réelles. Les blobs restent - conservés ; multipart, erreurs cloud mixtes par clé et cycles complets de + conservés ; erreurs cloud mixtes par clé et cycles complets de création de buckets restent hors de cette tranche mesurée. - **Le listing Exo et les téléchargements multi-plages, mesurés** (#850) : @@ -35,7 +46,7 @@ change ni l'un ni l'autre a sa place dans `git log`. Le format privé v3 persiste les dates de publication, lit v1/v2 sans les réécrire et migre à la prochaine mutation ; les anciens binaires ne lisent pas v3. Une page contenant un ancien objet sans date est refusée jusqu'à sa réécriture. - ETag, multipart et cycles complets de buckets Exoscale restent hors périmètre. + ETag génériques et création complète de buckets Exoscale restent hors périmètre. - **Des hôtes S3 synthétiques monoport, prouvés par les uploads Exo** (#850) : `serve/start --s3-host-doors`, opt-in, conserve les préfixes et sert les diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6ee3e8..8c6c8d60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,22 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Persistent multipart, proved with real Exo/AWS clients** (#856): + exo 1.95.1 multipart-uploads 6,340,608 bytes; AWS verifies assembled bytes, + resumes persisted parts after actual restart and completes another object. + Exo bucket removal aborts a real dangling session before deleting the bucket. + Private metadata v4 reads v1..v3; older binaries need a pre-upgrade whole-store + copy for rollback. Assembly uses the simple-PUT limit; retained parts and + assembly share the blob quota, and abort is not GC. Opaque SHA-256 part tags + are not generic ETags. Normal Exo bucket creation, native usage, mixed per-key + cloud errors and full Object Storage 1.0 remain outside this proof. + - **Measured Exo bulk deletion** (#856): exo 1.95.1 deletes four actual keys; AWS independently checks the empty inventory across restart and untouched Scaleway bytes. DeleteObjects verifies transport checksums and closed XML before one metadata transaction (1..1000 unversioned keys, 8 MiB XML). Verbose/quiet results describe actual deletions. Blobs remain retained; - multipart, mixed per-key cloud errors and complete bucket workflows remain + mixed per-key cloud errors and complete bucket-creation workflows remain outside this measured slice. - **Measured Exo object listing and multi-range downloads** (#850): @@ -33,7 +43,7 @@ what this project is judged on: **a response shape a client can observe**, and Private metadata v3 persists publication dates, reads v1/v2 without rewriting them and upgrades on mutation; older binaries cannot read v3. Listing pages containing legacy undated objects are explicitly refused until overwrite. - ETags, multipart and complete Exoscale bucket workflows remain unsupported. + Generic ETags and complete Exoscale bucket-creation workflows remain unsupported. - **Same-port synthetic S3 hosts, proved with Exo uploads** (#850): opt-in `serve/start --s3-host-doors` retains the prefix doors and serves diff --git a/docs/confidence.md b/docs/confidence.md index 9954127e..eb5534ec 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -44,7 +44,7 @@ prints the reason under the pack that owns it. | Authentication, IAM permissions, a signature that must fail | **no** | [limits.md](limits.md#authentication-is-accepted-never-verified) | | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | -| Simple S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download/delete | `mise run conformance:s3:mounted` drives prefixed bytes, private Scaleway buckets, AWS pagination and host-routed Exo uploads/downloads (including multi-range 6,340,608 bytes) and bulk deletion, with actual bytes, dates and empty-inventory proof across restart; complete Exoscale bucket workflows are not certified — [object-storage-routing.md](object-storage-routing.md#measured-exoscale-bulk-deletion) | +| S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download/delete | `mise run conformance:s3:mounted` drives private Scaleway buckets, Exo multipart/range downloads (6,340,608 bytes), AWS restart/resume/completion, and Exo dangling-upload abort/bucket removal; actual bytes, dates and inventories are checked. Normal Exoscale bucket creation is not certified — [object-storage-routing.md](object-storage-routing.md#measured-persistent-multipart-uploads) | | Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | diff --git a/docs/limits.md b/docs/limits.md index b5090ff2..9ad61b6c 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -17,18 +17,27 @@ HTTP proxy. AWS retrieval proves the uploaded binary bytes and SHA-256 survive actual restart. Real Exo downloads also preserve a 6,340,608-byte object across multiple HTTP ranges. ListObjectsV2 exposes actual persisted modification dates; legacy undated objects must be overwritten before their listing can be served. -Private byte metadata v3 reads v1/v2 without inventing dates; successful mutations -upgrade it, and older binaries cannot read v3. +Exo now uploads that fixture through real multipart. AWS session/part inventories +survive restart, resume and complete with identical bytes; Exo bucket removal +aborts a real dangling session. Private byte metadata v4 reads v1..v3 without +inventing dates; successful mutations upgrade it, and older binaries cannot +read v4. Rollback requires a pre-upgrade copy of the whole owned store. Exo also bulk-deletes actual unversioned keys; AWS independently confirms the empty inventory survives restart while the other provider's bytes remain. DeleteObjects requires a verified transport checksum and closed XML, bounded to 1000 keys and 8 MiB. Store failures return an explicit whole-request Feint refusal, not fabricated per-key cloud errors; immutable blobs remain retained. +Multipart assembly has the same configured object limit as simple PUT +(256 MiB default). Retained parts and assembly share the blob quota (1 GiB +default), including peak reservation; abort is not GC. Opaque SHA-256 part tags +are not generic object ETags. Completion Location consumers are not certified; +same-key pagination markers require a live session. Exo's bucket remover does +not paginate uploads, so removal beyond 1000 sessions is not certified. This does not implement bucket virtual hosts, DNS/TLS interception or a complete Exoscale lifecycle; use a dedicated shell because its S3 environment replaces proxy settings. Complete Exoscale bucket workflows, ListObjects v1, owner requests, writable tags/public ACLs, -version histories, multipart, ETags, checkpoint/reset and hardcoded Terraform +version histories, multipart copy, generic ETags, checkpoint/reset and hardcoded Terraform redirection remain unsupported. No MinIO service or dependency is used by this implementation. diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 8a10b75f..9302b5aa 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -167,8 +167,9 @@ hostile-response-checksum controls. This certifies simple CreateBucket, PutObject, GetObject, HeadObject, DeleteObject and DeleteBucket through the mounted gateway, plus the bounded Scaleway bucket workflow below, not complete `exo` lifecycles. Unsupported -ancillary operations, missing-object cloud errors and multipart still receive -explicit Feint refusals. No ETag is invented. +ancillary operations and missing-object cloud errors still receive explicit +Feint refusals. Multipart is measured separately below; ordinary object reads +still do not advertise ETags. The snapshot-import route still records an intention rather than reading these bytes. Object Storage 1.0 and hardcoded Terraform redirection are not delivered. @@ -218,9 +219,9 @@ index. Existing bytes, generations and quotas remain unchanged. Legacy buckets without a stored creation date receive an explicit listing refusal rather than an invented date. Resource snapshots still do not own this metadata. -Complete Exoscale bucket workflows, their mandatory ListMultipartUploads during -bucket removal, native bucket-usage API, and full Object Storage 1.0 remain -separate follow-up work. No empty multipart success stub was added. +Complete Exoscale bucket creation, the native bucket-usage API, and full Object +Storage 1.0 remain separate follow-up work. Multipart inventory and bucket +removal now operate on real sessions, as measured below. ## Exoscale prefix incompatibility measured @@ -309,7 +310,8 @@ through the same scoped host. It checks actual listing keys, sizes and dates and compares downloaded binary bytes and SHA-256 before and after a real process restart. The small object is 73,728 bytes. A 6,340,608-byte object exceeds the client's 5 MiB download part size and produces multiple `206` responses. -That object is seeded with a **simple AWS PUT**, not an Exo multipart upload. +That object is now uploaded by **Exo multipart**, with independently verified +stored parts and assembled bytes, as described below. AWS additionally decodes two actual one-entry listing pages, including a literal key with escaped characters. @@ -330,25 +332,25 @@ Every range still verifies the **entire stored blob** before seeking. Partial responses deliberately omit whole-object checksum headers; full reads retain the verified SHA-256. ETags remain absent. -Private byte metadata is now **version 3**. New stores and successful mutations -write v3; v1/v2 are still readable and reads do not rewrite them. PUT persists +Publication dates were introduced in private metadata **version 3**; the current +multipart format is **v4** and still reads v1/v2/v3 without rewriting them. PUT persists its publication date, reused by GET/HEAD and listings across restart. Old undated object references retain no Last-Modified header, and pages containing them refuse listing until those objects are overwritten. Neither blob mtime nor -restart time is substituted. **Older binaries cannot read an upgraded v3 index**: +restart time is substituted. **Older binaries cannot read an upgraded v4 index**: keep a pre-upgrade copy of the whole owned store if rollback is required. Resource snapshots still do not own these bytes. The date, listing and range properties are replayed by `tools/falsify/specs/s3-download.json`. This milestone does **not** certify normal -Exo bucket creation/cleanup, ACL/CORS/ownership discovery, native bucket usage, -multipart, version histories, TLS or Terraform redirection. +Exo bucket creation, ACL/CORS/ownership discovery, native bucket usage, +version histories, TLS or Terraform redirection. ## Measured Exoscale bulk deletion The pinned exo 1.95.1 `storage delete --force --recursive --verbose sos://BUCKET/` enumerates keys with ListObjectsV2, then sends DeleteObjects. The mounted witness -failed with an actual 501 before this support. It now deletes four real keys, +failed with an actual 501 before this support. It now deletes five real keys, including a literal escaped key, and checks every reported key. An independent AWS CLI inventory is empty before and after another actual process restart; same-name objects in the Scaleway namespace still return their original bytes. @@ -367,9 +369,55 @@ store publishes one metadata transaction and retains immutable blobs and their quota charge: this is not GC. A store error yields an explicit logged whole-request Feint refusal, not invented mixed per-key cloud errors. An indeterminate directory-sync failure still faults the store until reopen. -The index remains v3. +The current multipart-enabled index is v4. `tools/falsify/specs/s3-delete.json` holds the destructive-input checks and -publication properties. This closes only the bulk-deletion slice of #856: -multipart upload/list/abort/complete and ordinary `exo storage rb` remain -unsupported, and the complete bucket workflow remains #858. +publication properties. Mixed per-key cloud errors remain outside this measured +slice of #856; the complete bucket-creation workflow remains #858. + +## Measured persistent multipart uploads + +`conformance:s3:mounted` drives **exo 1.95.1** uploading the 6,340,608-byte +fixture through its actual multipart uploader. AWS retrieves identical assembled +bytes and SHA-256. The stopped owned store contains the distinct last-part blob; +this is not a simple PUT disguised as multipart. + +AWS CLI **2.32.17** additionally initiates two real sessions and uploads parts. +Nonempty session and part inventories remain identical across an actual process +restart. AWS resumes one session, completes it and retrieves the expected bytes. +Finally, `exo storage rb --force` deletes five actual objects, lists and aborts +the real remaining session, then removes the bucket. The stopped store confirms +the bucket is absent; same-name Scaleway objects retain their original bytes. +This certifies removal of an existing bucket, not normal Exo bucket creation. + +The served subset is initiate, upload/replace part, list uploads, list parts, +complete and abort. Part numbers are 1..10000; selected completion parts must be +ascending and match the exact observed tag. Every selected part except the last +must be at least 5 MiB. Completion verifies each immutable part before streaming +the assembled blob and commits object publication and session removal together. +Failed completion preserves the old object and session. + +Sessions survive restart in private metadata **v4**. New stores and successful +mutations write v4; versions 1..3 remain readable and reads do not rewrite them. +Older binaries cannot read v4: rollback requires a pre-upgrade copy of the +**whole owned store**, not just its index. Resource snapshots still omit bytes. +The assembled object uses the configured simple-PUT limit, **256 MiB** by +default. Parts and assembly both count against the shared retained/reserved blob +budget, **1 GiB** by default, unless identical blobs deduplicate. Completion +requires sufficient peak reservation; abort and deletion remove references but +perform no GC and release no retained-byte charge. + +Part ETags are quoted, verified SHA-256 opaque tokens, not promises of MD5 or +generic object ETag semantics. Completion returns Bucket, Key and this opaque +ETag; it does not certify consumers requiring a Location URL. Upload pages sort +by key and initiation time, with ID as tie-breaker. A same-key upload marker +must identify a live session; missing/deleted markers are explicitly refused. +The pinned Exo bucket remover does not paginate upload inventory, so its proof +does not certify cleanup beyond one upload page (1000 sessions). Multipart +copy, optional checksum negotiation beyond the measured requests and provider +cloud-error dialects remain explicit Feint refusals, not invented successes. + +`tools/falsify/specs/s3-multipart.json` replays the input, scope, bound and +transactional-publication checks. Full Object Storage 1.0, native Exoscale usage, +ACL/CORS/ownership discovery, public stacks and TLS/Terraform interception remain +outside this delivery. diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index 916a2c5e..97a66cfd 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "net" "net/http" "net/http/httptest" @@ -38,6 +39,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("local proxy control did not record its refusal: %v hits=%d", err, guardHits.Load()) } aws := conformancetest.AWS(t, guard.URL, "") + aws.Env = append(aws.Env, "AWS_REQUEST_CHECKSUM_CALCULATION=when_required") binary := filepath.Join(t.TempDir(), "feint") buildCtx, cancelBuild := context.WithTimeout(t.Context(), 2*time.Minute) build := exec.CommandContext(buildCtx, "go", "build", "-o", binary, "../../cmd/feint") @@ -70,7 +72,11 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { return out } // Cleanup uses the recorded process identity, not a process-name kill. + stoppedAtEnd := false t.Cleanup(func() { + if stoppedAtEnd { + return + } ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() cmd := exec.CommandContext(ctx, binary, "stop", "--addr", addr) @@ -248,10 +254,63 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatal(err) } largeKey := "large-byte-proof" - // Seed a simple PUT, not an unsupported multipart upload. The Exo downloader - // must then read more than its 5 MiB part size through multiple byte ranges. - call(first.ObjectStorage.Doors[1], "put-object", "--bucket", bucket, "--key", largeKey, - "--body", largeInput, "--checksum-algorithm", "CRC32") + exo.Run(t, "storage", "upload", largeInput, "sos://"+bucket+"/"+largeKey) + getKey(first.ObjectStorage.Doors[1], largeKey, largePayload) + firstPart := filepath.Join(aws.Home, "multipart-first") + lastPart := filepath.Join(aws.Home, "multipart-last") + if err := os.WriteFile(firstPart, largePayload[:5<<20], 0600); err != nil { + t.Fatal(err) + } + resumeKey, cleanupKey := "resumed-upload", "abandoned-upload" + createUpload := func(objectKey string) string { + t.Helper() + var result struct{ UploadId string } + out := call(first.ObjectStorage.Doors[1], "create-multipart-upload", "--bucket", bucket, "--key", objectKey) + if err := json.Unmarshal(out, &result); err != nil || result.UploadId == "" { + t.Fatalf("real AWS initiation omitted its ID: %v %q", err, out) + } + return result.UploadId + } + putPart := func(objectKey, id, number, file string) string { + t.Helper() + var result struct{ ETag string } + out := call(first.ObjectStorage.Doors[1], "upload-part", "--bucket", bucket, "--key", objectKey, + "--upload-id", id, "--part-number", number, "--body", file, "--checksum-algorithm", "CRC32") + if err := json.Unmarshal(out, &result); err != nil || result.ETag == "" { + t.Fatalf("real AWS part response omitted its opaque tag: %v %q", err, out) + } + return result.ETag + } + resumeID, cleanupID := createUpload(resumeKey), createUpload(cleanupKey) + firstTag := putPart(resumeKey, resumeID, "1", firstPart) + if err := os.WriteFile(lastPart, []byte("resume after restart"), 0600); err != nil { + t.Fatal(err) + } + putPart(cleanupKey, cleanupID, "1", lastPart) + uploadInventory := func() []byte { + t.Helper() + return call(first.ObjectStorage.Doors[1], "list-multipart-uploads", "--bucket", bucket, "--no-paginate") + } + beforeUploads := uploadInventory() + var uploads struct { + Uploads []struct{ Key, UploadId string } + } + if err := json.Unmarshal(beforeUploads, &uploads); err != nil || len(uploads.Uploads) != 2 { + t.Fatalf("real AWS did not see two actual upload sessions: %v %q", err, beforeUploads) + } + beforeParts := call(first.ObjectStorage.Doors[1], "list-parts", "--bucket", bucket, "--key", resumeKey, + "--upload-id", resumeID, "--no-paginate") + var partInventory struct { + Parts []struct { + PartNumber int + ETag string + Size int64 + } + } + if err := json.Unmarshal(beforeParts, &partInventory); err != nil || len(partInventory.Parts) != 1 || + partInventory.Parts[0].Size != 5<<20 || partInventory.Parts[0].ETag != firstTag { + t.Fatalf("real AWS did not see the persisted nonempty part: %v %q", err, beforeParts) + } download := func(objectKey string, expected []byte) { t.Helper() destination := filepath.Join(exo.Home, "download-"+objectKey) @@ -335,6 +394,32 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if after := exo.Run(t, "--output-format", "json", "storage", "list", "sos://"+bucket+"/"); !bytes.Equal(after, beforeObjects) { t.Fatalf("actual Exo object inventory changed across restart: before=%q after=%q", beforeObjects, after) } + if after := uploadInventory(); !bytes.Equal(after, beforeUploads) { + t.Fatalf("upload sessions changed across actual restart: before=%q after=%q", beforeUploads, after) + } + afterParts := call(first.ObjectStorage.Doors[1], "list-parts", "--bucket", bucket, "--key", resumeKey, + "--upload-id", resumeID, "--no-paginate") + if !bytes.Equal(afterParts, beforeParts) { + t.Fatalf("part metadata changed across actual restart: before=%q after=%q", beforeParts, afterParts) + } + lastTag := putPart(resumeKey, resumeID, "2", lastPart) + type completionPart struct { + ETag string + PartNumber int + } + completion := struct{ Parts []completionPart }{[]completionPart{{firstTag, 1}, {lastTag, 2}}} + completionBody, err := json.Marshal(completion) + if err != nil { + t.Fatal(err) + } + completionFile := filepath.Join(aws.Home, "complete.json") + if err := os.WriteFile(completionFile, completionBody, 0600); err != nil { + t.Fatal(err) + } + call(first.ObjectStorage.Doors[1], "complete-multipart-upload", "--bucket", bucket, "--key", resumeKey, + "--upload-id", resumeID, "--multipart-upload", "file://"+completionFile) + resumedPayload := append(bytes.Clone(largePayload[:5<<20]), []byte("resume after restart")...) + getKey(first.ObjectStorage.Doors[1], resumeKey, resumedPayload) exo.Run(t, "storage", "upload", input, "sos://"+bucket+"/after-restart") getKey(first.ObjectStorage.Doors[1], "after-restart", []byte("other-provider-host-key")) var live statusReport @@ -362,6 +447,11 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { // Inspect only our private store while the owner is stopped: GET of the // same cleaned key twice would otherwise masquerade as key preservation. run("stop", "--addr", addr) + remainderDigest := sha256.Sum256(largePayload[5<<20:]) + info, err := os.Stat(filepath.Join(root, "blobs", hex.EncodeToString(remainderDigest[:]))) + if err != nil || info.Size() != int64(len(largePayload)-(5<<20)) { + t.Fatal("Exo did not leave the actual distinct multipart remainder bytes:", err) + } store, err := objectstore.OpenWithLimit(t.Context(), root, 20<<20) if err != nil { t.Fatal(err) @@ -390,9 +480,9 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("independent inventory did not witness %d actual objects: %v %q", want, err, out) } } - checkInventory(4) + checkInventory(5) deleted := exo.Run(t, "storage", "delete", "--force", "--recursive", "--verbose", "sos://"+bucket+"/") - for _, objectKey := range []string{key, exoKey, largeKey, "after-restart"} { + for _, objectKey := range []string{key, exoKey, largeKey, "after-restart", resumeKey} { if !bytes.Contains(deleted, []byte(objectKey+"\n")) { t.Fatalf("Exo did not report the actual deleted key %q: %q", objectKey, deleted) } @@ -429,7 +519,11 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { checkEmpty() for _, door := range first.ObjectStorage.Doors { if door.Provider == "exoscale" { - call(door, "delete-bucket", "--bucket", bucket) + if out := uploadInventory(); json.Unmarshal(out, &uploads) != nil || len(uploads.Uploads) != 1 || + uploads.Uploads[0].UploadId != cleanupID { + t.Fatalf("dangling-session positive control was lost: %q", out) + } + exo.Run(t, "storage", "rb", "--force", "sos://"+bucket) continue } call(door, "head-object", "--bucket", bucket, "--key", key) @@ -440,9 +534,20 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if guardHits.Load() != 1 { t.Fatalf("official client reached the refusing egress proxy: delta=%d", guardHits.Load()-1) } + run("stop", "--addr", addr) + stoppedAtEnd = true + finalStore, err := objectstore.OpenWithLimit(t.Context(), root, 20<<20) + if err != nil { + t.Fatal(err) + } + bucketErr := finalStore.HeadBucket(t.Context(), "exoscale/ch-gva-2/"+bucket) + if err := finalStore.Close(); err != nil || !errors.Is(bucketErr, objectstore.ErrBucketNotFound) { + t.Fatal("real Exo rb did not remove the actual bucket and its dangling session:", bucketErr, err) + } t.Logf("Exo downloads before/after actual restart: small=%d bytes; multi-range=%d bytes sha256=%x", len(payload), len(largePayload), sha256.Sum256(largePayload)) - t.Log("Exo bulk-deleted four actual keys; independent AWS inventory empty across restart; other provider bytes retained") + t.Log("Exo multipart-uploaded >5 MiB; AWS part/session state survived restart and completed with identical bytes; Exo rb aborted a real dangling upload") + t.Log("Exo bulk-deleted five actual keys; independent AWS inventory empty across restart; other provider bytes retained") t.Logf("actual feint pid %d -> %d; two same-port namespaces; preserved key; bytes=%d sha256=%s; stop/reopen/restart; proxy delta=0 (not a network namespace)", first.PID, restarted.PID, len(payload), hex.EncodeToString(sum[:])) } diff --git a/internal/core/objectstore/bucket_metadata_test.go b/internal/core/objectstore/bucket_metadata_test.go index 240bfc14..86a1f898 100644 --- a/internal/core/objectstore/bucket_metadata_test.go +++ b/internal/core/objectstore/bucket_metadata_test.go @@ -135,7 +135,7 @@ func TestLegacyBucketMetadataUpgradeAndValidation(t *testing.T) { if err := reopened.SetBucketMetadata(t.Context(), "bucket", "key", "value"); err != nil { t.Fatal(err) } - if reopened.state.Version != 3 { + if reopened.state.Version != storeVersion { t.Fatal("write did not upgrade metadata version") } if _, err := reopened.HeadObject(t.Context(), "bucket", "key"); err != nil { diff --git a/internal/core/objectstore/manifest.go b/internal/core/objectstore/manifest.go index dcd0a7b8..e3c1f52c 100644 --- a/internal/core/objectstore/manifest.go +++ b/internal/core/objectstore/manifest.go @@ -14,7 +14,7 @@ import ( const ( storeFormat = "feint-objectstore" - storeVersion = 3 + storeVersion = 4 maxManifestBytes = 64 << 20 ) @@ -64,6 +64,14 @@ func decodeManifest(body []byte) (manifest, error) { if err := validMetadata(Object{Metadata: b.Metadata}); err != nil { return manifest{}, errors.Join(ErrCorrupt, err) } + if state.Version < 4 && len(b.Uploads) != 0 { + return manifest{}, fmt.Errorf("%w: upload sessions require metadata version 4", ErrCorrupt) + } + for id, upload := range b.Uploads { + if err := validUpload(id, upload); err != nil { + return manifest{}, errors.Join(ErrCorrupt, err) + } + } for key, obj := range b.Objects { if state.Version < 3 && !obj.Modified.IsZero() { return manifest{}, fmt.Errorf("%w: object modification dates require metadata version 3", ErrCorrupt) diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index 43f2b936..e2fdd90e 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -60,6 +60,7 @@ type bucket struct { Generation string `json:"generation"` Objects map[string]Object `json:"objects"` Metadata map[string]string `json:"metadata,omitempty"` + Uploads map[string]Upload `json:"uploads,omitempty"` } type manifest struct { @@ -190,7 +191,7 @@ func (s *Store) DeleteBucket(ctx context.Context, name string) error { if err != nil { return err } - if len(b.Objects) != 0 { + if len(b.Objects) != 0 || len(b.Uploads) != 0 { return ErrBucketNotEmpty } next := s.clone() @@ -401,7 +402,8 @@ func (s *Store) lookup(name string) (*bucket, error) { func (s *Store) clone() manifest { next := manifest{Format: s.state.Format, Version: storeVersion, Buckets: make(map[string]*bucket)} for name, b := range s.state.Buckets { - next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects), Metadata: maps.Clone(b.Metadata)} + next.Buckets[name] = &bucket{Generation: b.Generation, Objects: maps.Clone(b.Objects), + Metadata: maps.Clone(b.Metadata), Uploads: maps.Clone(b.Uploads)} } return next } diff --git a/internal/core/objectstore/store_test.go b/internal/core/objectstore/store_test.go index 53b66f9b..92f5b891 100644 --- a/internal/core/objectstore/store_test.go +++ b/internal/core/objectstore/store_test.go @@ -753,7 +753,7 @@ func TestManifestIsClosedVersionedAndValidated(t *testing.T) { } for _, body := range []string{ `null`, `{}`, `{"format":"operator","version":1,"buckets":{}}`, - `{"format":"feint-objectstore","version":4,"buckets":{}}`, + fmt.Sprintf(`{"format":"feint-objectstore","version":%d,"buckets":{}}`, storeVersion+1), `{"format":"feint-objectstore","version":1,"buckets":null}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":null}}`, `{"format":"feint-objectstore","version":1,"buckets":{"b":{"generation":"","objects":{}}}}`, @@ -820,7 +820,7 @@ func TestFailedOpenReleasesOwnershipLock(t *testing.T) { if err != nil { t.Fatal(err) } - if err := os.WriteFile(index, []byte(`{"format":"feint-objectstore","version":4,"buckets":{}}`), 0o600); err != nil { + if err := os.WriteFile(index, []byte(fmt.Sprintf(`{"format":"feint-objectstore","version":%d,"buckets":{}}`, storeVersion+1)), 0o600); err != nil { t.Fatal(err) } if broken, err := Open(t.Context(), path); !errors.Is(err, ErrCorrupt) || broken != nil { diff --git a/internal/core/objectstore/upload.go b/internal/core/objectstore/upload.go new file mode 100644 index 00000000..183c1a6f --- /dev/null +++ b/internal/core/objectstore/upload.go @@ -0,0 +1,322 @@ +package objectstore + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "maps" + "math" + "os" + "slices" + "strings" + "time" + + "github.com/stephrobert/feint/internal/core/serialise" +) + +// ErrUploadNotFound includes a session belonging to another logical key. +var ErrUploadNotFound = errors.New("upload session not found") + +// Upload holds durable, unpublished parts. IDs are the enclosing inventory keys. +// Abandoning or completing a session removes references, never immutable blobs. +type Upload struct { + Key string `json:"key"` + Created time.Time `json:"created"` + ContentType string `json:"content_type"` + Metadata map[string]string `json:"metadata"` + Parts map[int]Object `json:"parts"` +} + +// PartRef selects a part and the exact content digest the caller observed. +type PartRef struct { + Number int + Digest string +} + +// CreateUpload durably opens a session without publishing an object. +func (s *Store) CreateUpload(ctx context.Context, name, key string, options PutOptions) (string, error) { + if err := validKey(key); err != nil { + return "", err + } + if options.Size != 0 { + return "", fmt.Errorf("%w: an upload session has no initial bytes", ErrInvalid) + } + if err := validMetadata(Object{ContentType: options.ContentType, Metadata: options.Metadata}); err != nil { + return "", err + } + end, err := s.begin(ctx) + if err != nil { + return "", err + } + defer end() + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.lookup(name); err != nil { + return "", err + } + id := rand.Text() + next := s.clone() + b := next.Buckets[name] + if b.Uploads == nil { + b.Uploads = make(map[string]Upload) + } + b.Uploads[id] = Upload{Key: key, Created: time.Now().UTC(), ContentType: options.ContentType, + Metadata: maps.Clone(options.Metadata), Parts: make(map[int]Object)} + if err := s.commit(ctx, next); err != nil { + return "", err + } + return id, nil +} + +// ListUploads returns independent session and part metadata. +func (s *Store) ListUploads(ctx context.Context, name string) (map[string]Upload, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + b, err := s.lookup(name) + if err != nil { + return nil, err + } + result := make(map[string]Upload, len(b.Uploads)) + for id, upload := range b.Uploads { + result[id] = copyUpload(upload) + } + return result, nil +} + +// ReadUpload requires both the session ID and its original logical key. +func (s *Store) ReadUpload(ctx context.Context, name, key, id string) (Upload, error) { + end, err := s.begin(ctx) + if err != nil { + return Upload{}, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + upload, err := s.upload(name, key, id) + return copyUpload(upload), err +} + +// PutPart streams and verifies bytes before conditionally publishing a part. +// One session serializes part replacement, completion and abandonment. +func (s *Store) PutPart(ctx context.Context, name, key, id string, number int, body io.Reader, size int64) (Object, error) { + if number < 1 || number > 10000 || body == nil || size < 0 || size == math.MaxInt64 { + return Object{}, fmt.Errorf("%w: invalid upload part", ErrInvalid) + } + end, err := s.begin(ctx) + if err != nil { + return Object{}, err + } + defer end() + release := serialise.Lock(s.uploadLock(name, id)) + defer release() + s.mu.RLock() + _, err = s.upload(name, key, id) + s.mu.RUnlock() + if err != nil { + return Object{}, err + } + obj := Object{Size: size} + obj.Digest, err = s.writeBlob(ctx, body, size) + if err != nil { + return Object{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + upload, err := s.upload(name, key, id) + if err != nil { + return Object{}, err + } + upload = copyUpload(upload) + obj.Modified = time.Now().UTC() + upload.Parts[number] = obj + next := s.clone() + next.Buckets[name].Uploads[id] = upload + if err := s.commit(ctx, next); err != nil { + return Object{}, err + } + return obj, nil +} + +// AbortUpload removes a session but retains the bytes and their quota charge. +func (s *Store) AbortUpload(ctx context.Context, name, key, id string) error { + end, err := s.begin(ctx) + if err != nil { + return err + } + defer end() + release := serialise.Lock(s.uploadLock(name, id)) + defer release() + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.upload(name, key, id); err != nil { + return err + } + next := s.clone() + delete(next.Buckets[name].Uploads, id) + return s.commit(ctx, next) +} + +// CompleteUpload assembles verified selected parts within maxBytes and commits +// the object and session removal together. A failed assembly leaves the session. +func (s *Store) CompleteUpload(ctx context.Context, name, key, id string, refs []PartRef, maxBytes int64) (_ Object, err error) { + if len(refs) == 0 || len(refs) > 10000 || maxBytes <= 0 || maxBytes == math.MaxInt64 { + return Object{}, fmt.Errorf("%w: invalid assembly bounds", ErrInvalid) + } + refs = slices.Clone(refs) + end, err := s.begin(ctx) + if err != nil { + return Object{}, err + } + defer end() + release := serialise.Lock(s.uploadLock(name, id)) + defer release() + releaseKey := serialise.Lock(s.domain + "/" + name + "\x00" + key) + defer releaseKey() + s.mu.RLock() + upload, err := s.upload(name, key, id) + s.mu.RUnlock() + if err != nil { + return Object{}, err + } + var parts []Object + var size int64 + previous := 0 + for _, ref := range refs { + part, ok := upload.Parts[ref.Number] + if ref.Number <= previous || !ok || ref.Digest != part.Digest { + return Object{}, fmt.Errorf("%w: part order or observed digest does not match", ErrInvalid) + } + if part.Size > maxBytes-size { + return Object{}, fmt.Errorf("%w: assembled object exceeds its byte limit", ErrInvalid) + } + size += part.Size + previous = ref.Number + parts = append(parts, part) + } + body := &partSequence{store: s, ctx: ctx, parts: parts} + defer func() { err = errors.Join(err, body.Close()) }() + obj := Object{Size: size, ContentType: upload.ContentType, Metadata: maps.Clone(upload.Metadata)} + obj.Digest, err = s.writeBlob(ctx, body, size) + if err != nil { + return Object{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.upload(name, key, id); err != nil { + return Object{}, err + } + obj.Modified = time.Now().UTC() + next := s.clone() + next.Buckets[name].Objects[key] = obj + delete(next.Buckets[name].Uploads, id) + if err := s.commit(ctx, next); err != nil { + return Object{}, err + } + return copyObject(obj), nil +} + +func (s *Store) uploadLock(name, id string) string { + return "upload\x00" + s.domain + "/" + name + "\x00" + id +} + +// Called only under mu; the returned maps remain immutable until cloned. +func (s *Store) upload(name, key, id string) (Upload, error) { + b, err := s.lookup(name) + if err != nil { + return Upload{}, err + } + upload, ok := b.Uploads[id] + if !ok || upload.Key != key { + return Upload{}, ErrUploadNotFound + } + return upload, nil +} + +func copyUpload(upload Upload) Upload { + upload.Metadata = maps.Clone(upload.Metadata) + upload.Parts = maps.Clone(upload.Parts) + for number, part := range upload.Parts { + upload.Parts[number] = copyObject(part) + } + return upload +} + +func validUpload(id string, upload Upload) error { + if id == "" || len(id) > 128 || strings.IndexFunc(id, func(r rune) bool { + return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && + (r < '0' || r > '9') && r != '-' && r != '_' + }) >= 0 || upload.Created.IsZero() || upload.Created.Year() < 1 || upload.Created.Year() > 9999 || + upload.Parts == nil || len(upload.Parts) > 10000 { + return fmt.Errorf("%w: invalid persisted upload session", ErrInvalid) + } + if err := validKey(upload.Key); err != nil { + return err + } + if err := validMetadata(Object{ContentType: upload.ContentType, Metadata: upload.Metadata}); err != nil { + return err + } + for number, part := range upload.Parts { + if number < 1 || number > 10000 || !validDigest(part.Digest) || part.Size < 0 || part.Size == math.MaxInt64 || + part.Modified.IsZero() || part.Modified.Year() < 1 || part.Modified.Year() > 9999 { + return fmt.Errorf("%w: invalid persisted part reference", ErrInvalid) + } + if err := validMetadata(part); err != nil { + return err + } + } + return nil +} + +// Only one verified part file is open at a time, even for a large part list. +type partSequence struct { + store *Store + ctx context.Context + parts []Object + file *os.File +} + +func (r *partSequence) Read(p []byte) (int, error) { + for { + if err := r.ctx.Err(); err != nil { + return 0, err + } + if r.file == nil { + if len(r.parts) == 0 { + return 0, io.EOF + } + var err error + r.file, err = r.store.openBlob(r.ctx, r.parts[0]) + if err != nil { + return 0, err + } + r.parts = r.parts[1:] + } + n, err := r.file.Read(p) + if errors.Is(err, io.EOF) { + if err := r.Close(); err != nil { + return n, err + } + if n == 0 { + continue + } + return n, nil + } + return n, err + } +} + +func (r *partSequence) Close() error { + if r.file == nil { + return nil + } + err := r.file.Close() + r.file = nil + return err +} diff --git a/internal/core/objectstore/upload_integrity_test.go b/internal/core/objectstore/upload_integrity_test.go new file mode 100644 index 00000000..76acdae2 --- /dev/null +++ b/internal/core/objectstore/upload_integrity_test.go @@ -0,0 +1,35 @@ +package objectstore + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestCorruptPartCannotPublishAssembledObject(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + old := mustPut(t, s, "bucket", "key", "old") + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "blobs", part.Digest), []byte("bad"), 0600); err != nil { + t.Fatal(err) + } + if _, err := s.CompleteUpload(t.Context(), "bucket", "key", id, []PartRef{{1, part.Digest}}, 3); !errors.Is(err, ErrCorrupt) { + t.Fatal("corrupt part completed successfully:", err) + } + if obj, err := s.HeadObject(t.Context(), "bucket", "key"); err != nil || obj.Digest != old.Digest { + t.Fatal("corrupt assembly replaced old object:", obj, err) + } + if _, err := s.ReadUpload(t.Context(), "bucket", "key", id); err != nil { + t.Fatal("corrupt assembly lost the resumable session:", err) + } +} diff --git a/internal/core/objectstore/upload_test.go b/internal/core/objectstore/upload_test.go new file mode 100644 index 00000000..5a0c1de5 --- /dev/null +++ b/internal/core/objectstore/upload_test.go @@ -0,0 +1,251 @@ +package objectstore + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestUploadPartsResumeAndCompleteWithoutAliasing(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + metadata := map[string]string{"purpose": "resume"} + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{ContentType: "application/octet-stream", Metadata: metadata}) + if err != nil { + t.Fatal(err) + } + metadata["purpose"] = "caller changed" + a, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + before, err := s.ReadUpload(t.Context(), "bucket", "key", id) + if err != nil || before.Metadata["purpose"] != "resume" { + t.Fatal(before, err) + } + changed, err := s.ListUploads(t.Context(), "bucket") + if err != nil { + t.Fatal(err) + } + delete(changed[id].Parts, 1) + changed[id].Metadata["purpose"] = "aliased" + current, err := s.ReadUpload(t.Context(), "bucket", "key", id) + if err != nil || current.Metadata["purpose"] != "resume" || len(current.Parts) != 1 { + t.Fatal("returned inventory aliases the live session:", current, err) + } + if _, err := s.HeadObject(t.Context(), "bucket", "key"); !errors.Is(err, ErrObjectNotFound) { + t.Fatal("uncompleted session published an object:", err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }() + after, err := s.ReadUpload(t.Context(), "bucket", "key", id) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatal("restart changed durable upload state:", before, after, err) + } + b, err := s.PutPart(t.Context(), "bucket", "key", id, 2, bytes.NewBufferString("defg"), 4) + if err != nil { + t.Fatal(err) + } + obj, err := s.CompleteUpload(t.Context(), "bucket", "key", id, []PartRef{{1, a.Digest}, {2, b.Digest}}, 7) + if err != nil || obj.Size != 7 || obj.Metadata["purpose"] != "resume" { + t.Fatal(obj, err) + } + _, body, err := s.GetObject(t.Context(), "bucket", "key") + if err != nil { + t.Fatal(err) + } + got, readErr := io.ReadAll(body) + if err := errors.Join(readErr, body.Close()); err != nil || string(got) != "abcdefg" { + t.Fatal("assembled wrong bytes:", string(got), err) + } + if _, err := s.ReadUpload(t.Context(), "bucket", "key", id); !errors.Is(err, ErrUploadNotFound) { + t.Fatal("completed session survived:", err) + } +} + +func TestUploadCompletionRefusalsPreserveSessionAndOldObject(t *testing.T) { + for _, name := range []string{"scope", "digest", "order", "missing", "limit", "canceled", "commit"} { + t.Run(name, func(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "bucket") + old := mustPut(t, s, "bucket", "key", "old") + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + a, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + refs, key, limit, ctx := []PartRef{{1, a.Digest}}, "key", int64(3), t.Context() + switch name { + case "scope": + key = "other" + case "digest": + refs[0].Digest = old.Digest + case "order": + refs = append(refs, refs[0]) + limit = 6 + case "missing": + refs[0].Number = 2 + case "limit": + limit = 2 + case "canceled": + canceled, cancel := context.WithCancel(ctx) + cancel() + ctx = canceled + case "commit": + index, saved := filepath.Join(path, "index.json"), filepath.Join(path, "saved-index") + if err := os.Rename(index, saved); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(index, 0700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Remove(index); err != nil { + t.Error(err) + } + if err := os.Rename(saved, index); err != nil { + t.Error(err) + } + }) + } + if _, err := s.CompleteUpload(ctx, "bucket", key, id, refs, limit); err == nil { + t.Fatal("invalid completion accepted") + } + if obj, err := s.HeadObject(t.Context(), "bucket", "key"); err != nil || obj.Digest != old.Digest { + t.Fatal("failed completion replaced old object:", obj, err) + } + if upload, err := s.ReadUpload(t.Context(), "bucket", "key", id); err != nil || len(upload.Parts) != 1 { + t.Fatal("failed completion lost parts:", upload, err) + } + }) + } +} + +func TestUploadQuotaAndAbortKeepRetainedBytes(t *testing.T) { + path := filepath.Join(privateDir(t), "objects") + s, err := OpenWithLimit(t.Context(), path, 4) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }() + mustBucket(t, s, "bucket") + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + if _, err := s.CompleteUpload(t.Context(), "bucket", "key", id, []PartRef{{1, part.Digest}}, 3); !errors.Is(err, ErrQuota) { + t.Fatal("assembly bypassed shared reservation:", err) + } + if err := s.DeleteBucket(t.Context(), "bucket"); !errors.Is(err, ErrBucketNotEmpty) { + t.Fatal("bucket deletion ignored active sessions:", err) + } + if err := s.AbortUpload(t.Context(), "bucket", "key", id); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(path, "blobs", part.Digest)); err != nil { + t.Fatal("abort reclaimed immutable bytes:", err) + } + if _, err := s.PutObject(t.Context(), "bucket", "other", bytes.NewBufferString("xy"), PutOptions{Size: 2}); !errors.Is(err, ErrQuota) { + t.Fatal("abort released retained byte charge:", err) + } + if err := s.DeleteBucket(t.Context(), "bucket"); err != nil { + t.Fatal("aborted session still blocked deletion:", err) + } +} + +func TestRestoredUploadMetadataIsClosedAndValidated(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "bucket") + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"legacy", "identifier", "date", "nil-parts", "digest", "number", "size", "part-date"} { + t.Run(name, func(t *testing.T) { + state := s.clone() + upload := copyUpload(state.Buckets["bucket"].Uploads[id]) + targetID, badPart := id, part + switch name { + case "legacy": + state.Version = 3 + case "identifier": + delete(state.Buckets["bucket"].Uploads, id) + targetID = "../foreign" + case "date": + upload.Created = time.Time{} + case "nil-parts": + upload.Parts = nil + case "digest": + badPart.Digest = "../../foreign" + upload.Parts[1] = badPart + case "number": + upload.Parts[0] = badPart + case "size": + badPart.Size = -1 + upload.Parts[1] = badPart + case "part-date": + badPart.Modified = time.Time{} + upload.Parts[1] = badPart + } + state.Buckets["bucket"].Uploads[targetID] = upload + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if _, err := decodeManifest(raw); !errors.Is(err, ErrCorrupt) { + t.Fatal("unsafe upload metadata admitted:", name, err) + } + }) + } +} + +func TestFailedPartChecksumCannotReplaceDurablePart(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "bucket") + id, err := s.CreateUpload(t.Context(), "bucket", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + old, err := s.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + if _, err := s.PutPart(context.Background(), "bucket", "key", id, 1, bytes.NewBufferString("x"), 3); err == nil { + t.Fatal("truncated replacement accepted") + } + if upload, err := s.ReadUpload(t.Context(), "bucket", "key", id); err != nil || upload.Parts[1].Digest != old.Digest { + t.Fatal("failed part replaced durable metadata:", upload, err) + } +} diff --git a/internal/s3/multipart.go b/internal/s3/multipart.go new file mode 100644 index 00000000..55d491da --- /dev/null +++ b/internal/s3/multipart.go @@ -0,0 +1,220 @@ +package s3 + +import ( + "crypto/md5" //nolint:gosec // Legacy UploadPart transport integrity, not authentication or an ETag. + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +func multipartQueryKey(key string) bool { + switch key { + case "uploads", "uploadId", "partNumber", "max-uploads", "key-marker", "upload-id-marker", + "prefix", "max-parts", "part-number-marker", "x-id": + return true + } + return false +} + +func (s *Server) multipart(w http.ResponseWriter, r *http.Request, bucket, key string, q url.Values) { + if _, err := listText(key, ""); err != nil { + s.refuse(w, r, http.StatusBadRequest, "multipart keys must be lossless XML text", err) + return + } + if len(r.Header.Values("Range")) != 0 { + s.refuse(w, r, http.StatusNotImplemented, "Range requires an object byte GET, not multipart inventory", nil) + return + } + operation := "" + allowed := map[string]bool{"x-id": true} + switch { + case q.Has("uploads") && !q.Has("uploadId") && q.Get("uploads") == "" && key != "" && r.Method == http.MethodPost: + operation, allowed["uploads"] = "CreateMultipartUpload", true + case q.Has("uploads") && !q.Has("uploadId") && q.Get("uploads") == "" && key == "" && r.Method == http.MethodGet: + operation, allowed["uploads"] = "ListMultipartUploads", true + for _, name := range []string{"max-uploads", "key-marker", "upload-id-marker", "prefix"} { + allowed[name] = true + } + case q.Get("uploadId") != "" && !q.Has("uploads") && key != "": + allowed["uploadId"] = true + switch r.Method { + case http.MethodPut: + operation, allowed["partNumber"] = "UploadPart", true + case http.MethodGet: + operation = "ListParts" + allowed["max-parts"], allowed["part-number-marker"] = true, true + case http.MethodPost: + operation = "CompleteMultipartUpload" + case http.MethodDelete: + operation = "AbortMultipartUpload" + } + } + if operation == "" { + s.refuse(w, r, http.StatusNotImplemented, "multipart operation is not supported", nil) + return + } + for name, values := range q { + if !allowed[name] || len(values) != 1 || + (name == "x-id" && values[0] != operation) || len(values[0]) > 1024 { + s.refuse(w, r, http.StatusBadRequest, "invalid or unsupported multipart query", nil) + return + } + } + switch operation { + case "CreateMultipartUpload": + options, err := requestMetadata(r) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart metadata", err) + return + } + body, err := xmlBody(w, r, 1) + if err != nil || len(body) != 0 { + s.refuse(w, r, http.StatusBadRequest, "multipart initiation requires an empty body", err) + return + } + id, err := s.store.CreateUpload(r.Context(), s.name(bucket), key, options) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + s.writeXML(w, r, http.StatusOK, initiatedUpload{Bucket: bucket, Key: key, ID: id}) + case "UploadPart": + s.uploadPart(w, r, bucket, key, q) + case "ListMultipartUploads": + s.listUploads(w, r, bucket, q) + case "ListParts": + s.listParts(w, r, bucket, key, q) + case "CompleteMultipartUpload": + s.completeUpload(w, r, bucket, key, q.Get("uploadId")) + case "AbortMultipartUpload": + if err := s.store.AbortUpload(r.Context(), s.name(bucket), key, q.Get("uploadId")); err != nil { + s.storeError(w, r, bucket, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +type initiatedUpload struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"` + Bucket string + Key string + ID string `xml:"UploadId"` +} + +type completedUpload struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"` + Bucket string + Key string + ETag string +} + +func partTag(digest string) string { return `"` + digest + `"` } + +// The SDK defines ETags as opaque entity tags, not necessarily MD5. Multipart +// tokens identify verified SHA-256 content; no MD5 or generic ETag claim is made. +func (s *Server) uploadPart(w http.ResponseWriter, r *http.Request, bucket, key string, q url.Values) { + number, err := multipartNumber(q, "partNumber", 0, 1, 10000) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart part number", err) + return + } + upload, size, err := newUpload(w, r, s.cfg.MaxObjectBytes) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart payload", err) + return + } + var body io.Reader = upload + if values := r.Header.Values("Content-MD5"); len(values) != 0 { + sum, err := base64.StdEncoding.DecodeString(values[0]) + if len(values) != 1 || err != nil || len(sum) != md5.Size { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart Content-MD5", err) + return + } + body = &partChecksum{reader: upload, hash: md5.New(), expected: values[0]} //nolint:gosec // Verify the legacy transport checksum before publication. + } + part, err := s.store.PutPart(r.Context(), s.name(bucket), key, q.Get("uploadId"), number, body, size) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + if !s.digestHeader(w, r, part) { + return + } + w.Header().Set("ETag", partTag(part.Digest)) + w.Header().Set("X-Amz-Checksum-Crc32", upload.crc32()) + w.WriteHeader(http.StatusOK) +} + +type partChecksum struct { + reader io.Reader + hash hash.Hash + expected string +} + +func (r *partChecksum) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if _, hashErr := r.hash.Write(p[:n]); hashErr != nil { + return n, hashErr + } + if errors.Is(err, io.EOF) && base64.StdEncoding.EncodeToString(r.hash.Sum(nil)) != r.expected { + return n, fmt.Errorf("%w: multipart Content-MD5 mismatch", errUpload) + } + return n, err +} + +func multipartNumber(q url.Values, name string, fallback, min, max int) (int, error) { + if !q.Has(name) { + if fallback < min { + return 0, errors.New("missing numeric multipart query") + } + return fallback, nil + } + raw := q.Get(name) + if raw == "" || strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 { + return 0, errors.New("multipart query requires decimal digits") + } + n, err := strconv.Atoi(raw) + if err != nil || n < min || n > max { + return 0, errors.New("multipart query exceeds its bounds") + } + return n, nil +} + +func (s *Server) completeUpload(w http.ResponseWriter, r *http.Request, bucket, key, id string) { + body, err := xmlBody(w, r, maxDeleteXMLBytes) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart completion body", err) + return + } + refs, err := completionParts(body) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart completion XML", err) + return + } + upload, err := s.store.ReadUpload(r.Context(), s.name(bucket), key, id) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + for i, ref := range refs { + part, ok := upload.Parts[ref.Number] + if !ok || ref.Digest != part.Digest || (i < len(refs)-1 && part.Size < 5<<20) { + s.refuse(w, r, http.StatusBadRequest, "multipart parts or minimum size do not match", nil) + return + } + } + obj, err := s.store.CompleteUpload(r.Context(), s.name(bucket), key, id, refs, s.cfg.MaxObjectBytes) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + s.writeXML(w, r, http.StatusOK, completedUpload{Bucket: bucket, Key: key, ETag: partTag(obj.Digest)}) +} diff --git a/internal/s3/multipart_list.go b/internal/s3/multipart_list.go new file mode 100644 index 00000000..aeea7e7a --- /dev/null +++ b/internal/s3/multipart_list.go @@ -0,0 +1,139 @@ +package s3 + +import ( + "encoding/xml" + "errors" + "net/http" + "net/url" + "slices" + "strings" + "time" +) + +type listedUpload struct { + Key string + ID string `xml:"UploadId"` + Initiated string + StorageClass string +} + +type uploadInventory struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"` + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKey string `xml:"NextKeyMarker,omitempty"` + NextID string `xml:"NextUploadIdMarker,omitempty"` + Prefix string + Max int `xml:"MaxUploads"` + Truncated bool `xml:"IsTruncated"` + Uploads []listedUpload `xml:"Upload"` +} + +func (s *Server) listUploads(w http.ResponseWriter, r *http.Request, bucket string, q url.Values) { + max, err := multipartNumber(q, "max-uploads", 1000, 1, 1000) + prefix, prefixErr := listText(q.Get("prefix"), "") + keyMarker, markerErr := listText(q.Get("key-marker"), "") + if err != nil || (q.Get("upload-id-marker") != "" && q.Get("key-marker") == "") || + prefixErr != nil || markerErr != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid multipart inventory bounds or XML text", errors.Join(err, prefixErr, markerErr)) + return + } + uploads, err := s.store.ListUploads(r.Context(), s.name(bucket)) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + result := uploadInventory{Bucket: bucket, Max: max, Prefix: prefix, + KeyMarker: keyMarker, UploadIDMarker: q.Get("upload-id-marker")} + markerUpload, hasMarker := uploads[result.UploadIDMarker] + if result.UploadIDMarker != "" && (!hasMarker || markerUpload.Key != result.KeyMarker) { + s.refuse(w, r, http.StatusBadRequest, "multipart marker no longer names a live upload for its key", nil) + return + } + var entries []listedUpload + for id, upload := range uploads { + if !strings.HasPrefix(upload.Key, result.Prefix) { + continue + } + if upload.Key < result.KeyMarker || (upload.Key == result.KeyMarker && + (result.UploadIDMarker == "" || upload.Created.Before(markerUpload.Created) || + (upload.Created.Equal(markerUpload.Created) && id <= result.UploadIDMarker))) { + continue + } + if _, err := listText(upload.Key, ""); err != nil { + s.refuse(w, r, http.StatusNotImplemented, "multipart inventory contains an XML-inexpressible key", err) + return + } + entries = append(entries, listedUpload{Key: upload.Key, ID: id, + Initiated: upload.Created.UTC().Format(time.RFC3339Nano), StorageClass: "STANDARD"}) + } + slices.SortFunc(entries, func(a, b listedUpload) int { + if a.Key != b.Key { + return strings.Compare(a.Key, b.Key) + } + if order := uploads[a.ID].Created.Compare(uploads[b.ID].Created); order != 0 { + return order + } + return strings.Compare(a.ID, b.ID) + }) + result.Truncated = len(entries) > max + if result.Truncated { + result.NextKey, result.NextID = entries[max-1].Key, entries[max-1].ID + entries = entries[:max] + } + result.Uploads = entries + s.writeXML(w, r, http.StatusOK, result) +} + +type listedPart struct { + Number int `xml:"PartNumber"` + Modified string `xml:"LastModified"` + ETag string + Size int64 +} + +type partInventory struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"` + Bucket string + Key string + ID string `xml:"UploadId"` + Marker int `xml:"PartNumberMarker"` + Next int `xml:"NextPartNumberMarker,omitempty"` + Max int `xml:"MaxParts"` + Truncated bool `xml:"IsTruncated"` + Parts []listedPart `xml:"Part"` +} + +func (s *Server) listParts(w http.ResponseWriter, r *http.Request, bucket, key string, q url.Values) { + max, err := multipartNumber(q, "max-parts", 1000, 1, 1000) + marker, markerErr := multipartNumber(q, "part-number-marker", 0, 0, 10000) + if err != nil || markerErr != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid part inventory bounds", errors.Join(err, markerErr)) + return + } + upload, err := s.store.ReadUpload(r.Context(), s.name(bucket), key, q.Get("uploadId")) + if err != nil { + s.storeError(w, r, bucket, err) + return + } + var numbers []int + for number := range upload.Parts { + if number > marker { + numbers = append(numbers, number) + } + } + slices.Sort(numbers) + result := partInventory{Bucket: bucket, Key: key, ID: q.Get("uploadId"), Marker: marker, Max: max, + Truncated: len(numbers) > max} + if result.Truncated { + result.Next = numbers[max-1] + numbers = numbers[:max] + } + for _, number := range numbers { + part := upload.Parts[number] + result.Parts = append(result.Parts, listedPart{Number: number, Size: part.Size, + ETag: partTag(part.Digest), Modified: part.Modified.UTC().Format(time.RFC3339Nano)}) + } + s.writeXML(w, r, http.StatusOK, result) +} diff --git a/internal/s3/multipart_test.go b/internal/s3/multipart_test.go new file mode 100644 index 00000000..5bb79136 --- /dev/null +++ b/internal/s3/multipart_test.go @@ -0,0 +1,200 @@ +package s3 + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "fmt" + "net/url" + "strings" + "testing" +) + +func beginMultipart(t *testing.T, s *Server, key string) string { + t.Helper() + w := request(t, s, "POST", "/bucket/"+url.PathEscape(key)+"?uploads", nil, nil) + var result initiatedUpload + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &result) != nil || result.ID == "" || result.Key != key { + t.Fatal("initiation did not publish a scoped session:", w.Code, w.Body.String()) + } + return result.ID +} + +func completedPartXML(number int, digest string) string { + return fmt.Sprintf(`%d"%s"`, number, digest) +} + +func TestMultipartLifecycleUsesRealPartsAndOpaqueTags(t *testing.T) { + s, store := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + id := beginMultipart(t, s, "a//../%2F?&") + path := "/bucket/" + url.PathEscape("a//../%2F?&") + "?uploadId=" + id + payload := []byte("real part bytes") + sum := sha256.Sum256(payload) + digest := hex.EncodeToString(sum[:]) + w := request(t, s, "PUT", path+"&partNumber=1", payload, map[string]string{"Content-MD5": deleteMD5(payload)}) + if w.Code != 200 || w.Header().Get("ETag") != partTag(digest) { + t.Fatal("uploaded tag is not the verified content token:", w.Code, w.Header(), w.Body.String()) + } + w = request(t, s, "GET", path, nil, nil) + var parts partInventory + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &parts) != nil || len(parts.Parts) != 1 || + parts.Parts[0].Size != int64(len(payload)) || parts.Parts[0].ETag != partTag(digest) || parts.Parts[0].Modified == "" { + t.Fatal("part inventory not derived from durable bytes:", w.Code, w.Body.String()) + } + body := []byte("" + completedPartXML(1, digest) + "") + w = request(t, s, "POST", path, body, nil) + var completed completedUpload + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &completed) != nil || completed.ETag != partTag(digest) { + t.Fatal("completion failed:", w.Code, w.Body.String()) + } + w = request(t, s, "GET", "/bucket/"+url.PathEscape("a//../%2F?&"), nil, nil) + if w.Code != 200 || !bytes.Equal(w.Body.Bytes(), payload) { + t.Fatal("completion lied about bytes:", w.Code, w.Body.String()) + } + uploads, err := store.ListUploads(t.Context(), "bucket") + if err != nil || len(uploads) != 0 { + t.Fatal("completed upload still listed:", uploads, err) + } +} + +func TestMultipartInventoryPaginationScopesAndAbort(t *testing.T) { + s, store := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + first, second := beginMultipart(t, s, "key"), beginMultipart(t, s, "key") + w := request(t, s, "GET", "/bucket?uploads&max-uploads=1", nil, nil) + var page uploadInventory + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Uploads) != 1 || + !page.Truncated || page.Uploads[0].ID != first || page.NextID != first { + t.Fatal("uploads are not ordered by initiation time:", w.Code, w.Body.String()) + } + w = request(t, s, "GET", "/bucket?uploads&max-uploads=1&key-marker=key&upload-id-marker="+first, nil, nil) + page = uploadInventory{} + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Uploads) != 1 || + page.Truncated || page.Uploads[0].ID != second { + t.Fatal("upload cursor did not advance:", w.Code, w.Body.String()) + } + if _, err := store.ReadUpload(t.Context(), "bucket", "wrong", first); err == nil { + t.Fatal("upload ID crossed its key scope") + } + w = request(t, s, "DELETE", "/bucket/key?uploadId="+first, nil, nil) + if w.Code != 204 { + t.Fatal(w.Code, w.Body.String()) + } + w = request(t, s, "GET", "/bucket?uploads", nil, nil) + page = uploadInventory{} + if w.Code != 200 || xml.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Uploads) != 1 || page.Uploads[0].ID != second { + t.Fatal("abort removed another session:", w.Code, w.Body.String()) + } +} + +func TestMultipartRefusalsPreserveDurableParts(t *testing.T) { + for _, name := range []string{"checksum", "number-zero", "number-large", "number-sign", "scope", "duplicate-query", "range", "prefix-xml", "marker-xml", "limit", "small", "tag", "order"} { + t.Run(name, func(t *testing.T) { + s, store := fixture(t) + request(t, s, "PUT", "/bucket", nil, nil) + id := beginMultipart(t, s, "key") + a, err := store.PutPart(t.Context(), "bucket", "key", id, 1, bytes.NewBufferString("abc"), 3) + if err != nil { + t.Fatal(err) + } + path := "/bucket/key?uploadId=" + id + method, body, headers := "PUT", []byte("new"), map[string]string{} + path += "&partNumber=1" + switch name { + case "checksum": + headers["Content-MD5"] = deleteMD5([]byte("wrong")) + case "number-zero": + path = "/bucket/key?uploadId=" + id + "&partNumber=0" + case "number-large": + path = "/bucket/key?uploadId=" + id + "&partNumber=10001" + case "number-sign": + path = "/bucket/key?uploadId=" + id + "&partNumber=%2B1" + case "scope": + path = "/bucket/other?uploadId=" + id + "&partNumber=1" + case "duplicate-query": + path += "&partNumber=1" + case "range": + method, path, body = "GET", "/bucket/key?uploadId="+id, nil + headers["Range"] = "bytes=0-1" + case "prefix-xml": + method, path, body = "GET", "/bucket?uploads&prefix=%01", nil + case "marker-xml": + method, path, body = "GET", "/bucket?uploads&key-marker=%01", nil + case "limit", "small", "tag", "order": + method, path = "POST", "/bucket/key?uploadId="+id + ref := completedPartXML(1, a.Digest) + switch name { + case "limit": + s.cfg.MaxObjectBytes = 2 + case "small": + b, err := store.PutPart(t.Context(), "bucket", "key", id, 2, bytes.NewBufferString("def"), 3) + if err != nil { + t.Fatal(err) + } + ref += completedPartXML(2, b.Digest) + case "tag": + ref = completedPartXML(1, strings.Repeat("0", 64)) + case "order": + ref += ref + } + body = []byte("" + ref + "") + } + w := request(t, s, method, path, body, headers) + upload, err := store.ReadUpload(t.Context(), "bucket", "key", id) + if w.Code < 400 || err != nil || upload.Parts[1].Digest != a.Digest { + t.Fatal("refusal lost or replaced durable part:", name, w.Code, upload, err) + } + }) + } +} + +func TestCompletionXMLClosedShape(t *testing.T) { + part := completedPartXML(1, strings.Repeat("a", 64)) + valid := "" + part + "" + if refs, err := completionParts([]byte(valid)); err != nil || len(refs) != 1 { + t.Fatal("accepting completion was refused:", refs, err) + } + for _, body := range []string{ + ``, + valid + valid, + valid + ``, + strings.Replace(valid, "", ``, 1), + strings.Replace(valid, "", ``, 1), + strings.Replace(valid, "", `"`+strings.Repeat("a", 64)+`"`, 1), + strings.Replace(valid, "", `1`, 1), + strings.Replace(valid, `1`, "", 1), + strings.Replace(valid, `"`+strings.Repeat("a", 64)+`"`, "", 1), + strings.ReplaceAll(valid, strings.Repeat("a", 64), strings.Repeat("g", 64)), + strings.Replace(valid, "1", "+1", 1), + strings.Replace(valid, "1", "10001", 1), + "" + valid, + strings.ReplaceAll(valid, "CompleteMultipartUpload", "bad:CompleteMultipartUpload"), + } { + if _, err := completionParts([]byte(body)); err == nil { + t.Fatal("open or invalid completion shape admitted:", body) + } + } +} + +func FuzzMultipartCompletionClosedShape(f *testing.F) { + f.Add("" + completedPartXML(1, strings.Repeat("a", 64)) + "") + f.Add("") + f.Fuzz(func(t *testing.T, body string) { + if len(body) > maxDeleteXMLBytes { + return + } + refs, err := completionParts([]byte(body)) + if err == nil { + if len(refs) == 0 || len(refs) > 10000 { + t.Fatal("empty or unbounded completion") + } + for i, ref := range refs { + if ref.Number < 1 || ref.Number > 10000 || (i > 0 && ref.Number <= refs[i-1].Number) { + t.Fatal("invalid accepted part ordering") + } + } + } + }) +} diff --git a/internal/s3/multipart_xml.go b/internal/s3/multipart_xml.go new file mode 100644 index 00000000..ff429072 --- /dev/null +++ b/internal/s3/multipart_xml.go @@ -0,0 +1,117 @@ +package s3 + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "io" + "strconv" + "strings" + + "github.com/stephrobert/feint/internal/core/objectstore" +) + +func completionParts(body []byte) ([]objectstore.PartRef, error) { + decoder := xml.NewDecoder(bytes.NewReader(body)) //nolint:gosec // Bounded input; directives and unrecognised elements are rejected. + var refs []objectstore.PartRef + var stack []string + var text strings.Builder + var part objectstore.PartRef + roots, numbers, tags := 0, 0, 0 + declaration := false + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + switch t := token.(type) { + case xml.StartElement: + if t.Name.Space != "" && t.Name.Space != s3XMLNamespace { + return nil, errors.New("invalid multipart XML namespace") + } + for _, attr := range t.Attr { + if len(stack) != 0 || attr.Name.Space != "" || attr.Name.Local != "xmlns" || attr.Value != s3XMLNamespace { + return nil, errors.New("invalid multipart XML attribute") + } + } + switch len(stack) { + case 0: + if roots != 0 || t.Name.Local != "CompleteMultipartUpload" { + return nil, errors.New("one multipart completion root is required") + } + roots++ + case 1: + if t.Name.Local != "Part" { + return nil, errors.New("unsupported multipart completion member") + } + part, numbers, tags = objectstore.PartRef{}, 0, 0 + case 2: + switch t.Name.Local { + case "PartNumber": + numbers++ + case "ETag": + tags++ + default: + return nil, errors.New("unsupported completed part member") + } + text.Reset() + default: + return nil, errors.New("nested completed part field") + } + stack = append(stack, t.Name.Local) + case xml.EndElement: + switch t.Name.Local { + case "PartNumber": + raw := text.String() + if strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 { + return nil, errors.New("part number requires decimal digits") + } + part.Number, err = strconv.Atoi(raw) + if err != nil || part.Number < 1 || part.Number > 10000 { + return nil, errors.New("part number exceeds its bounds") + } + case "ETag": + raw := text.String() + if len(raw) != 66 || raw[0] != '"' || raw[len(raw)-1] != '"' { + return nil, errors.New("completed part requires its opaque uploaded tag") + } + part.Digest = raw[1 : len(raw)-1] + digest, err := hex.DecodeString(part.Digest) + if err != nil || len(digest) != sha256.Size { + return nil, errors.New("invalid completed part tag") + } + case "Part": + if numbers != 1 || tags != 1 || (len(refs) != 0 && part.Number <= refs[len(refs)-1].Number) { + return nil, errors.New("one number and tag per ascending part are required") + } + refs = append(refs, part) + if len(refs) > 10000 { + return nil, errors.New("too many completed parts") + } + } + stack = stack[:len(stack)-1] + case xml.CharData: + if len(stack) == 3 { + text.Write(t) + } else if strings.TrimSpace(string(t)) != "" { + return nil, errors.New("unexpected multipart completion text") + } + case xml.ProcInst: + if roots != 0 || t.Target != "xml" || declaration { + return nil, errors.New("invalid multipart completion instruction") + } + declaration = true + default: + return nil, errors.New("unsupported multipart completion token") + } + } + if len(refs) == 0 || len(stack) != 0 { + return nil, errors.New("multipart completion requires parts") + } + return refs, nil +} diff --git a/internal/s3/server.go b/internal/s3/server.go index 2a4b204c..54d86fe2 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -110,7 +110,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") listing := r.Method == http.MethodGet && rawKey == "" && query.Get("list-type") == "2" bulkDelete := r.Method == http.MethodPost && rawKey == "" && query.Has("delete") + multipart := query.Has("uploads") || query.Has("uploadId") for key := range query { + if multipart && multipartQueryKey(key) { + continue + } if listing && listQueryKey(key) { continue } @@ -129,7 +133,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.refuse(w, r, http.StatusNotImplemented, "S3 bucket header feature is not implemented", err) return } - if bulkDelete { + if bulkDelete || (multipart && r.Method == http.MethodPut) { checked = checked.Clone(r.Context()) checked.Header.Del("Content-MD5") } @@ -154,6 +158,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.listObjects(w, r, bucket, query) return } + if multipart { + s.multipart(w, r, bucket, key, query) + return + } if bulkDelete { s.deleteObjects(w, r, bucket, query) return @@ -240,18 +248,9 @@ func (s *Server) bucket(w http.ResponseWriter, r *http.Request, name string) { } func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) { - metadata := make(map[string]string) - for name, values := range r.Header { - if suffix, ok := strings.CutPrefix(strings.ToLower(name), "x-amz-meta-"); ok { - if len(values) != 1 || !headerName(suffix) || !headerValue(values[0]) { - s.refuse(w, r, http.StatusBadRequest, "invalid object metadata header", nil) - return - } - metadata[suffix] = values[0] - } - } - if !headerValue(r.Header.Get("Content-Type")) { - s.refuse(w, r, http.StatusBadRequest, "invalid content type", nil) + options, err := requestMetadata(r) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "invalid object metadata header or content type", err) return } upload, size, err := newUpload(w, r, s.cfg.MaxObjectBytes) @@ -259,9 +258,8 @@ func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) s.refuse(w, r, http.StatusBadRequest, "invalid S3 upload", err) return } - obj, err := s.store.PutObject(r.Context(), s.name(bucket), key, upload, objectstore.PutOptions{ - Size: size, ContentType: r.Header.Get("Content-Type"), Metadata: metadata, - }) + options.Size = size + obj, err := s.store.PutObject(r.Context(), s.name(bucket), key, upload, options) if err != nil { s.storeError(w, r, bucket, err) return @@ -274,6 +272,22 @@ func (s *Server) put(w http.ResponseWriter, r *http.Request, bucket, key string) w.WriteHeader(http.StatusOK) } +func requestMetadata(r *http.Request) (objectstore.PutOptions, error) { + metadata := make(map[string]string) + for name, values := range r.Header { + if suffix, ok := strings.CutPrefix(strings.ToLower(name), "x-amz-meta-"); ok { + if len(values) != 1 || !headerName(suffix) || !headerValue(values[0]) { + return objectstore.PutOptions{}, errors.New("invalid object metadata header") + } + metadata[suffix] = values[0] + } + } + if !headerValue(r.Header.Get("Content-Type")) { + return objectstore.PutOptions{}, errors.New("invalid content type") + } + return objectstore.PutOptions{ContentType: r.Header.Get("Content-Type"), Metadata: metadata}, nil +} + func (s *Server) get(w http.ResponseWriter, r *http.Request, bucket, key string) { obj, body, err := s.store.GetObject(r.Context(), s.name(bucket), key) if err != nil { @@ -363,7 +377,7 @@ func (s *Server) storeError(w http.ResponseWriter, r *http.Request, bucket strin switch { case errors.Is(err, objectstore.ErrQuota): status, message = http.StatusInsufficientStorage, "object storage blob byte budget exceeded" - case errors.Is(err, objectstore.ErrObjectNotFound), errors.Is(err, objectstore.ErrBucketExists), + case errors.Is(err, objectstore.ErrObjectNotFound), errors.Is(err, objectstore.ErrUploadNotFound), errors.Is(err, objectstore.ErrBucketExists), errors.Is(err, objectstore.ErrBucketNotEmpty): status, message = http.StatusNotImplemented, "cloud error shape has not been measured for this store outcome" case errors.As(err, &tooLarge): diff --git a/internal/s3/server_test.go b/internal/s3/server_test.go index 8cf7667e..dcab27c8 100644 --- a/internal/s3/server_test.go +++ b/internal/s3/server_test.go @@ -194,6 +194,7 @@ func TestRefusalsDoNotMasqueradeAsCloudSupport(t *testing.T) { {"GET", "/bucket/missing", nil}, {"PUT", "/bucket", nil}, {"PUT", "/bucket/key?uploads", nil}, + {"PUT", "/bucket/key?future-feature=", nil}, {"PUT", "/bucket/key", map[string]string{"X-Amz-Acl": "public-read"}}, {"GET", "/bucket/key", map[string]string{"Range": "items=0-1"}}, {"PUT", "/bucket/key", map[string]string{"If-None-Match": "*"}}, diff --git a/mise.toml b/mise.toml index b3e873da..137e9497 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove AWS/scw bytes, listing and host-routed exo 1.95.1 upload/range download/bulk deletion across actual restart" +description = "Prove AWS/scw/exo bytes, multipart resume/abort, listing/ranges and bulk deletion across actual restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:exo:endpoint"] diff --git a/tools/falsify/specs/neutral-object-bytes.json b/tools/falsify/specs/neutral-object-bytes.json index 66fb9eca..435b51ec 100644 --- a/tools/falsify/specs/neutral-object-bytes.json +++ b/tools/falsify/specs/neutral-object-bytes.json @@ -39,8 +39,8 @@ { "label": "a nonempty bucket can be deleted", "file": "internal/core/objectstore/store.go", - "find": "if len(b.Objects) != 0 {", - "replace": "if len(b.Objects) != 0 && false {", + "find": "if len(b.Objects) != 0 || len(b.Uploads) != 0 {", + "replace": "if (len(b.Objects) != 0 && false) || len(b.Uploads) != 0 {", "test": "TestEmptyObjectsAndBucketLifecycle" }, { diff --git a/tools/falsify/specs/s3-multipart.json b/tools/falsify/specs/s3-multipart.json new file mode 100644 index 00000000..ba4ffc18 --- /dev/null +++ b/tools/falsify/specs/s3-multipart.json @@ -0,0 +1,268 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "multipart inventory silently changes an XML-inexpressible prefix", + "file": "internal/s3/multipart_list.go", + "find": "prefixErr != nil || markerErr != nil", + "replace": "(prefixErr != nil && false) || markerErr != nil", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "multipart inventory silently changes an XML-inexpressible marker", + "file": "internal/s3/multipart_list.go", + "find": "prefixErr != nil || markerErr != nil", + "replace": "prefixErr != nil || (markerErr != nil && false)", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "multipart assembly trusts same-length corrupt part bytes", + "file": "internal/core/objectstore/files.go", + "find": "if n != obj.Size || hex.EncodeToString(hash.Sum(nil)) != obj.Digest {", + "replace": "if n != obj.Size || (hex.EncodeToString(hash.Sum(nil)) != obj.Digest && false) {", + "package": "./internal/core/objectstore/", + "test": "TestCorruptPartCannotPublishAssembledObject" + }, + { + "label": "returned session metadata aliases live state", + "file": "internal/core/objectstore/upload.go", + "find": "upload.Metadata = maps.Clone(upload.Metadata)", + "replace": "if upload.Metadata == nil { upload.Metadata = maps.Clone(upload.Metadata) }", + "package": "./internal/core/objectstore/", + "test": "TestUploadPartsResumeAndCompleteWithoutAliasing" + }, + { + "label": "returned part inventory aliases live state", + "file": "internal/core/objectstore/upload.go", + "find": "upload.Parts = maps.Clone(upload.Parts)", + "replace": "if upload.Parts == nil { upload.Parts = maps.Clone(upload.Parts) }", + "package": "./internal/core/objectstore/", + "test": "TestUploadPartsResumeAndCompleteWithoutAliasing" + }, + { + "label": "session removal modifies live state before durable commit", + "file": "internal/core/objectstore/store.go", + "find": "Uploads: maps.Clone(b.Uploads)", + "replace": "Uploads: func() map[string]Upload { _ = maps.Clone(b.Uploads); return b.Uploads }()", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "multipart silently accepts duplicate selectors", + "file": "internal/s3/multipart.go", + "find": "if !allowed[name] || len(values) != 1 ||", + "replace": "if !allowed[name] || (len(values) != 1 && false) ||", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "multipart inventory silently accepts Range", + "file": "internal/s3/multipart.go", + "find": "if len(r.Header.Values(\"Range\")) != 0 {", + "replace": "if len(r.Header.Values(\"Range\")) != 0 && false {", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "part Content-MD5 mismatches publish replacement metadata", + "file": "internal/s3/multipart.go", + "find": "if errors.Is(err, io.EOF) && base64.StdEncoding.EncodeToString(r.hash.Sum(nil)) != r.expected {", + "replace": "if errors.Is(err, io.EOF) && base64.StdEncoding.EncodeToString(r.hash.Sum(nil)) != r.expected && false {", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "signed decimal part numbers are silently accepted", + "file": "internal/s3/multipart.go", + "find": "raw == \"\" || strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0", + "replace": "raw == \"\" || (strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 && false)", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "non-final parts smaller than 5 MiB complete", + "file": "internal/s3/multipart.go", + "find": "(i < len(refs)-1 && part.Size < 5<<20)", + "replace": "(i < len(refs)-1 && part.Size < 5<<20 && false)", + "test": "TestMultipartRefusalsPreserveDurableParts" + }, + { + "label": "the uploaded part response loses its observed content tag", + "file": "internal/s3/multipart.go", + "find": "w.Header().Set(\"ETag\", partTag(part.Digest))", + "replace": "w.Header().Set(\"ETag\", partTag(part.Digest[:32]))", + "test": "TestMultipartLifecycleUsesRealPartsAndOpaqueTags" + }, + { + "label": "completion XML accepts a foreign namespace", + "file": "internal/s3/multipart_xml.go", + "find": "if t.Name.Space != \"\" && t.Name.Space != s3XMLNamespace {", + "replace": "if t.Name.Space != \"\" && t.Name.Space != s3XMLNamespace && false {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML silently discards unknown attributes", + "file": "internal/s3/multipart_xml.go", + "find": "if len(stack) != 0 || attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace {", + "replace": "if (len(stack) != 0 || attr.Name.Space != \"\" || attr.Name.Local != \"xmlns\" || attr.Value != s3XMLNamespace) && false {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML accepts a trailing root", + "file": "internal/s3/multipart_xml.go", + "find": "if roots != 0 || t.Name.Local != \"CompleteMultipartUpload\" {", + "replace": "if (roots != 0 && false) || t.Name.Local != \"CompleteMultipartUpload\" {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML accepts signed part numbers", + "file": "internal/s3/multipart_xml.go", + "find": "if strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 {", + "replace": "if strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 && false {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML exceeds its part-number bound", + "file": "internal/s3/multipart_xml.go", + "find": "err != nil || part.Number < 1 || part.Number > 10000", + "replace": "err != nil || part.Number < 1 || (part.Number > 10000 && false)", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML accepts malformed content tags", + "file": "internal/s3/multipart_xml.go", + "find": "if err != nil || len(digest) != sha256.Size {", + "replace": "if (err != nil || len(digest) != sha256.Size) && false {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML silently chooses among duplicate part numbers", + "file": "internal/s3/multipart_xml.go", + "find": "if numbers != 1 || tags != 1 ||", + "replace": "if (numbers != 1 && false) || tags != 1 ||", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "completion XML silently chooses among duplicate content tags", + "file": "internal/s3/multipart_xml.go", + "find": "if numbers != 1 || tags != 1 ||", + "replace": "if numbers != 1 || (tags != 1 && false) ||", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "empty completion is acknowledged", + "file": "internal/s3/multipart_xml.go", + "find": "if len(refs) == 0 || len(stack) != 0 {", + "replace": "if (len(refs) == 0 && false) || len(stack) != 0 {", + "test": "TestCompletionXMLClosedShape" + }, + { + "label": "upload IDs authorize another logical object key", + "file": "internal/core/objectstore/upload.go", + "find": "if !ok || upload.Key != key {", + "replace": "if !ok || (upload.Key != key && false) {", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "assembly ignores the exact observed part digest", + "file": "internal/core/objectstore/upload.go", + "find": "ref.Number <= previous || !ok || ref.Digest != part.Digest", + "replace": "ref.Number <= previous || !ok || (ref.Digest != part.Digest && false)", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "assembly permits duplicate or descending part numbers", + "file": "internal/core/objectstore/upload.go", + "find": "ref.Number <= previous || !ok || ref.Digest != part.Digest", + "replace": "(ref.Number <= previous && false) || !ok || ref.Digest != part.Digest", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "the assembled-object byte bound is ignored", + "file": "internal/core/objectstore/upload.go", + "find": "if part.Size > maxBytes-size {", + "replace": "if part.Size > maxBytes-size && false {", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "a failed completion publishes its new object before commit", + "file": "internal/core/objectstore/upload.go", + "find": "delete(next.Buckets[name].Uploads, id)\n\tif err := s.commit(ctx, next); err != nil {", + "replace": "delete(next.Buckets[name].Uploads, id)\n\ts.state = next\n\tif err := s.commit(ctx, next); err != nil {", + "package": "./internal/core/objectstore/", + "test": "TestUploadCompletionRefusalsPreserveSessionAndOldObject" + }, + { + "label": "completed sessions remain in the persisted inventory", + "file": "internal/core/objectstore/upload.go", + "find": "delete(next.Buckets[name].Uploads, id)\n\tif err := s.commit(ctx, next); err != nil {", + "replace": "if id == \"\\x00never\" { delete(next.Buckets[name].Uploads, id) }\n\tif err := s.commit(ctx, next); err != nil {", + "package": "./internal/core/objectstore/", + "test": "TestUploadPartsResumeAndCompleteWithoutAliasing" + }, + { + "label": "bucket deletion ignores unfinished upload sessions", + "file": "internal/core/objectstore/store.go", + "find": "if len(b.Objects) != 0 || len(b.Uploads) != 0 {", + "replace": "if len(b.Objects) != 0 || (len(b.Uploads) != 0 && false) {", + "package": "./internal/core/objectstore/", + "test": "TestUploadQuotaAndAbortKeepRetainedBytes" + }, + { + "label": "restored legacy versions admit session metadata", + "file": "internal/core/objectstore/manifest.go", + "find": "if state.Version < 4 && len(b.Uploads) != 0 {", + "replace": "if state.Version < 4 && len(b.Uploads) != 0 && false {", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored sessions admit malformed identifiers", + "file": "internal/core/objectstore/upload.go", + "find": "}) >= 0 || upload.Created.IsZero()", + "replace": "}) >= 0 && false || upload.Created.IsZero()", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored sessions invent an initiation date", + "file": "internal/core/objectstore/upload.go", + "find": "upload.Created.IsZero() || upload.Created.Year() < 1", + "replace": "(upload.Created.IsZero() || upload.Created.Year() < 1) && false", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored sessions admit a missing parts map", + "file": "internal/core/objectstore/upload.go", + "find": "upload.Parts == nil || len(upload.Parts) > 10000", + "replace": "(upload.Parts == nil && false) || len(upload.Parts) > 10000", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored part references permit path traversal digests", + "file": "internal/core/objectstore/upload.go", + "find": "!validDigest(part.Digest) || part.Size < 0", + "replace": "(!validDigest(part.Digest) && false) || part.Size < 0", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored parts permit invalid part numbers", + "file": "internal/core/objectstore/upload.go", + "find": "if number < 1 || number > 10000 || !validDigest(part.Digest)", + "replace": "if (number < 1 && false) || number > 10000 || !validDigest(part.Digest)", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + }, + { + "label": "restored parts permit negative lengths", + "file": "internal/core/objectstore/upload.go", + "find": "!validDigest(part.Digest) || part.Size < 0 || part.Size == math.MaxInt64", + "replace": "!validDigest(part.Digest) || (part.Size < 0 && false) || part.Size == math.MaxInt64", + "package": "./internal/core/objectstore/", + "test": "TestRestoredUploadMetadataIsClosedAndValidated" + } + ] +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index bb14b5af..49c3e2da 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -95,7 +95,7 @@ var rules = []rule{ Path: "internal/providers/exoscale/object_storage.go", Why: "declared SOS dialect and configurable endpoint variables, outside control-plane SDK coverage", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, - Unproven: "AWS CLI byte operations, not a complete exo lifecycle, bucket-usage API or multipart support", + Unproven: "pinned AWS/exo bytes and multipart resume/abort, not normal Exo bucket creation or the native bucket-usage API", }, { Path: "internal/providers/scaleway/", @@ -254,15 +254,15 @@ var rules = []rule{ }, { Path: "internal/core/objectstore/", - Why: "neutral persisted bytes, dates and transactional deletions used by the S3 component and real emulator process", + Why: "neutral persisted bytes, dates, multipart sessions and transactional deletions used by the real emulator process", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, Unproven: "complete provider-client lifecycles, checkpoint/reset, power-loss recovery and network filesystem locking", }, { Path: "internal/s3/", - Why: "bounded S3 prefix/host dispatch, listing/ranges/bulk deletion, private Scaleway buckets and official-client proof", + Why: "bounded S3 prefix/host dispatch, multipart, listing/ranges/bulk deletion and official-client proof", Runs: []string{"conformance:s3", "conformance:s3:mounted"}, - Unproven: "one AWS/scw/exo version, not complete Exoscale lifecycles, public ACLs, version histories, TLS interception, cloud virtual hosts, ETags, multipart or public stacks", + Unproven: "one AWS/scw/exo version, not complete Exoscale lifecycles, public ACLs, version histories, TLS interception, cloud virtual hosts, generic ETags or public stacks", }, { Path: "internal/conformancetest/", From a601a595c56abb0efac51a1c161a58429fdd4b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Sat, 7 Nov 2026 17:27:24 +0100 Subject: [PATCH 11/28] feat(objectstore): add durable named checkpoints and explicit garbage collection Keep full validated manifests including upload sessions in the exclusively owned store root. Verify blobs before reset, exclude active mutations through the lifecycle gate, and protect checkpoint references during explicit quota-aware collection. Assisted-by: GitHub Copilot (model not exposed by agent runtime) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- internal/core/objectstore/checkpoint.go | 261 ++++++++++++++ .../checkpoint_concurrency_test.go | 132 ++++++++ internal/core/objectstore/checkpoint_test.go | 320 ++++++++++++++++++ internal/core/objectstore/files.go | 12 +- internal/core/objectstore/gc.go | 192 +++++++++++ internal/core/objectstore/gc_test.go | 226 +++++++++++++ internal/core/objectstore/manifest.go | 6 +- internal/core/objectstore/store.go | 6 +- .../specs/objectstore-checkpoints.json | 61 ++++ 9 files changed, 1204 insertions(+), 12 deletions(-) create mode 100644 internal/core/objectstore/checkpoint.go create mode 100644 internal/core/objectstore/checkpoint_concurrency_test.go create mode 100644 internal/core/objectstore/checkpoint_test.go create mode 100644 internal/core/objectstore/gc.go create mode 100644 internal/core/objectstore/gc_test.go create mode 100644 tools/falsify/specs/objectstore-checkpoints.json diff --git a/internal/core/objectstore/checkpoint.go b/internal/core/objectstore/checkpoint.go new file mode 100644 index 00000000..51c2366b --- /dev/null +++ b/internal/core/objectstore/checkpoint.go @@ -0,0 +1,261 @@ +package objectstore + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" +) + +// ErrCheckpointNotFound means the requested durable name does not exist. +var ErrCheckpointNotFound = errors.New("object store checkpoint not found") + +// beginExclusive shares Close's lifecycle boundary. TestCheckpointExclusiveGate +// holds both halves: streaming operations retain readers; resets take a writer. +// Waiting for the lock, like Close, cannot interrupt a blocked caller's Read. +func (s *Store) beginExclusive(ctx context.Context) (func(), error) { + s.life.Lock() + end := s.life.Unlock + if err := ctx.Err(); err != nil { + end() + return nil, err + } + if s.closed { + end() + return nil, ErrClosed + } + s.mu.RLock() + err := s.fault + s.mu.RUnlock() + if err != nil { + end() + return nil, err + } + return end, nil +} + +func validCheckpointName(name string) bool { + if len(name) == 0 || len(name) > 64 { + return false + } + for _, c := range name { + if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && + (c < '0' || c > '9') && c != '-' && c != '_' { + return false + } + } + return true +} + +// checkpointRoot is called only under life's writer lock. Legacy stores acquire +// the optional private directory lazily, without changing manifest versions. +func (s *Store) checkpointRoot(create bool) (*os.Root, error) { + info, err := s.root.Lstat("checkpoints") + if errors.Is(err, os.ErrNotExist) && create { + if err := s.root.Mkdir("checkpoints", 0o700); err != nil { + return nil, fmt.Errorf("create checkpoint directory: %w", err) + } + info, err = s.root.Lstat("checkpoints") + } + if err != nil { + return nil, err + } + if err := private(info, true); err != nil { + return nil, fmt.Errorf("checkpoint directory: %w", errors.Join(ErrCorrupt, err)) + } + root, err := s.root.OpenRoot("checkpoints") + if err != nil { + return nil, err + } + opened, err := root.Stat(".") + if err == nil { + err = private(opened, true) + } + if err == nil && !os.SameFile(info, opened) { + err = fmt.Errorf("%w: checkpoint directory changed while opening", ErrCorrupt) + } + // TestCheckpointDirectorySyncFailureIsReportedAndRetried holds the parent + // sync, including retry after an earlier mkdir succeeded but its sync failed. + if err == nil && create { + err = s.dir.Sync() + if err != nil { + err = fmt.Errorf("sync checkpoint directory creation: %w", err) + } + } + if err != nil { + return nil, errors.Join(err, root.Close()) + } + return root, nil +} + +func (s *Store) verifyManifest(ctx context.Context, state manifest) error { + for _, b := range state.Buckets { + for _, obj := range b.Objects { + if err := s.verifyReference(ctx, obj); err != nil { + return err + } + } + for _, upload := range b.Uploads { + for _, part := range upload.Parts { + if err := s.verifyReference(ctx, part); err != nil { + return err + } + } + } + } + return ctx.Err() +} + +func (s *Store) verifyReference(ctx context.Context, obj Object) error { + file, err := s.openBlob(ctx, obj) + if err != nil { + return err + } + return file.Close() +} + +// SaveCheckpoint atomically saves the full manifest, including unfinished upload +// sessions, after verifying all referenced bytes. Saving an existing name replaces +// it. Names are 1–64 ASCII letters, digits, underscores or hyphens. Blobs remain +// in this store's root: checkpoints are not independent exports or backup copies. +func (s *Store) SaveCheckpoint(ctx context.Context, name string) (err error) { + if !validCheckpointName(name) { + return fmt.Errorf("%w: unsafe checkpoint name", ErrInvalid) + } + end, err := s.beginExclusive(ctx) + if err != nil { + return err + } + defer end() + if err := s.verifyManifest(ctx, s.state); err != nil { + return err + } + root, err := s.checkpointRoot(true) + if err != nil { + return err + } + defer func() { err = errors.Join(err, root.Close()) }() + return s.writeCheckpoint(ctx, root, name+".json") +} + +func (s *Store) writeCheckpoint(ctx context.Context, root *os.Root, name string) (err error) { + // Never silently replace a symlink, special file or foreign-format checkpoint. + if _, err := readNamedManifest(ctx, root, name); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + body, err := json.Marshal(s.state) + if err != nil { + return err + } + if len(body) > maxManifestBytes { + return fmt.Errorf("%w: checkpoint metadata is too large", ErrInvalid) + } + file, temp, err := tempFile(root) + if err != nil { + return err + } + published := false + defer func() { + if !published { + err = errors.Join(err, root.Remove(temp)) + } + }() + _, writeErr := file.Write(body) + if writeErr == nil { + writeErr = file.Sync() + } + if err := errors.Join(writeErr, file.Close()); err != nil { + return fmt.Errorf("write checkpoint: %w", err) + } + if err := ctx.Err(); err != nil { + return err + } + if err := root.Rename(temp, name); err != nil { + return fmt.Errorf("publish checkpoint: %w", err) + } + published = true + return syncDirectory(root, "checkpoint published") +} + +// ResetCheckpoint verifies every referenced blob before atomically replacing the +// live manifest. Extra buckets, objects and sessions disappear; bytes are retained +// until CollectGarbage. Repeating a reset is idempotent. It waits for in-flight +// operations, so none can publish pre-reset work after the reset returns. +func (s *Store) ResetCheckpoint(ctx context.Context, name string) (err error) { + if !validCheckpointName(name) { + return fmt.Errorf("%w: unsafe checkpoint name", ErrInvalid) + } + end, err := s.beginExclusive(ctx) + if err != nil { + return err + } + defer end() + root, err := s.checkpointRoot(false) + if errors.Is(err, os.ErrNotExist) { + return ErrCheckpointNotFound + } + if err != nil { + return err + } + defer func() { err = errors.Join(err, root.Close()) }() + next, err := readNamedManifest(ctx, root, name+".json") + if errors.Is(err, os.ErrNotExist) { + return ErrCheckpointNotFound + } + if err != nil { + return err + } + // TestCheckpointCorruptBytesNeverPublish covers objects and upload parts. + if err := s.verifyManifest(ctx, next); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + return s.commit(ctx, next) +} + +// DeleteCheckpoint durably removes a validated named manifest, not its blobs. +// Use CollectGarbage explicitly to reclaim newly unreferenced retained bytes. +func (s *Store) DeleteCheckpoint(ctx context.Context, name string) (err error) { + if !validCheckpointName(name) { + return fmt.Errorf("%w: unsafe checkpoint name", ErrInvalid) + } + end, err := s.beginExclusive(ctx) + if err != nil { + return err + } + defer end() + root, err := s.checkpointRoot(false) + if errors.Is(err, os.ErrNotExist) { + return ErrCheckpointNotFound + } + if err != nil { + return err + } + defer func() { err = errors.Join(err, root.Close()) }() + if _, err := readNamedManifest(ctx, root, name+".json"); err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrCheckpointNotFound + } + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := root.Remove(name + ".json"); err != nil { + return err + } + return syncDirectory(root, "checkpoint deleted") +} + +func syncDirectory(root *os.Root, operation string) error { + dir, err := root.Open(".") + if err != nil { + return fmt.Errorf("%s; open directory for sync failed: %w", operation, err) + } + if err := errors.Join(dir.Sync(), dir.Close()); err != nil { + return fmt.Errorf("%s but directory sync failed; durability is indeterminate: %w", operation, err) + } + return nil +} diff --git a/internal/core/objectstore/checkpoint_concurrency_test.go b/internal/core/objectstore/checkpoint_concurrency_test.go new file mode 100644 index 00000000..a6038395 --- /dev/null +++ b/internal/core/objectstore/checkpoint_concurrency_test.go @@ -0,0 +1,132 @@ +package objectstore + +import ( + "context" + "io" + "reflect" + "strings" + "testing" +) + +type checkpointBlockingReader struct { + entered chan struct{} + release chan struct{} + reader io.Reader +} + +func (r *checkpointBlockingReader) Read(p []byte) (int, error) { + if r.entered != nil { + close(r.entered) + r.entered = nil + <-r.release + } + return r.reader.Read(p) +} + +// The second cancellation check is in assembly's contextReader, after its +// reservation and temporary file exist, not in the short metadata transaction. +type checkpointAssemblyContext struct { + context.Context + calls int + entered chan struct{} + release chan struct{} +} + +func (c *checkpointAssemblyContext) Err() error { + c.calls++ + if c.calls == 2 { + close(c.entered) + <-c.release + } + return c.Context.Err() +} + +func TestCheckpointExclusiveGate(t *testing.T) { + s, _ := newStore(t) + end, err := s.beginExclusive(t.Context()) + if err != nil { + t.Fatal(err) + } + shared := s.life.TryRLock() + if shared { + s.life.RUnlock() + } + end() + if shared { + t.Fatal("checkpoint gate admitted an active operation") + } + // Accepting half: normal operations resume immediately after release. + mustBucket(t, s, "b") +} + +func TestCheckpointResetAndCollectionWaitForActiveMutations(t *testing.T) { + for _, mutation := range []string{"put", "part", "completion"} { + t.Run(mutation, func(t *testing.T) { + s, _ := limitedCheckpointStore(t, 64) + mustPut(t, s, "b", "key", "old") + id, err := s.CreateUpload(t.Context(), "b", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "b", "key", id, 1, strings.NewReader("part"), 4) + if err != nil { + t.Fatal(err) + } + before := s.clone() + mustSave(t, s, "saved") + entered, release := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + go func() { + body := &checkpointBlockingReader{entered: entered, release: release, reader: strings.NewReader("new")} + var err error + switch mutation { + case "put": + _, err = s.PutObject(t.Context(), "b", "key", body, PutOptions{Size: 3}) + case "part": + _, err = s.PutPart(t.Context(), "b", "key", id, 1, body, 3) + case "completion": + ctx := &checkpointAssemblyContext{Context: t.Context(), entered: entered, release: release} + _, err = s.CompleteUpload(ctx, "b", "key", id, []PartRef{{1, part.Digest}}, 10) + } + done <- err + }() + <-entered + // Lock assertions, not scheduling delays, prove both sides of the + // lifecycle boundary while each operation has bytes reserved. + exclusive := s.life.TryLock() + if exclusive { + s.life.Unlock() + } + metadataFree := s.mu.TryLock() + if metadataFree { + s.mu.Unlock() + } + s.budget.mu.Lock() + held := s.budget.held + s.budget.mu.Unlock() + resetDone, gcDone := make(chan error, 1), make(chan error, 1) + go func() { resetDone <- s.ResetCheckpoint(t.Context(), "saved") }() + go func() { gcDone <- s.CollectGarbage(t.Context()) }() + close(release) + for _, result := range []<-chan error{done, resetDone, gcDone} { + if err := <-result; err != nil { + t.Fatal(err) + } + } + if exclusive || !metadataFree || held == 0 { + t.Fatalf("slow %s did not hold exclusive-operation seam outside metadata lock: exclusive=%v free=%v held=%d", + mutation, exclusive, metadataFree, held) + } + if !reflect.DeepEqual(s.state, before) { + t.Fatalf("%s resurrected/overwrote reset state", mutation) + } + _, data := readObject(t, s, "b", "key") + if string(data) != "old" { + t.Fatalf("reset lost old bytes after %s: %q", mutation, data) + } + if err := s.ResetCheckpoint(t.Context(), "saved"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/core/objectstore/checkpoint_test.go b/internal/core/objectstore/checkpoint_test.go new file mode 100644 index 00000000..527b49bd --- /dev/null +++ b/internal/core/objectstore/checkpoint_test.go @@ -0,0 +1,320 @@ +package objectstore + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" +) + +func mustSave(t *testing.T, s *Store, name string) { + t.Helper() + if err := s.SaveCheckpoint(t.Context(), name); err != nil { + t.Fatal(err) + } +} + +func TestCheckpointRestoresFullManifestAndBytes(t *testing.T) { + s, path := newStore(t) + if err := s.CreateBucketWithMetadata(t.Context(), "b", map[string]string{"opaque": "value"}); err != nil { + t.Fatal(err) + } + original := mustPut(t, s, "b", "old", "original\x00bytes") + id, err := s.CreateUpload(t.Context(), "b", "assembled", PutOptions{ + ContentType: "text/plain", Metadata: map[string]string{"session": "retained"}, + }) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "b", "assembled", id, 1, strings.NewReader("part"), 4) + if err != nil { + t.Fatal(err) + } + before := s.clone() + mustSave(t, s, "Before_1") + mustPut(t, s, "b", "old", "replacement") + mustPut(t, s, "b", "extra", "extra") + mustBucket(t, s, "extra-bucket") + if _, err := s.CreateUpload(t.Context(), "b", "extra-session", PutOptions{}); err != nil { + t.Fatal(err) + } + if _, err := s.CompleteUpload(t.Context(), "b", "assembled", id, []PartRef{{1, part.Digest}}, 20); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }() + for range 2 { + if err := s.ResetCheckpoint(t.Context(), "Before_1"); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, s.state) { + t.Fatalf("reset did not restore exact full metadata: %+v", s.state) + } + got, body := readObject(t, s, "b", "old") + if !reflect.DeepEqual(got, original) || string(body) != "original\x00bytes" { + t.Fatalf("reset changed object: %+v %q", got, body) + } + } + if _, err := s.CompleteUpload(t.Context(), "b", "assembled", id, []PartRef{{1, part.Digest}}, 20); err != nil { + t.Fatal(err) + } + _, body := readObject(t, s, "b", "assembled") + if string(body) != "part" { + t.Fatalf("restored session lost its bytes: %q", body) + } +} + +func TestCheckpointCorruptBytesNeverPublish(t *testing.T) { + for _, target := range []string{"object", "part"} { + t.Run(target, func(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + obj := mustPut(t, s, "b", "key", "good") + id, err := s.CreateUpload(t.Context(), "b", "upload", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "b", "upload", id, 1, strings.NewReader("part"), 4) + if err != nil { + t.Fatal(err) + } + mustSave(t, s, "old") + if err := s.AbortUpload(t.Context(), "b", "upload", id); err != nil { + t.Fatal(err) + } + mustPut(t, s, "b", "key", "live") + live := s.clone() + digest := obj.Digest + if target == "part" { + digest = part.Digest + } + if err := os.WriteFile(filepath.Join(path, "blobs", digest), []byte("evil"), 0o600); err != nil { + t.Fatal(err) + } + indexBefore, err := os.ReadFile(filepath.Join(path, "index.json")) + if err != nil { + t.Fatal(err) + } + if err := s.ResetCheckpoint(t.Context(), "old"); !errors.Is(err, ErrCorrupt) { + t.Fatalf("corrupt %s checkpoint reset: %v", target, err) + } + indexAfter, err := os.ReadFile(filepath.Join(path, "index.json")) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(s.state, live) || string(indexBefore) != string(indexAfter) { + t.Fatal("failed reset published corrupt metadata") + } + if err := s.CollectGarbage(t.Context()); !errors.Is(err, ErrCorrupt) { + t.Fatalf("GC accepted corrupt checkpoint: %v", err) + } + good := "good" + if target == "part" { + good = "part" + } + if err := os.WriteFile(filepath.Join(path, "blobs", digest), []byte(good), 0o600); err != nil { + t.Fatal(err) + } + if err := s.ResetCheckpoint(t.Context(), "old"); err != nil { + t.Fatalf("accepting half after repairing blob: %v", err) + } + if err := os.WriteFile(filepath.Join(path, "blobs", digest), []byte("evil"), 0o600); err != nil { + t.Fatal(err) + } + if err := s.SaveCheckpoint(t.Context(), "corrupt"); !errors.Is(err, ErrCorrupt) { + t.Fatalf("saved corrupt live %s reference: %v", target, err) + } + if _, err := os.Stat(filepath.Join(path, "checkpoints", "corrupt.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed save published a checkpoint: %v", err) + } + if err := os.WriteFile(filepath.Join(path, "blobs", digest), []byte(good), 0o600); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestCheckpointDirectorySyncFailureIsReportedAndRetried(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + if err := s.dir.Close(); err != nil { + t.Fatal(err) + } + if err := s.SaveCheckpoint(t.Context(), "saved"); err == nil { + t.Fatal("directory sync failure reported success") + } + if _, err := os.Stat(filepath.Join(path, "checkpoints", "saved.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("checkpoint published before parent directory was durable: %v", err) + } + // Closing the handle released the process lock. No second Store enters + // during this injection; reacquire the lock before resuming normal operations. + dir, err := s.root.Open(".") + if err != nil { + t.Fatal(err) + } + s.dir = dir + if err := syscall.Flock(int(dir.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatal(err) + } + mustSave(t, s, "saved") + if err := s.ResetCheckpoint(t.Context(), "saved"); err != nil { + t.Fatal(err) + } +} + +func TestCheckpointNamesAndFormats(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + for _, name := range []string{"", ".", "..", "../outside", "a/b", "a.json", "white space", "é", strings.Repeat("a", 65)} { + for _, operation := range []func(context.Context, string) error{s.SaveCheckpoint, s.ResetCheckpoint, s.DeleteCheckpoint} { + if err := operation(t.Context(), name); !errors.Is(err, ErrInvalid) { + t.Fatalf("unsafe name %q: %v", name, err) + } + } + } + for _, operation := range []func(context.Context, string) error{s.ResetCheckpoint, s.DeleteCheckpoint} { + if err := operation(t.Context(), "missing"); !errors.Is(err, ErrCheckpointNotFound) { + t.Fatalf("missing checkpoint: %v", err) + } + } + mustSave(t, s, strings.Repeat("a", 64)) + mustSave(t, s, "valid-Name_2") + checkpoint := filepath.Join(path, "checkpoints", "valid-Name_2.json") + good, err := os.ReadFile(checkpoint) + if err != nil { + t.Fatal(err) + } + for _, body := range []string{ + `{"format":"feint-objectstore","version":5,"buckets":{}}`, + `{"format":"other","version":4,"buckets":{}}`, + `{"format":"feint-objectstore","version":4,"buckets":null}`, + strings.TrimSuffix(string(good), "}") + `,"unknown":true}`, + string(good) + `{}`, + `{"format":"feint-objectstore","version":4,"buckets":{"b":null}}`, + } { + if err := os.WriteFile(checkpoint, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if err := s.ResetCheckpoint(t.Context(), "valid-Name_2"); err == nil { + t.Fatalf("accepted malformed checkpoint: %s", body) + } + if err := s.CollectGarbage(t.Context()); err == nil { + t.Fatalf("GC accepted malformed checkpoint: %s", body) + } + } + for version := 1; version <= 4; version++ { + state := s.clone() + state.Version = version + body, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(checkpoint, body, 0o600); err != nil { + t.Fatal(err) + } + if err := s.ResetCheckpoint(t.Context(), "valid-Name_2"); err != nil { + t.Fatalf("legacy v%d: %v", version, err) + } + } + mustPut(t, s, "b", "saved", "bytes") + mustSave(t, s, "valid-Name_2") // Replacement is supported. + if err := s.ResetCheckpoint(t.Context(), "valid-Name_2"); err != nil { + t.Fatal(err) + } + if err := s.DeleteCheckpoint(t.Context(), "valid-Name_2"); err != nil { + t.Fatal(err) + } + if err := s.ResetCheckpoint(t.Context(), "valid-Name_2"); !errors.Is(err, ErrCheckpointNotFound) { + t.Fatalf("deleted checkpoint still exists: %v", err) + } +} + +func TestCheckpointPrivatePaths(t *testing.T) { + for _, attack := range []string{"directory-link", "directory-permissions", "file-link", "file-permissions"} { + t.Run(attack, func(t *testing.T) { + s, path := newStore(t) + mustSave(t, s, "valid") + directory := filepath.Join(path, "checkpoints") + file := filepath.Join(directory, "valid.json") + switch attack { + case "directory-link": + moved := filepath.Join(path, "moved") + if err := os.Rename(directory, moved); err != nil { + t.Fatal(err) + } + if err := os.Symlink("moved", directory); err != nil { + t.Fatal(err) + } + case "directory-permissions": + if err := os.Chmod(directory, 0o755); err != nil { + t.Fatal(err) + } + case "file-link": + if err := os.Rename(file, filepath.Join(directory, "other.json")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("other.json", file); err != nil { + t.Fatal(err) + } + case "file-permissions": + if err := os.Chmod(file, 0o644); err != nil { + t.Fatal(err) + } + } + for _, operation := range []func(context.Context, string) error{s.SaveCheckpoint, s.ResetCheckpoint, s.DeleteCheckpoint} { + if err := operation(t.Context(), "valid"); err == nil { + t.Fatalf("%s accepted unsafe checkpoint path", attack) + } + } + if err := s.CollectGarbage(t.Context()); err == nil { + t.Fatalf("%s: GC accepted unsafe checkpoint path", attack) + } + }) + } +} + +func TestCheckpointCancellationClosedAndFault(t *testing.T) { + s, _ := newStore(t) + mustSave(t, s, "valid") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + for _, operation := range []func(context.Context, string) error{s.SaveCheckpoint, s.ResetCheckpoint, s.DeleteCheckpoint} { + if err := operation(ctx, "valid"); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled operation: %v", err) + } + } + if err := s.CollectGarbage(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled collection: %v", err) + } + s.fault = errors.New("indeterminate commit") + if err := s.ResetCheckpoint(t.Context(), "valid"); !errors.Is(err, s.fault) { + t.Fatalf("fault ignored: %v", err) + } + s.fault = nil + if err := s.Close(); err != nil { + t.Fatal(err) + } + for _, operation := range []func(context.Context, string) error{s.SaveCheckpoint, s.ResetCheckpoint, s.DeleteCheckpoint} { + if err := operation(t.Context(), "valid"); !errors.Is(err, ErrClosed) { + t.Fatalf("closed operation: %v", err) + } + } + if err := s.CollectGarbage(t.Context()); !errors.Is(err, ErrClosed) { + t.Fatalf("closed collection: %v", err) + } +} diff --git a/internal/core/objectstore/files.go b/internal/core/objectstore/files.go index 5c149957..4592357b 100644 --- a/internal/core/objectstore/files.go +++ b/internal/core/objectstore/files.go @@ -137,16 +137,8 @@ func Open(ctx context.Context, path string) (result *Store, err error) { if err := private(opened, true); err != nil { return nil, fmt.Errorf("opened blob directory: %w", errors.Join(ErrCorrupt, err)) } - for _, b := range s.state.Buckets { - for _, obj := range b.Objects { - file, err := s.openBlob(ctx, obj) - if err != nil { - return nil, err - } - if err := file.Close(); err != nil { - return nil, fmt.Errorf("close startup blob verification: %w", err) - } - } + if err := s.verifyManifest(ctx, s.state); err != nil { + return nil, err } return s, nil } diff --git a/internal/core/objectstore/gc.go b/internal/core/objectstore/gc.go new file mode 100644 index 00000000..7451dde4 --- /dev/null +++ b/internal/core/objectstore/gc.go @@ -0,0 +1,192 @@ +package objectstore + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "syscall" +) + +// CollectGarbage explicitly deletes only unreferenced owned blob files. +// References in the live manifest and every durable checkpoint are protected, +// including upload parts. Active operations/reservations finish before inventory. +// Every entry is validated before any deletion; unsafe names, permissions, +// symlinks or corrupt checkpoints fail closed. Cancellation or I/O failure during +// deletion may leave a partially collected inventory; quota accounting is updated +// for successful unlinks, and retry is safe. Open returned readers keep their +// descriptors valid; their unlinked bytes are outside the retained-file budget. +func (s *Store) CollectGarbage(ctx context.Context) (err error) { + end, err := s.beginExclusive(ctx) + if err != nil { + return err + } + defer end() + refs := make(map[string]bool) + if err := s.protectManifest(ctx, s.state, refs); err != nil { + return err + } + if err := s.protectCheckpoints(ctx, refs); err != nil { + return err + } + entries, err := s.blobInventory(ctx) + if err != nil { + return err + } + changed := false + defer func() { + if s.budget != nil { + seen := make(map[string]bool) + var used int64 + for _, entry := range entries { + if !entry.removed && !seen[entry.identity] { + seen[entry.identity] = true + used += entry.size + } + } + s.budget.mu.Lock() + s.budget.used = used + s.budget.mu.Unlock() + } + if changed { + err = errors.Join(err, syncDirectory(s.blobs, "garbage collected")) + } + }() + for i := range entries { + entry := &entries[i] + // TestCheckpointProtectsGarbageAndReleasesBudget holds this retention guard. + if refs[entry.name] { + continue + } + if err := ctx.Err(); err != nil { + return err + } + if err := s.blobs.Remove(entry.name); err != nil { + return fmt.Errorf("collect blob %s: %w", entry.name, err) + } + entry.removed = true + changed = true + } + return nil +} + +func (s *Store) protectManifest(ctx context.Context, state manifest, refs map[string]bool) error { + if err := s.verifyManifest(ctx, state); err != nil { + return err + } + for _, b := range state.Buckets { + for _, obj := range b.Objects { + refs[obj.Digest] = true + } + for _, upload := range b.Uploads { + for _, part := range upload.Parts { + refs[part.Digest] = true + } + } + } + return nil +} + +func (s *Store) protectCheckpoints(ctx context.Context, refs map[string]bool) (err error) { + root, err := s.checkpointRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer func() { err = errors.Join(err, root.Close()) }() + dir, err := root.Open(".") + if err != nil { + return err + } + defer func() { err = errors.Join(err, dir.Close()) }() + entries, err := dir.ReadDir(-1) + if err != nil { + return err + } + for _, entry := range entries { + name := entry.Name() + if validPendingName(name) { + file, err := openPrivate(root, name) + if err != nil { + return errors.Join(ErrCorrupt, err) + } + if err := file.Close(); err != nil { + return err + } + continue // Unpublished crash debris is not a durable checkpoint. + } + if !strings.HasSuffix(name, ".json") || !validCheckpointName(strings.TrimSuffix(name, ".json")) { + return fmt.Errorf("%w: unsafe persisted checkpoint filename", ErrCorrupt) + } + state, err := readNamedManifest(ctx, root, name) + if err != nil { + return err + } + if err := s.protectManifest(ctx, state, refs); err != nil { + return err + } + } + return ctx.Err() +} + +type retainedBlob struct { + name string + identity string + size int64 + removed bool +} + +func (s *Store) blobInventory(ctx context.Context) (_ []retainedBlob, err error) { + dir, err := s.blobs.Open(".") + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, dir.Close()) }() + entries, err := dir.ReadDir(-1) + if err != nil { + return nil, err + } + inventory := make([]retainedBlob, 0, len(entries)) + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return nil, err + } + name := entry.Name() + // TestGarbageCollectionRejectsUnsafeInventory includes an accepting half. + if !validDigest(name) && !validPendingName(name) { + return nil, fmt.Errorf("%w: unsafe persisted blob filename", ErrCorrupt) + } + file, err := openPrivate(s.blobs, name) + if err != nil { + return nil, errors.Join(ErrCorrupt, err) + } + info, statErr := file.Stat() + if err := errors.Join(statErr, file.Close()); err != nil { + return nil, err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || info.Size() < 0 { + return nil, fmt.Errorf("%w: invalid blob file identity or size", ErrCorrupt) + } + inventory = append(inventory, retainedBlob{ + name: name, identity: fmt.Sprintf("%v:%v", stat.Dev, stat.Ino), size: info.Size(), + }) + } + return inventory, nil +} + +func validPendingName(name string) bool { + const prefix = ".pending-" + if !strings.HasPrefix(name, prefix) || len(name) != len(prefix)+26 { + return false + } + for _, c := range name[len(prefix):] { + if (c < 'A' || c > 'Z') && (c < '2' || c > '7') { + return false + } + } + return true +} diff --git a/internal/core/objectstore/gc_test.go b/internal/core/objectstore/gc_test.go new file mode 100644 index 00000000..3133684e --- /dev/null +++ b/internal/core/objectstore/gc_test.go @@ -0,0 +1,226 @@ +package objectstore + +import ( + "crypto/rand" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func limitedCheckpointStore(t *testing.T, limit int64) (*Store, string) { + t.Helper() + path := filepath.Join(privateDir(t), "objects") + s, err := OpenWithLimit(t.Context(), path, limit) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }) + mustBucket(t, s, "b") + return s, path +} + +func TestCheckpointProtectsGarbageAndReleasesBudget(t *testing.T) { + s, path := limitedCheckpointStore(t, 8) + obj := mustPut(t, s, "b", "key", "same") + mustPut(t, s, "b", "duplicate", "same") + id, err := s.CreateUpload(t.Context(), "b", "upload", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "b", "upload", id, 1, strings.NewReader("part"), 4) + if err != nil { + t.Fatal(err) + } + mustSave(t, s, "one") + mustSave(t, s, "two") + for _, key := range []string{"key", "duplicate"} { + if err := s.DeleteObject(t.Context(), "b", key); err != nil { + t.Fatal(err) + } + } + if err := s.AbortUpload(t.Context(), "b", "upload", id); err != nil { + t.Fatal(err) + } + if err := s.DeleteCheckpoint(t.Context(), "one"); err != nil { + t.Fatal(err) + } + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + for _, digest := range []string{obj.Digest, part.Digest} { + if _, err := os.Stat(filepath.Join(path, "blobs", digest)); err != nil { + t.Fatalf("checkpoint-protected bytes lost: %v", err) + } + } + if _, err := s.PutObject(t.Context(), "b", "new", strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrQuota) { + t.Fatalf("checkpoint-retained bytes no longer count: %v", err) + } + if err := s.ResetCheckpoint(t.Context(), "two"); err != nil { + t.Fatal(err) + } + _, body := readObject(t, s, "b", "key") + if string(body) != "same" { + t.Fatalf("checkpoint was not preserved: %q", body) + } + for _, key := range []string{"key", "duplicate"} { + if err := s.DeleteObject(t.Context(), "b", key); err != nil { + t.Fatal(err) + } + } + if err := s.AbortUpload(t.Context(), "b", "upload", id); err != nil { + t.Fatal(err) + } + if err := s.DeleteCheckpoint(t.Context(), "two"); err != nil { + t.Fatal(err) + } + // Deleting a checkpoint/aborting still does not automatically collect. + if s.budget.used != 8 { + t.Fatalf("automatic collection occurred: %d", s.budget.used) + } + for range 2 { + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + } + if s.budget.used != 0 || s.budget.held != 0 { + t.Fatalf("collection leaked budget: %+v", s.budget) + } + mustPut(t, s, "b", "new", "12345678") +} + +func TestGarbageCollectionCountsHardlinksOnce(t *testing.T) { + s, path := limitedCheckpointStore(t, 4) + obj := mustPut(t, s, "b", "key", "data") + alias := strings.Repeat("f", 64) + if err := os.Link(filepath.Join(path, "blobs", obj.Digest), filepath.Join(path, "blobs", alias)); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err := OpenWithLimit(t.Context(), path, 4) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Error(err) + } + }() + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(path, "blobs", alias)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("orphan hardlink retained: %v", err) + } + if s.budget.used != 4 { + t.Fatalf("deleting one hardlink released still-retained bytes: %d", s.budget.used) + } + if _, err := s.PutObject(t.Context(), "b", "new", strings.NewReader("x"), PutOptions{Size: 1}); !errors.Is(err, ErrQuota) { + t.Fatalf("retained hardlink budget ignored: %v", err) + } + if err := s.DeleteObject(t.Context(), "b", "key"); err != nil { + t.Fatal(err) + } + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + mustPut(t, s, "b", "new", "more") +} + +func TestGarbageCollectionRejectsUnsafeInventory(t *testing.T) { + for _, attack := range []string{"foreign-name", "symlink", "permissions", "checkpoint-name", "checkpoint-symlink"} { + t.Run(attack, func(t *testing.T) { + s, path := newStore(t) + mustBucket(t, s, "b") + orphan := mustPut(t, s, "b", "orphan", "orphan") + if err := s.DeleteObject(t.Context(), "b", "orphan"); err != nil { + t.Fatal(err) + } + mustSave(t, s, "safe") + name := strings.Repeat("f", 64) + dir := filepath.Join(path, "blobs") + switch attack { + case "foreign-name": + name = "operator-file" + if err := os.WriteFile(filepath.Join(dir, name), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + case "symlink": + if err := os.Symlink(orphan.Digest, filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + case "permissions": + if err := os.WriteFile(filepath.Join(dir, name), []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + case "checkpoint-name": + dir = filepath.Join(path, "checkpoints") + name = "operator-file" + body, err := os.ReadFile(filepath.Join(dir, "safe.json")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), body, 0o600); err != nil { + t.Fatal(err) + } + case "checkpoint-symlink": + dir = filepath.Join(path, "checkpoints") + name = "unsafe.json" + if err := os.Symlink("safe.json", filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + if err := s.CollectGarbage(t.Context()); err == nil { + t.Fatalf("unsafe %s inventory was collected", attack) + } + if _, err := os.Lstat(filepath.Join(dir, name)); err != nil { + t.Fatalf("unsafe entry was deleted: %v", err) + } + if _, err := os.Stat(filepath.Join(path, "blobs", orphan.Digest)); err != nil { + t.Fatalf("partial deletion before validation finished: %v", err) + } + if err := os.Remove(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + // Accept known crash debris; reject arbitrary names, not all inventory. + pending := filepath.Join(path, "blobs", ".pending-"+rand.Text()) + if err := os.WriteFile(pending, []byte("debris"), 0o600); err != nil { + t.Fatal(err) + } + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatalf("accepting half: %v", err) + } + for _, name := range []string{pending, filepath.Join(path, "blobs", orphan.Digest)} { + if _, err := os.Stat(name); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("owned garbage was not deleted: %v", err) + } + } + }) + } +} + +func TestGarbageCollectionProtectsLiveUploads(t *testing.T) { + s, _ := newStore(t) + mustBucket(t, s, "b") + id, err := s.CreateUpload(t.Context(), "b", "key", PutOptions{}) + if err != nil { + t.Fatal(err) + } + part, err := s.PutPart(t.Context(), "b", "key", id, 1, strings.NewReader("part"), 4) + if err != nil { + t.Fatal(err) + } + if err := s.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := s.CompleteUpload(t.Context(), "b", "key", id, []PartRef{{1, part.Digest}}, 10); err != nil { + t.Fatalf("GC removed live part: %v", err) + } +} diff --git a/internal/core/objectstore/manifest.go b/internal/core/objectstore/manifest.go index e3c1f52c..1a21a94f 100644 --- a/internal/core/objectstore/manifest.go +++ b/internal/core/objectstore/manifest.go @@ -19,7 +19,11 @@ const ( ) func readManifest(ctx context.Context, root *os.Root) (_ manifest, err error) { - file, err := openPrivate(root, "index.json") + return readNamedManifest(ctx, root, "index.json") +} + +func readNamedManifest(ctx context.Context, root *os.Root, name string) (_ manifest, err error) { + file, err := openPrivate(root, name) if err != nil { return manifest{}, fmt.Errorf("read storage ownership metadata: %w", errors.Join(ErrUnowned, err)) } diff --git a/internal/core/objectstore/store.go b/internal/core/objectstore/store.go index e2fdd90e..1e7c8082 100644 --- a/internal/core/objectstore/store.go +++ b/internal/core/objectstore/store.go @@ -3,7 +3,9 @@ // Blobs are immutable and addressed by SHA-256, never by a bucket or object key. // Mutations publish metadata only after the blob is durable. One Store owns a // private directory exclusively; Close releases it so Open can load it again. -// This package provides no S3 protocol, resource snapshots, checkpoints or GC. +// Named checkpoints retain full metadata and blobs until explicitly deleted. +// Garbage collection is explicit; aborting an upload never collects bytes. +// This package provides no provider protocol or resource snapshots. package objectstore import ( @@ -73,6 +75,8 @@ type manifest struct { // writes and deletes to the same key are serialized. Metadata commits are short // exclusive transactions. Close waits for in-flight operations, not for the // independent readers returned by GetObject. +// Checkpoint operations and garbage collection exclude all active operations, +// including slow uploads, without holding the metadata mutex during body reads. type Store struct { life sync.RWMutex mu sync.RWMutex diff --git a/tools/falsify/specs/objectstore-checkpoints.json b/tools/falsify/specs/objectstore-checkpoints.json new file mode 100644 index 00000000..b7938c81 --- /dev/null +++ b/tools/falsify/specs/objectstore-checkpoints.json @@ -0,0 +1,61 @@ +{ + "package": "./internal/core/objectstore/", + "mutations": [ + { + "label": "reset ignores failed object and part integrity verification", + "file": "internal/core/objectstore/checkpoint.go", + "find": "if err := s.verifyManifest(ctx, next); err != nil {", + "replace": "if err := s.verifyManifest(ctx, next); err != nil && false {", + "test": "^TestCheckpointCorruptBytesNeverPublish$" + }, + { + "label": "checkpoint lifecycle gate admits concurrent mutations", + "file": "internal/core/objectstore/checkpoint.go", + "find": "s.life.Lock()\n\tend := s.life.Unlock", + "replace": "s.life.RLock()\n\tend := s.life.RUnlock", + "test": "^TestCheckpointExclusiveGate$" + }, + { + "label": "save accepts unsafe checkpoint names", + "file": "internal/core/objectstore/checkpoint.go", + "find": "func (s *Store) SaveCheckpoint(ctx context.Context, name string) (err error) {\n\tif !validCheckpointName(name) {", + "replace": "func (s *Store) SaveCheckpoint(ctx context.Context, name string) (err error) {\n\tif !validCheckpointName(name) && false {", + "test": "^TestCheckpointNamesAndFormats$" + }, + { + "label": "collection deletes referenced checkpoint objects and parts", + "file": "internal/core/objectstore/gc.go", + "find": "if refs[entry.name] {", + "replace": "if refs[entry.name] && false {", + "test": "^TestCheckpointProtectsGarbageAndReleasesBudget$" + }, + { + "label": "collection deletes arbitrary private blob filenames", + "file": "internal/core/objectstore/gc.go", + "find": "if !validDigest(name) && !validPendingName(name) {", + "replace": "if !validDigest(name) && !validPendingName(name) && false {", + "test": "^TestGarbageCollectionRejectsUnsafeInventory$/foreign-name$" + }, + { + "label": "collection accepts a manifest at an unsafe persisted checkpoint name", + "file": "internal/core/objectstore/gc.go", + "find": "if !strings.HasSuffix(name, \".json\") || !validCheckpointName(strings.TrimSuffix(name, \".json\")) {", + "replace": "if (!strings.HasSuffix(name, \".json\") || !validCheckpointName(strings.TrimSuffix(name, \".json\"))) && false {", + "test": "^TestGarbageCollectionRejectsUnsafeInventory$/checkpoint-name$" + }, + { + "label": "checkpoint save skips syncing the parent directory", + "file": "internal/core/objectstore/checkpoint.go", + "find": "if err == nil && create {", + "replace": "if err == nil && create && false {", + "test": "^TestCheckpointDirectorySyncFailureIsReportedAndRetried$" + }, + { + "label": "collection fails to release retained quota", + "file": "internal/core/objectstore/gc.go", + "find": "s.budget.used = used", + "replace": "s.budget.used = used + s.budget.used", + "test": "^TestCheckpointProtectsGarbageAndReleasesBudget$" + } + ] +} From 2702f8d666b6a5c69580a26af6dcc19f186b3eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 15:56:08 +0100 Subject: [PATCH 12/28] feat(s3): the Exoscale bucket cycle, the shared S3 discovery and the probe that proves the native inventory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One slice of #850, and it does NOT complete it. #850, #853, #858 and #849 stay open; what each still lacks is at the end of this message. ## What the slice carries Exoscale, the ordinary bucket cycle (#858). `exo storage mb`, `storage show`, removal and the native inventory, with the ACL/CORS/ownership discovery the client performs before anything else. The refusals are the recorded ones rather than invented successes: absent CORS is a 404 NoSuchCORSConfiguration whose message carries its final period, absent ownership controls a 400 OwnershipControlsNotFoundError. Both come from the authorized empty-bucket recording kept at docs/evidence/exoscale-sos-empty-bucket.json, whose own header states what was redacted and that it is not an unmodified transcript. The native inventory, exoscale/v2.list-sos-buckets-usage, serves Store.Usage: logical published object bytes, never retained blobs or unpublished parts. It errors on disabled storage and on malformed or legacy scopes and dates rather than reporting a zero it cannot justify. Shared S3 discovery, and the transport under it. s3.LocalTransport forwards only declared HTTPS cloud authorities to one explicit loopback HTTP emulator, never an inherited proxy and never a cloud-capable fallback. It preserves dotted virtual-host buckets and literally escaped keys, refuses an undeclared authority BEFORE any dial, and refuses two declarations where one is a suffix of the other. Shapes come from the pinned official provider cassette. Scaleway Terraform, measured rather than assumed: the published provider 2.82.0 ignores SCW_S3_ENDPOINT in its S3 client, which both the pinned source and an isolated failure proved. The witness in internal/cli/object_storage_terraform_conformance_test.go gets apply, two empty plans, and 122,880 identical bytes across byte-store close/reopen, destroy and explicit garbage collection. ## What this session added to it The probe proves the native route instead of failing on it. It answered 503 "Object Storage is disabled" and the probe — which builds its request from the operation's own schema, and that schema declares a 200 body — counted that as a route answering the wrong shape. Two populations needed fixing and only the first was obvious: internal/probe/probe_test.go wires a byte inventory, 0700 on its parent because the store refuses a writable parent with no sticky bit, and it is right to mise.toml, task conformance starts the emulator with --objectstore under the run's own mktemp, so `feint probe` in the real pass sees what the in-process test sees Proved both ways: without the flag, 98 operations probed and 2 failures; with it, 100 probed and 0. Fixing only the Go test cost one evidence run, which is the population trap this repository keeps a skill about. Three easier answers were measured and rejected. Declaring the axis Unearnable is a lie, because the axis CAN be earned and TestAnUnearnableAxisIsNotAlreadyEarned would bite. Declaring the 503 in the contract is a worse lie, because Exoscale documents no such answer and the contract would describe an API nobody has. Teaching the probe to skip would sub-declare an operation that is served and checkable. `exo storage list` now drives the route in the Exoscale suite. TestEveryUndrivenOperationSaysWhy demanded either a client or a reason, and the reason would have been false: exo calls ListSOSBucketsUsage from cmd/storage/storage_list.go. A local harness said otherwise and was wrong — it failed on the CLI's default zone ch-dk-2 before calling anything, which is the failure exo-cli.sh documents in as many words. The assertion is an EMPTY inventory over zero buckets, because that is the honest derivation at its smallest and #858 asks for values derived from real buckets. Two falsification mutations, which the handoff recorded as interrupted. The three existing ones all drive LocalTransport's accepting path; neither touched the two refusals in NewLocalTransport, where the blast radius is: a non-loopback target forwards cloud-shaped requests off the machine, and overlapping authorities let map iteration order pick a provider's door. Both keep every name and suffix the condition with `&& false`, because the harness refuses a mutation that drops an identifier. 13 -> 15, all compiled, all bit. ## Verification, all run on this tree mise run check pass mise run docs:check pass mise run shapes:check pass mise run falsify:lint pass, 1528 mutations / 225 specs mise run lint-shell pass mise run release:surface pass, 1 named 0 excused 0 neither go vet -tags=conformance (cli, s3, exoscale, objectstore) pass mise run falsify -- s3-storage-discovery.json 15/15 compiled and bit mise run falsify -- s3-gateway.json bit mise run falsify -- s3-bucket-configuration.json bit tools/conformance/exoscale/exo-cli.sh pass, new assertion included mise run conformance:s3:mounted PASS 21.64s mise run evidence:update regenerated, both legs, 0 failure The isolated Terraform witness, rebuilt from this tree and replayed under a disposable loopback-only network namespace (operator-authorized sudo for the namespace only, privileges dropped by setpriv immediately after): Scaleway 2.82.0: apply, two empty plans, 122880 identical bytes across reopen, destroy and owned GC; cloud-host TLS interception --- PASS (1.07s) conformance:s3:mounted witnessed real bytes rather than exit codes: 6,340,608 bytes verified by sha256 across an ACTUAL process restart (pid 2537132 -> 2537864), a multipart session above 5 MiB resumed after that restart and completed with identical bytes, and `exo rb` aborting a genuinely dangling upload. The evidence row the whole slice turned on: exoscale/v2.list-sos-buckets-usage driven:true probed:response contract:clean dataplane:true shape:unobserved behaviour:false negative:false Four axes earned, three not, and the three say why: no cloud recording arbitrates its shape, and no suite asserts its behaviour or its refusals yet. That row could not have been hand-written honestly, which is why the task was run three times rather than the file edited once. s3-bucket-configuration looked dead and was not: its mutation 20 targets internal/cli, where the evidence test was red. Its own test passes alone. ## What is NOT proved, and what each issue still needs The Terraform witness is a TEST HARNESS, not a shipped `feint run`. It uses a tiny arbitrary payload named qcow2 and proves no image or runtime path. #850 asks that a surveyed stack stop escaping to real Object Storage; what exists proves the transport. #850 also still needs: an OUTSCALE or OOS client path, a dotted-bucket TLS proof with an actual client, and a generated bounded capability statement in docs/confidence.md — whose table is hand-written on purpose, as docs_confidence.go says, because a verdict is a judgement. #853 is broader than the committed byte-checkpoint library: no coordinated CLI/control-plane/runtime checkpoint exists. #849's interception exceeds this harness. #858's closure criteria are not all met. Debts left visible rather than refreshed away: `mise run drift:check` is red here on lb/v1/ZonedAPI.AttachIpamIP and TransferToIpam, which is #859 — triaged and resolved upstream in main by #860, and an integration of main will carry it. An earlier local run of drift:check in this worktree reported "no drift" only because .upstream/ was empty, which is a false green and not a clearance. #857, the limits freshness review, is untouched. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 21 +- CHANGELOG.md | 20 +- README.fr.md | 4 +- README.md | 14 +- coverage/evidence.json | 15 +- coverage/exoscale-coverage.json | 21 +- docs/confidence.md | 2 +- docs/evidence/exoscale-sos-empty-bucket.json | 33 +++ docs/object-storage-routing.md | 96 +++++++- docs/routes.md | 24 +- internal/cli/cli.go | 1 + .../cli/object_storage_conformance_test.go | 67 ++++-- ...ject_storage_terraform_conformance_test.go | 215 ++++++++++++++++++ internal/conformancetest/aws.go | 5 +- internal/conformancetest/scw.go | 2 +- internal/core/emulator/emulator.go | 8 +- internal/core/objectstore/bucket_metadata.go | 33 +++ internal/core/objectstore/usage_test.go | 35 +++ internal/probe/probe_test.go | 42 ++++ internal/providers/exoscale/object_storage.go | 64 +++++- .../providers/exoscale/object_storage_test.go | 136 +++++++++++ .../exoscale/object_storage_usage_test.go | 69 ++++++ internal/providers/exoscale/pack.go | 6 +- internal/providers/scaleway/object_storage.go | 21 +- internal/s3/bucket_configuration.go | 99 +++++++- internal/s3/discovery_test.go | 156 +++++++++++++ internal/s3/doors.go | 23 +- internal/s3/list.go | 63 ++++- internal/s3/local_transport.go | 100 ++++++++ internal/s3/local_transport_test.go | 96 ++++++++ internal/s3/server.go | 34 ++- mise.toml | 35 ++- tools/conformance/exoscale/exo-cli.sh | 28 +++ .../specs/s3-bucket-configuration.json | 4 +- tools/falsify/specs/s3-gateway.json | 4 +- tools/falsify/specs/s3-storage-discovery.json | 114 ++++++++++ 36 files changed, 1607 insertions(+), 103 deletions(-) create mode 100644 docs/evidence/exoscale-sos-empty-bucket.json create mode 100644 internal/cli/object_storage_terraform_conformance_test.go create mode 100644 internal/core/objectstore/usage_test.go create mode 100644 internal/providers/exoscale/object_storage_test.go create mode 100644 internal/providers/exoscale/object_storage_usage_test.go create mode 100644 internal/s3/discovery_test.go create mode 100644 internal/s3/local_transport.go create mode 100644 internal/s3/local_transport_test.go create mode 100644 tools/falsify/specs/s3-storage-discovery.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index e5ca9683..3547ac16 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,21 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **Le socle de livraison Object Storage** (#850, #858) : création normale, + lecture et suppression de bucket Exo, usage natif issu des octets publiés et + erreurs de découverte enregistrées. L'inventaire natif est + `exoscale/v2.list-sos-buckets-usage`, un refus retiré : il répond à + `GET /v2/sos-buckets-usage` depuis les mêmes octets publiés plutôt que depuis + un catalogue, `exo storage list` le pilote (le client appelle + ListSOSBucketsUsage), et il répond toujours 503 sans `--objectstore`, parce + que l'inventaire d'un inventaire absent n'est pas zéro. + Terraform Scaleway 2.82.0 passe apply, + plans vides, réouverture du store, GET identique de 122 880 octets, destroy + et GC explicite en namespace réseau limité à loopback, via interception TLS + ciblée. C'est un harnais, pas `feint run` ni une certification de stack entière. + Les checkpoints d'octets neutres conservent le multipart et protègent le GC ; + le reset coordonné CLI/plan de contrôle/runtime reste à finir. + - **Le multipart persistant, prouvé avec Exo/AWS réels** (#856) : exo 1.95.1 téléverse 6 340 608 octets en multipart ; AWS vérifie l'assemblage, reprend les parties après redémarrage réel et termine un autre objet. @@ -26,9 +41,9 @@ change ni l'un ni l'autre a sa place dans `git log`. Le format privé v4 lit v1..v3 ; un retour à un ancien binaire demande une copie complète du store avant migration. L'assemblage garde la limite du PUT simple ; parties et assemblage partagent le quota, et l'abandon n'est pas du GC. - Les tags SHA-256 opaques ne promettent pas des ETag génériques. Création normale - de bucket Exo, usage natif, erreurs cloud mixtes et Object Storage 1.0 complet - restent hors de cette preuve. + Les tags SHA-256 opaques ne promettent pas des ETag génériques. La tranche + suivante ajoute création Exo normale et usage natif ; erreurs cloud mixtes et + Object Storage 1.0 complet restent hors de cette preuve. - **La suppression en lot Exo, mesurée** (#856) : exo 1.95.1 supprime quatre vraies clés ; AWS vérifie l'inventaire vide après redémarrage et les octets diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c6c8d60..23c9ee31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,21 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **Object Storage release groundwork** (#850, #858): ordinary Exo bucket + creation/show/removal and native usage now use recorded default discovery + errors and the actual published byte inventory. The native inventory is + `exoscale/v2.list-sos-buckets-usage`, a refusal withdrawn: it answers + `GET /v2/sos-buckets-usage` from the same published bytes rather than a + catalogue, `exo storage list` drives it (the client calls + ListSOSBucketsUsage), and it still answers 503 without `--objectstore` + because an inventory of an inventory that does not exist is not zero. + Scaleway Terraform 2.82.0 + completes apply, empty plans, byte-store reopen, identical 122,880-byte GET, + destroy and explicit GC in a loopback-only namespace through scoped TLS + interception. This is a test harness, not `feint run` or full-stack certification. + Neutral named byte checkpoints preserve multipart sessions and protect GC + references; coordinated CLI/control-plane/runtime reset remains unfinished. + - **Persistent multipart, proved with real Exo/AWS clients** (#856): exo 1.95.1 multipart-uploads 6,340,608 bytes; AWS verifies assembled bytes, resumes persisted parts after actual restart and completes another object. @@ -24,8 +39,9 @@ what this project is judged on: **a response shape a client can observe**, and Private metadata v4 reads v1..v3; older binaries need a pre-upgrade whole-store copy for rollback. Assembly uses the simple-PUT limit; retained parts and assembly share the blob quota, and abort is not GC. Opaque SHA-256 part tags - are not generic ETags. Normal Exo bucket creation, native usage, mixed per-key - cloud errors and full Object Storage 1.0 remain outside this proof. + are not generic ETags. The newer slice above adds normal Exo creation and + native usage; mixed per-key cloud errors and full Object Storage 1.0 remain + outside this proof. - **Measured Exo bulk deletion** (#856): exo 1.95.1 deletes four actual keys; AWS independently checks the empty inventory across restart and untouched diff --git a/README.fr.md b/README.fr.md index 2a29efd3..cee07230 100644 --- a/README.fr.md +++ b/README.fr.md @@ -29,7 +29,7 @@ > [!IMPORTANT] > **Ce qu'on peut pointer vers cet émulateur, et ce qu'on ne peut pas.** > -> **Prouvé** : 383 des 408 opérations montées sont pilotées par un vrai client, à chaque pull request. `scw`, `octl`, `exo`, Terraform et OpenTofu tournent contre l'émulateur en CI, et les machines démarrent réellement : connexion ssh sur le compte par défaut de chaque provider, subnets isolés, pare-feu qui filtre. La chaîne complète est décrite dans [docs/conformance.md](docs/conformance.md). +> **Prouvé** : 384 des 409 opérations montées sont pilotées par un vrai client, à chaque pull request. `scw`, `octl`, `exo`, Terraform et OpenTofu tournent contre l'émulateur en CI, et les machines démarrent réellement : connexion ssh sur le compte par défaut de chaque provider, subnets isolés, pare-feu qui filtre. La chaîne complète est décrite dans [docs/conformance.md](docs/conformance.md). > > **Pas prouvé** : quotas, prix, capacité réelle, validation des identifiants, authentification, cohérence à terme. Les 58 sections de [docs/limits.md](docs/limits.md) disent chacune ce qu'elle coûte. Un émulateur avec un seul compte implicite et aucune grille tarifaire devrait inventer ces chiffres, et quelqu'un agirait dessus. > @@ -142,7 +142,7 @@ feint serve # feint dev listening on 127.0.0.1:4599 # scaleway 204 routes # outscale 100 routes -# exoscale 104 routes +# exoscale 105 routes # machines none ``` diff --git a/README.md b/README.md index 065f4bc5..c5b1a013 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ > [!IMPORTANT] > **What is safe to point at this emulator, and what is not.** > -> **Proven**: 383 of the 408 mounted operations are driven by a real client, on every pull request: each pack's own official CLI, and an infrastructure engine wherever a pack admits one, all of them running against the emulator in CI, and machines really boot: an ssh login on each provider's own default account, isolated subnets, a firewall that filters. The whole chain is described in [docs/conformance.md](docs/conformance.md). +> **Proven**: 384 of the 409 mounted operations are driven by a real client, on every pull request: each pack's own official CLI, and an infrastructure engine wherever a pack admits one, all of them running against the emulator in CI, and machines really boot: an ssh login on each provider's own default account, isolated subnets, a firewall that filters. The whole chain is described in [docs/conformance.md](docs/conformance.md). > > **Not proven**: quotas, prices, real capacity, identifier validation, authentication, eventual consistency. The 58 sections of [docs/limits.md](docs/limits.md) each say what one costs. An emulator with a single implicit account and no price list would have to invent those figures, and somebody would act on them. > @@ -131,7 +131,7 @@ feint serve # feint dev listening on 127.0.0.1:4599 # scaleway 204 routes # outscale 100 routes -# exoscale 104 routes +# exoscale 105 routes # machines none ``` @@ -856,7 +856,7 @@ same block is published in the body of every release. -408 routes are mounted across the three packs. The tables count *upstream operations* +409 routes are mounted across the three packs. The tables count *upstream operations* rather than routes: what the provider's own SDK or API description declares, against what this emulator serves, declines on purpose, or has not triaged yet. @@ -880,8 +880,8 @@ what this emulator serves, declines on purpose, or has not triaged yet. #### Exoscale -104 routes mounted. Of the 374 operations upstream declares: 27% served, -72% declined on purpose, 0% untriaged. +105 routes mounted. Of the 374 operations upstream declares: 28% served, +71% declined on purpose, 0% untriaged. | Group | Served | Declined | Untriaged | Upstream | |---|--:|--:|--:|--:| @@ -893,8 +893,8 @@ what this emulator serves, declines on purpose, or has not triaged yet. | `kms` | 0 | 16 | 0 | 16 | | `block-storage` | 13 | 0 | 0 | 13 | | `dns` | 0 | 10 | 0 | 10 | -| *… 6 smaller groups* | 6 | 7 | 0 | 13 | -| **Total** | **104** | **270** | **0** | **374** | +| *… 6 smaller groups* | 7 | 6 | 0 | 13 | +| **Total** | **105** | **269** | **0** | **374** | #### Outscale diff --git a/coverage/evidence.json b/coverage/evidence.json index f983114d..44da995a 100644 --- a/coverage/evidence.json +++ b/coverage/evidence.json @@ -8,7 +8,7 @@ "generated_from": { "contracts": "ed36f9b500bddee0", "shapes": "cac046c206cc9fc8", - "suites": "ecc2680b2b1c058a" + "suites": "cfe3e26f5afb40b2" }, "operations": { "account/v3/ProjectAPI.CreateProject": { @@ -130,7 +130,7 @@ }, "block/v1/API.ImportSnapshotFromObjectStorage": { "driven": false, - "probed": "none", + "probed": "refusal", "contract": "unchecked", "dataplane": false, "shape": "unobserved", @@ -238,7 +238,7 @@ }, "block/v1alpha1/API.ImportSnapshotFromObjectStorage": { "driven": false, - "probed": "none", + "probed": "refusal", "contract": "unchecked", "dataplane": false, "shape": "unobserved", @@ -929,6 +929,15 @@ "behaviour": true, "negative": false }, + "exoscale/v2.list-sos-buckets-usage": { + "driven": true, + "probed": "response", + "contract": "clean", + "dataplane": true, + "shape": "unobserved", + "behaviour": false, + "negative": false + }, "exoscale/v2.list-ssh-keys": { "driven": true, "probed": "response", diff --git a/coverage/exoscale-coverage.json b/coverage/exoscale-coverage.json index ddb1e9a6..42e8ecfc 100644 --- a/coverage/exoscale-coverage.json +++ b/coverage/exoscale-coverage.json @@ -1,23 +1,23 @@ { "provider": "exoscale", "total": 374, - "implemented": 104, - "declined": 270, + "implemented": 105, + "declined": 269, "unknown": 0, "orphans": null, "products": [ { "product": "exoscale", "total": 374, - "implemented": 104, - "declined": 270, + "implemented": 105, + "declined": 269, "unknown": 0, "versions": [ { "version": "2.0.0", "total": 374, - "implemented": 104, - "declined": 270, + "implemented": 105, + "declined": 269, "unknown": 0 } ] @@ -118,8 +118,8 @@ { "group": "sos", "total": 2, - "implemented": 0, - "declined": 2, + "implemented": 1, + "declined": 1, "unknown": 0 } ], @@ -1782,7 +1782,7 @@ { "operation": "exoscale/v2.get-sos-presigned-url", "product": "exoscale", - "reason": "object storage is refused across this project for a measured reason: clients build the S3 endpoint in code, so serving it needs DNS interception and a certificate, recorded in docs/limits.md; two surveyed stacks in fifteen ask for it (platform's backup buckets, eu-data-platform's Terraform state backend, 2026-08 — #284), and both leave for the real sos-\u003czone\u003e.exo.io by that same self-resolution, the escape #280 exists to detect", + "reason": "signed public download URLs require authentication and externally reachable S3 authorities, neither of which the explicit local byte doors promise", "version": "2.0.0", "group": "sos", "status": "declined" @@ -2126,10 +2126,9 @@ { "operation": "exoscale/v2.list-sos-buckets-usage", "product": "exoscale", - "reason": "object storage is refused across this project for a measured reason: clients build the S3 endpoint in code, so serving it needs DNS interception and a certificate, recorded in docs/limits.md; two surveyed stacks in fifteen ask for it (platform's backup buckets, eu-data-platform's Terraform state backend, 2026-08 — #284), and both leave for the real sos-\u003czone\u003e.exo.io by that same self-resolution, the escape #280 exists to detect", "version": "2.0.0", "group": "sos", - "status": "declined" + "status": "implemented" }, { "operation": "exoscale/v2.list-ssh-keys", diff --git a/docs/confidence.md b/docs/confidence.md index eb5534ec..850428f1 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -17,7 +17,7 @@ operation has earned — the generated block below carries the count — and -**408 operations mounted, 383 driven by a real client** in the recorded run. The +**409 operations mounted, 384 driven by a real client** in the recorded run. The 25 that are not each state why at their route, and [routes.md](routes.md) prints the reason under the pack that owns it. diff --git a/docs/evidence/exoscale-sos-empty-bucket.json b/docs/evidence/exoscale-sos-empty-bucket.json new file mode 100644 index 00000000..dd2dc94b --- /dev/null +++ b/docs/evidence/exoscale-sos-empty-bucket.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "provider": "exoscale", + "region": "ch-gva-2", + "observed_at": "2026-11-07", + "client": "exo 1.95.1 (7aaf76d0b13b8e10a343ec88c9cb03116f09b4c1)", + "provenance": "User-authorized temporary empty bucket, current account used only internally by exo; scoped TLS recorder allowed only that bucket. Seven exchanges, zero drops, zero out-of-scope forwards. DELETE 204 witnessed. No object or other resource written.", + "redaction": "Identity values, request identifiers and authentication removed before persistence. XML below retains recorded protocol members, with canonical owner identity replaced by a local fixture; it is not an unmodified wire transcript.", + "exchanges": [ + {"method": "PUT", "query": "", "status": 200, "body": ""}, + { + "method": "GET", + "query": "acl=", + "status": 200, + "body": "REDACTEDREDACTEDREDACTEDREDACTEDFULL_CONTROL" + }, + { + "method": "GET", + "query": "cors=", + "status": 404, + "body": "NoSuchCORSConfigurationThe CORS configuration does not exist.feint-measured-bucket" + }, + { + "method": "GET", + "query": "ownershipControls=", + "status": 400, + "body": "OwnershipControlsNotFoundErrorThe bucket ownership controls were not found" + }, + {"method": "HEAD", "query": "", "status": 403, "body": "", "note": "Unsigned discovery; region header present. This is not evidence of authenticated access refusal."}, + {"method": "GET", "query": "uploads=", "status": 200, "body": "feint-measured-bucket1000false"}, + {"method": "DELETE", "query": "", "status": 204, "body": ""} + ] +} diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index 9302b5aa..aba45a8d 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -92,9 +92,10 @@ The configured-client door must be proved against clients that actually support the chosen endpoint prefix. If a client rejects it, revisit this decision rather than quietly add a second port or change its workload. -Cloud virtual hosts, TLS interception, hardcoded Scaleway Terraform endpoints, -checkpoint/reset, garbage collection and public-stack certification remain -later work. The synthetic HTTP host slice is described below. +The sections below preserve the sequence of measured slices. The final sections +describe the current Exoscale discovery, isolated Terraform witness and neutral +byte checkpoints. Coordinated resource/byte reset, a user-facing isolated run +command and public-stack certification remain later work. ## Implemented mounted slice @@ -171,7 +172,8 @@ ancillary operations and missing-object cloud errors still receive explicit Feint refusals. Multipart is measured separately below; ordinary object reads still do not advertise ETags. The snapshot-import route still records an intention rather than reading these -bytes. Object Storage 1.0 and hardcoded Terraform redirection are not delivered. +bytes. Object Storage 1.0 is not delivered; the isolated Terraform storage +witness below now intercepts its hardcoded S3 endpoint. ## Measured Scaleway bucket workflow @@ -219,9 +221,9 @@ index. Existing bytes, generations and quotas remain unchanged. Legacy buckets without a stored creation date receive an explicit listing refusal rather than an invented date. Resource snapshots still do not own this metadata. -Complete Exoscale bucket creation, the native bucket-usage API, and full Object -Storage 1.0 remain separate follow-up work. Multipart inventory and bucket -removal now operate on real sessions, as measured below. +Normal Exoscale creation and native usage are now measured below. Full Object +Storage 1.0 remains separate work. Multipart inventory and bucket removal +operate on real sessions. ## Exoscale prefix incompatibility measured @@ -266,7 +268,7 @@ exo storage upload input.bin sos://existing-bucket/key eval "$(feint env exoscale --s3 --unset)" ``` -The bucket must already exist. Normal Exoscale bucket creation, download and +For this initial upload-only slice, the bucket had to exist. Creation, download and cleanup workflows are **not** certified by this upload proof. For Exoscale, `env --s3` selects the live declared authority and exports upper/lower-case @@ -418,6 +420,78 @@ copy, optional checksum negotiation beyond the measured requests and provider cloud-error dialects remain explicit Feint refusals, not invented successes. `tools/falsify/specs/s3-multipart.json` replays the input, scope, bound and -transactional-publication checks. Full Object Storage 1.0, native Exoscale usage, -ACL/CORS/ownership discovery, public stacks and TLS/Terraform interception remain -outside this delivery. +transactional-publication checks. The following slices add default discovery, +native usage and a bounded Terraform witness. Full Object Storage 1.0 and +public-stack certification remain outside this delivery. + +## Ordinary Exoscale bucket lifecycle and native usage + +`mise run conformance:s3:mounted` now runs ordinary `exo storage mb` and +`storage show`, then upload/list/download and bucket removal. Default private +ACL, missing CORS (404) and missing ownership controls (**400**, not 404) follow +the seven-exchange, user-authorized empty-bucket recording in +[exoscale-sos-empty-bucket.json](./evidence/exoscale-sos-empty-bucket.json). +That temporary cloud bucket was deleted; the ongoing witnesses use fake keys. +Setting writes remain refused, not invented successes. + +`GET /v2/sos-buckets-usage` reports persisted creation dates and logical +published object sizes from the same neutral store. It excludes retained blobs +and unpublished upload parts. Disabled storage and malformed legacy metadata +produce explicit errors. The mounted witness compares `exo storage show` and +native usage across process restart, and proves native inventory empty after +removal. Its native control plane and S3 door both use `ch-gva-2`; this does not +certify a mismatched zone configuration. + +## Isolated Scaleway Terraform storage witness + +The official **Scaleway provider 2.82.0** ignores `SCW_S3_ENDPOINT` in its S3 +client and uses cloud-shaped HTTPS virtual hosts. `s3.LocalTransport` maps only +declared authorities to an explicit loopback emulator; unknown hosts never dial. +The existing proxy supplies a temporary CA and exact-SNI leaves. Dotted bucket +names and literal encoded keys are held by transport tests; the Terraform +witness uses an explicitly allowed bucket hostname. + +Recorded private bucket ACL writes, absent CORS/object-lock/lifecycle errors, +ListObjects v1 and object private-ACL/empty-tag reads follow the pinned provider's +[official object cassette](https://github.com/scaleway/terraform-provider-scaleway/blob/731e3dce545df08853ea490cd144de10b3216492/internal/services/object/testdata/object-basic.cassette.yaml). +The Scaleway ACL identity names the first declared local project (the default +project if none was declared). This is a single storage-tenant model, not IAM. +Public ACLs, tag writes and object-version histories remain refused. + +`TestScalewayTerraformStorageInIsolatedNetwork` requires a disposable Linux +network namespace with **only loopback up**, and refuses root execution. +Download the published provider outside that namespace first; point +`FEINT_TF_PLUGIN_DIR` at its installed provider directory. Then: + +```bash +testbin="$(mktemp)" +go test -c -tags=conformance ./internal/cli -o "$testbin" +cd internal/cli +sudo -n unshare --net -- sh -c \ + 'ip link set lo up && exec setpriv --reuid="$1" --regid="$2" --init-groups env PATH="$3" FEINT_TF_PLUGIN_DIR="$4" "$5" -test.run=^TestScalewayTerraformStorageInIsolatedNetwork$ -test.v -test.timeout=5m' \ + sh "$(id -u)" "$(id -g)" "$PATH" "$FEINT_TF_PLUGIN_DIR" "$testbin" +rm "$testbin" +``` + +The witness checks an unreachable external address, apply, an empty second +plan, **122,880 identical bytes**, byte-store close/reopen, another empty plan, +destroy and explicit owned GC. This is the storage-shaped bucket/object graph, +not a certified Flatcar image: the qcow2-named payload is arbitrary test bytes. +There is no provider fork, global hosts/DNS/CA change or cloud-capable fallback. +The proxy is a test harness, **not a shipped `feint run` command**. + +## Neutral byte checkpoints and explicit GC + +The neutral store now exposes durable named save/reset/delete checkpoints and +explicit collection. Checkpoints preserve complete byte manifests and upload +sessions in the same owned root. Exclusive checkpoint/reset/GC operations block +active mutations; collection preserves current and checkpoint-referenced blobs, +with deduplication/hardlink accounting. + +Race tests and `tools/falsify/specs/objectstore-checkpoints.json` hold this +library slice. Readers can delay exclusive operations; quota accounting excludes +unlinked files still held open by a reader. Failed GC can have partially collected +unreferenced files and reports the error. Malformed checkpoints require repair. +These checkpoints are neither portable exports nor coordinated control-plane/ +runtime snapshots. The CLI's resource snapshots still omit object bytes; #853 +is not complete. diff --git a/docs/routes.md b/docs/routes.md index 82d80be2..cfacb875 100644 --- a/docs/routes.md +++ b/docs/routes.md @@ -24,7 +24,7 @@ up to. -408 operations served across 3 packs, counted from the record of the last +409 operations served across 3 packs, counted from the record of the last recorded conformance run (machines: incus, none). Reproduce it yourself, offline, from the committed artefact: @@ -41,10 +41,10 @@ a workflow. None of them opens a socket. | Cloud | Served | `driven` | `probed` | `contract` | `dataplane` | `shape` | `behaviour` | `negative` | |---|---|---|---|---|---|---|---|---| -| Exoscale | 104 | 88 % (91) | 92 % (96) | 92 % (96) | 88 % (91) | 32 % (33) | 80 % (83) | 17 % (18) | +| Exoscale | 105 | 88 % (92) | 92 % (97) | 92 % (97) | 88 % (92) | 31 % (33) | 79 % (83) | 17 % (18) | | Outscale | 100 | 100 % (100) | 100 % (100) | 100 % (100) | 100 % (100) | 94 % (94) | 89 % (89) | 96 % (96) | -| Scaleway | 204 | 94 % (192) | 98 % (199) | 99 % (201) | 94 % (192) | 49 % (100) | 87 % (178) | 70 % (142) | -| **All three** | 408 | 94 % (383) | 97 % (395) | 97 % (397) | 94 % (383) | 56 % (227) | 86 % (350) | 63 % (256) | +| Scaleway | 204 | 94 % (192) | 99 % (201) | 99 % (201) | 94 % (192) | 49 % (100) | 87 % (178) | 70 % (142) | +| **All three** | 409 | 94 % (384) | 97 % (398) | 97 % (398) | 94 % (384) | 56 % (227) | 86 % (350) | 63 % (256) | What each axis says, one line each. They are independent and are never added into one number: none of them implies another, and an operation can be driven @@ -81,7 +81,7 @@ span that still cannot attribute a touch says how many it lost (#398). -410 routes across 3 packs. Every route names the upstream operation it +411 routes across 3 packs. Every route names the upstream operation it stands for, in the provider's own spelling: that name is what the drift scan matches against the upstream SDK, so a route that renames it stops being counted. @@ -185,10 +185,10 @@ disappears from the suite. | `PATCH` | `/block/v1/zones/{zone}/volumes/{id}` | `block/v1/API.UpdateVolume` | `client` `contract` `runtime` `probe` `behaviour` | | `PATCH` | `/block/v1alpha1/zones/{zone}/snapshots/{id}` | `block/v1alpha1/API.UpdateSnapshot` | `client` `contract` `shape` `runtime` `probe` `behaviour` `negative` | | `PATCH` | `/block/v1alpha1/zones/{zone}/volumes/{id}` | `block/v1alpha1/API.UpdateVolume` | `client` `contract` `shape` `runtime` `probe` `behaviour` `negative` | -| `POST` | `/block/v1/zones/{zone}/snapshots/import-from-object-storage` | `block/v1/API.ImportSnapshotFromObjectStorage` | `no-client` | +| `POST` | `/block/v1/zones/{zone}/snapshots/import-from-object-storage` | `block/v1/API.ImportSnapshotFromObjectStorage` | `no-client` `probe-refusal` | | `POST` | `/block/v1/zones/{zone}/snapshots` | `block/v1/API.CreateSnapshot` | `client` `contract` `runtime` `probe` `behaviour` | | `POST` | `/block/v1/zones/{zone}/volumes` | `block/v1/API.CreateVolume` | `client` `contract` `runtime` `probe` `behaviour` | -| `POST` | `/block/v1alpha1/zones/{zone}/snapshots/import-from-object-storage` | `block/v1alpha1/API.ImportSnapshotFromObjectStorage` | `no-client` | +| `POST` | `/block/v1alpha1/zones/{zone}/snapshots/import-from-object-storage` | `block/v1alpha1/API.ImportSnapshotFromObjectStorage` | `no-client` `probe-refusal` | | `POST` | `/block/v1alpha1/zones/{zone}/snapshots` | `block/v1alpha1/API.CreateSnapshot` | `client` `contract` `shape` `runtime` `probe` `behaviour` `negative` | | `POST` | `/block/v1alpha1/zones/{zone}/volumes` | `block/v1alpha1/API.CreateVolume` | `client` `contract` `shape` `runtime` `probe` `behaviour` `negative` | @@ -891,6 +891,12 @@ are in `coverage/`, one artefact per provider. | `GET` | `/v2/quota/{name}` | `exoscale/v2.get-quota` | `no-client` | | `GET` | `/v2/quota` | `exoscale/v2.list-quotas` | `client` `contract` `shape` `runtime` `probe` | +### `sos` + +| Method | Path | Upstream operation | Proven by | +|---|---|---|---| +| `GET` | `/v2/sos-buckets-usage` | `exoscale/v2.list-sos-buckets-usage` | `client` `contract` `runtime` `probe` | + ### Served, and driven by no client (13) Mounted operations no official client reaches, each with the reason. They @@ -912,7 +918,7 @@ reason that outlived its cause. - `compute` — 1 operation — the list this pack serves is empty, because an emulated account owns no dedicated hardware, so no client ever holds an id to read; the Terraform provider does model deploy targets and is stopped by the same emptiness, not by a gap in the client - `quotas` — 1 operation — `exo limits` reads the whole quota list and prints it, so the per-name read has no client path even though the SDK declares one, and the word Quota does not appear anywhere in the Terraform provider's own code (v0.71.0) -### Declined on purpose (270) +### Declined on purpose (269) Operations this pack knowingly does not serve, and why. Declining is a decision the drift gate records, which is what separates it from having @@ -933,5 +939,5 @@ are in `coverage/`, one artefact per provider. - `kms` — 16 operations — a key management service that emulated its own cryptography would return ciphertext no real key ever produced, which is worse than refusing - `organization` — 3 operations — there is no account behind this emulator and no consumption to report, so any figure here would be one it invented and somebody planned against - `sks` — 25 operations — a managed Kubernetes control plane on hosts the emulator does not run, so a kubeconfig it issued would point nowhere -- `sos` — 2 operations — object storage is refused across this project for a measured reason: clients build the S3 endpoint in code, so serving it needs DNS interception and a certificate, recorded in docs/limits.md; two surveyed stacks in fifteen ask for it (platform's backup buckets, eu-data-platform's Terraform state backend, 2026-08 — #284), and both leave for the real sos-.exo.io by that same self-resolution, the escape #280 exists to detect +- `sos` — 1 operation — signed public download URLs require authentication and externally reachable S3 authorities, neither of which the explicit local byte doors promise diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 9483c544..9f2af9ec 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1174,6 +1174,7 @@ func serve(args []string, stdout io.Writer) (result error) { return err } if objects != nil { + env.Objects = objects defer func() { if err := objects.Close(); err != nil { result = errors.Join(result, fmt.Errorf("close object storage: %w", err)) diff --git a/internal/cli/object_storage_conformance_test.go b/internal/cli/object_storage_conformance_test.go index 97a66cfd..caaf3d8c 100644 --- a/internal/cli/object_storage_conformance_test.go +++ b/internal/cli/object_storage_conformance_test.go @@ -39,7 +39,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("local proxy control did not record its refusal: %v hits=%d", err, guardHits.Load()) } aws := conformancetest.AWS(t, guard.URL, "") - aws.Env = append(aws.Env, "AWS_REQUEST_CHECKSUM_CALCULATION=when_required") + aws.Env = append(aws.Env, "AWS_REQUEST_CHECKSUM_CALCULATION=when_required", "FEINT_EXOSCALE_ZONE=ch-gva-2") binary := filepath.Join(t.TempDir(), "feint") buildCtx, cancelBuild := context.WithTimeout(t.Context(), 2*time.Minute) build := exec.CommandContext(buildCtx, "go", "build", "-o", binary, "../../cmd/feint") @@ -183,22 +183,6 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } } get := func(door s3.Door, expected []byte) { getKey(door, key, expected) } - for i, door := range first.ObjectStorage.Doors { - if door.Provider == "scaleway" { - bucketCall("create", true) - } else { - call(door, "create-bucket", "--bucket", bucket) - } - if i == 1 { - if err := os.WriteFile(input, []byte("separate-sos-scope"), 0600); err != nil { - t.Fatal(err) - } - } - call(door, "put-object", "--bucket", bucket, "--key", key, "--body", input, - "--content-type", "application/octet-stream", "--checksum-algorithm", "CRC32") - } - get(first.ObjectStorage.Doors[0], payload) - get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) // Use each client's exact exported environment, without evaluating shell text. var exoVars map[string]string hostSCW := scw @@ -242,6 +226,33 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { } hostSCW.Run(t, "-o", "json", "object", "bucket", "delete", "feint-host-scw", "region=fr-par") exo := conformancetest.EXO(t, aws, exoVars) + beforeExoBucket := exo.Run(t, "-O", "json", "storage", "mb", "--zone", "ch-gva-2", "sos://"+bucket) + var discovery struct { + Name string `json:"name"` + Zone string `json:"zone"` + CORS []any `json:"cors"` + ObjectOwnership string `json:"objectOwnership"` + } + if err := json.Unmarshal(beforeExoBucket, &discovery); err != nil || discovery.Name != bucket || + discovery.Zone != "ch-gva-2" || discovery.ObjectOwnership != "BucketOwnerEnforced" || len(discovery.CORS) != 0 { + t.Fatalf("ordinary Exo creation/discovery did not reflect the actual private bucket: %v %q", err, beforeExoBucket) + } + for i, door := range first.ObjectStorage.Doors { + if door.Provider == "scaleway" { + bucketCall("create", true) + } + initial := payload + if i == 1 { + initial = []byte("separate-sos-scope") + } + if err := os.WriteFile(input, initial, 0600); err != nil { + t.Fatal(err) + } + call(door, "put-object", "--bucket", bucket, "--key", key, "--body", input, + "--content-type", "application/octet-stream", "--checksum-algorithm", "CRC32") + } + get(first.ObjectStorage.Doors[0], payload) + get(first.ObjectStorage.Doors[1], []byte("separate-sos-scope")) if err := os.WriteFile(input, payload, 0600); err != nil { t.Fatal(err) } @@ -361,6 +372,17 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if !foundLarge { t.Fatal("actual Exo listing omitted size/date of the uploaded object:", string(beforeObjects)) } + beforeNativeUsage := exo.Run(t, "-O", "json", "storage", "list", "--zone", "ch-gva-2") + var nativeUsage []struct { + Name, Zone, Created string + Size int64 + } + expectedUsage := int64(len("separate-sos-scope") + len(payload) + len(largePayload)) + if err := json.Unmarshal(beforeNativeUsage, &nativeUsage); err != nil || len(nativeUsage) != 1 || + nativeUsage[0].Name != bucket || nativeUsage[0].Zone != "ch-gva-2" || + nativeUsage[0].Created == "" || nativeUsage[0].Size != expectedUsage { + t.Fatalf("native Exo usage is not derived from the actual published bytes: %v %q", err, beforeNativeUsage) + } // A same-name object in the other provider is a positive isolation control. if err := os.WriteFile(input, []byte("other-provider-host-key"), 0600); err != nil { t.Fatal(err) @@ -394,6 +416,12 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { if after := exo.Run(t, "--output-format", "json", "storage", "list", "sos://"+bucket+"/"); !bytes.Equal(after, beforeObjects) { t.Fatalf("actual Exo object inventory changed across restart: before=%q after=%q", beforeObjects, after) } + if after := exo.Run(t, "-O", "json", "storage", "show", "sos://"+bucket); !bytes.Equal(after, beforeExoBucket) { + t.Fatalf("ordinary Exo bucket discovery changed across restart: before=%q after=%q", beforeExoBucket, after) + } + if after := exo.Run(t, "-O", "json", "storage", "list", "--zone", "ch-gva-2"); !bytes.Equal(after, beforeNativeUsage) { + t.Fatalf("native Exo usage changed across restart: before=%q after=%q", beforeNativeUsage, after) + } if after := uploadInventory(); !bytes.Equal(after, beforeUploads) { t.Fatalf("upload sessions changed across actual restart: before=%q after=%q", beforeUploads, after) } @@ -424,7 +452,7 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { getKey(first.ObjectStorage.Doors[1], "after-restart", []byte("other-provider-host-key")) var live statusReport if err := json.Unmarshal(run("status", "--addr", addr, "--format", "json"), &live); err != nil || - live.ObjectStorage == nil || live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 4 { + live.ObjectStorage == nil || live.ObjectStorage.HTTPRequests["http://"+first.ObjectStorage.Doors[1].Host+" 200"] != 6 { t.Fatalf("actual Exo HEAD/PUT after restart did not traverse its declared host: %+v %v", live.ObjectStorage, err) } download(exoKey, payload) @@ -524,6 +552,9 @@ func TestMountedS3OfficialAWSCLIBytes(t *testing.T) { t.Fatalf("dangling-session positive control was lost: %q", out) } exo.Run(t, "storage", "rb", "--force", "sos://"+bucket) + if out := exo.Run(t, "-O", "json", "storage", "list", "--zone", "ch-gva-2"); json.Unmarshal(out, &nativeUsage) != nil || len(nativeUsage) != 0 { + t.Fatalf("native Exo usage retained the deleted bucket: %q", out) + } continue } call(door, "head-object", "--bucket", bucket, "--key", key) diff --git a/internal/cli/object_storage_terraform_conformance_test.go b/internal/cli/object_storage_terraform_conformance_test.go new file mode 100644 index 00000000..4caa3725 --- /dev/null +++ b/internal/cli/object_storage_terraform_conformance_test.go @@ -0,0 +1,215 @@ +//go:build conformance + +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/conformancetest" + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" + "github.com/stephrobert/feint/internal/providers/scaleway" + "github.com/stephrobert/feint/internal/proxy" + "github.com/stephrobert/feint/internal/s3" +) + +// This witness refuses to run on the host network. The pinned provider ignores +// its S3 endpoint; the scoped TLS proxy captures its real cloud authorities. +func TestScalewayTerraformStorageInIsolatedNetwork(t *testing.T) { + if os.Geteuid() == 0 { + t.Fatal("drop privileges after creating the disposable network namespace") + } + interfaces, err := net.Interfaces() + if err != nil { + t.Fatal(err) + } + if len(interfaces) != 1 || interfaces[0].Name != "lo" || interfaces[0].Flags&net.FlagUp == 0 { + t.Fatal("explicit proof requires a fresh network namespace with only loopback up") + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp4", "192.0.2.1:443") + if err == nil { + connection.Close() + t.Fatal("namespace escape control connected outside loopback") + } + pluginDir := os.Getenv("FEINT_TF_PLUGIN_DIR") + if !filepath.IsAbs(pluginDir) { + t.Fatal("FEINT_TF_PLUGIN_DIR must name the predownloaded, pinned provider directory") + } + terraform, err := exec.LookPath("terraform") + if err != nil { + t.Fatal("the explicit Terraform witness requires Terraform:", err) + } + home := privateStorageDir(t) + payload := bytes.Repeat([]byte("feint Terraform storage bytes\n"), 4096) + if err := os.WriteFile(filepath.Join(home, "payload"), payload, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, "terraform.rc"), nil, 0600); err != nil { + t.Fatal(err) + } + objects, err := objectstore.OpenWithLimit(t.Context(), filepath.Join(home, "bytes"), 16<<20) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := objects.Close(); err != nil { + t.Error(err) + } + }() + env := emulator.DefaultEnv() + profile := scaleway.New(env).ObjectStorage("fr-par") + mount := func() *httptest.Server { + t.Helper() + handler, err := s3.Mount(http.NotFoundHandler(), objects, []s3.Profile{profile}, + 1<<20, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + return httptest.NewServer(handler) + } + server := mount() + defer func() { server.Close() }() + endpoint := server.URL + strings.TrimSuffix(s3.DoorPath("scaleway", "fr-par"), "/") + authority, err := proxy.MintAuthority() + if err != nil { + t.Fatal(err) + } + ca := filepath.Join(home, "ca.pem") + if err := authority.WriteCA(ca); err != nil { + t.Fatal(err) + } + writer := proxy.NewWriter(io.Discard, 64) + defer func() { + if err := writer.Close(); err != nil { + t.Error(err) + } + }() + startProxy := func() (*httptest.Server, *s3.LocalTransport) { + t.Helper() + transport, err := s3.NewLocalTransport(server.URL, []s3.Profile{profile}) + if err != nil { + t.Fatal(err) + } + forward, err := proxy.NewForward(proxy.ForwardOptions{ + Hosts: []string{profile.CloudHost, "feint-terraform-storage." + profile.CloudHost}, + Writer: writer, Authority: authority, Transport: transport, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatal(err) + } + return httptest.NewServer(forward), transport + } + egress, transport := startProxy() + defer func() { egress.Close(); transport.CloseIdleConnections() }() + config := ` +terraform { + required_providers { + scaleway = { + source = "scaleway/scaleway" + version = "= 2.82.0" + } + } +} +provider "scaleway" {} +resource "scaleway_object_bucket" "storage" { + name = "feint-terraform-storage" + region = "fr-par" +} +resource "scaleway_object" "payload" { + bucket = scaleway_object_bucket.storage.name + key = "flatcar_production_scaleway_image.qcow2" + file = "${path.module}/payload" + hash = filemd5("${path.module}/payload") + content_type = "application/octet-stream" +} +` + if err := os.WriteFile(filepath.Join(home, "main.tf"), []byte(config), 0600); err != nil { + t.Fatal(err) + } + fake := conformancetest.FakeCredentials(t) + run := func(args ...string) []byte { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, terraform, args...) + cmd.Dir = home + cmd.Env = []string{"HOME=" + home, "PATH=" + os.Getenv("PATH"), "TF_IN_AUTOMATION=1", "CHECKPOINT_DISABLE=1", + "TF_CLI_CONFIG_FILE=" + filepath.Join(home, "terraform.rc"), + "SCW_S3_ENDPOINT=" + endpoint, + "SCW_API_URL=" + server.URL, "SCW_INSECURE=false", "SSL_CERT_FILE=" + ca, + "HTTP_PROXY=" + egress.URL, "HTTPS_PROXY=" + egress.URL, "NO_PROXY=127.0.0.1,localhost"} + for key, value := range fake { + if key != "SCW_API_URL" && key != "SCW_INSECURE" { + cmd.Env = append(cmd.Env, key+"="+value) + } + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("isolated terraform %v: %v\n%s", args, err, out) + } + return out + } + run("init", "-input=false", "-no-color", "-plugin-dir="+pluginDir) + run("apply", "-auto-approve", "-input=false", "-no-color") + run("plan", "-input=false", "-no-color", "-detailed-exitcode") + checkBytes := func() { + t.Helper() + response, err := http.Get(endpoint + "/feint-terraform-storage/flatcar_production_scaleway_image.qcow2") //nolint:noctx // loopback test server with the test's lifetime + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + actual, err := io.ReadAll(response.Body) + if err != nil || response.StatusCode != 200 || !bytes.Equal(actual, payload) { + t.Fatal("Terraform did not publish identical bytes:", response.StatusCode, err) + } + } + checkBytes() + egress.Close() + transport.CloseIdleConnections() + server.Close() + if err := objects.Close(); err != nil { + t.Fatal(err) + } + objects, err = objectstore.OpenWithLimit(t.Context(), filepath.Join(home, "bytes"), 16<<20) + if err != nil { + t.Fatal(err) + } + server = mount() + egress, transport = startProxy() + endpoint = server.URL + strings.TrimSuffix(s3.DoorPath("scaleway", "fr-par"), "/") + checkBytes() + run("plan", "-input=false", "-no-color", "-detailed-exitcode") + run("destroy", "-auto-approve", "-input=false", "-no-color") + inventory, err := objects.Usage(t.Context()) + if err != nil || len(inventory) != 0 { + t.Fatal("destroy left actual object references:", inventory, err) + } + if err := objects.CollectGarbage(t.Context()); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + written, dropped := writer.Stats() + if written == 0 || dropped != 0 { + t.Fatal("interception witness was empty or incomplete:", written, dropped) + } + t.Log(fmt.Sprintf("Scaleway 2.82.0: apply, two empty plans, %d identical bytes across reopen, destroy and owned GC; cloud-host TLS interception, loopback-only namespace", len(payload))) +} diff --git a/internal/conformancetest/aws.go b/internal/conformancetest/aws.go index 5d172c7a..fa887639 100644 --- a/internal/conformancetest/aws.go +++ b/internal/conformancetest/aws.go @@ -34,7 +34,7 @@ func AWS(t *testing.T, proxy, ca string) Client { if err := os.Chmod(home, 0700); err != nil { //nolint:gosec // a private directory, not a file t.Fatal(err) } - fake := fakeCredentials(t) + fake := FakeCredentials(t) cli := Client{Path: binary, Home: home, Env: []string{ "HOME=" + home, "PATH=" + filepath.Dir(binary) + ":/usr/bin:/bin", "XDG_STATE_HOME=" + home, "XDG_RUNTIME_DIR=" + home, @@ -56,7 +56,8 @@ func AWS(t *testing.T, proxy, ca string) Client { return cli } -func fakeCredentials(t *testing.T) map[string]string { +// FakeCredentials reads the single public-fake credential fixture. +func FakeCredentials(t *testing.T) map[string]string { t.Helper() creds, err := os.ReadFile("../../tools/conformance/scaleway/fake-credentials.env") if err != nil { diff --git a/internal/conformancetest/scw.go b/internal/conformancetest/scw.go index 61e709d8..374bd01b 100644 --- a/internal/conformancetest/scw.go +++ b/internal/conformancetest/scw.go @@ -36,7 +36,7 @@ func SCW(t *testing.T, base Client, endpoint string) Client { } client.Env = append(client.Env, "PATH="+filepath.Dir(binary)+":/usr/bin:/bin", "SCW_S3_ENDPOINT="+endpoint, "SCW_API_URL="+api.String(), "SCW_INSECURE=true", "SCW_CACHE_DIR="+cache) - for key, value := range fakeCredentials(t) { + for key, value := range FakeCredentials(t) { if key != "SCW_API_URL" { client.Env = append(client.Env, key+"="+value) } diff --git a/internal/core/emulator/emulator.go b/internal/core/emulator/emulator.go index 251ada27..dd457954 100644 --- a/internal/core/emulator/emulator.go +++ b/internal/core/emulator/emulator.go @@ -20,6 +20,7 @@ import ( "github.com/stephrobert/feint/internal/contract" "github.com/stephrobert/feint/internal/core/machine" + "github.com/stephrobert/feint/internal/core/objectstore" "github.com/stephrobert/feint/internal/core/store" "github.com/stephrobert/feint/internal/trace" ) @@ -28,8 +29,11 @@ import ( // so tests can pin timestamps and identifiers and compare golden output. type Env struct { Store *store.Store - Now func() time.Time - NewID func() string + // Objects is the optional neutral durable byte inventory, wired before HTTP + // starts. A nil value means Object Storage was not enabled. + Objects *objectstore.Store + Now func() time.Time + NewID func() string // machines backs powered-on servers with something that actually runs. // Defaults to the metadata-only driver, so nothing ever starts unless the // operator asked for it. diff --git a/internal/core/objectstore/bucket_metadata.go b/internal/core/objectstore/bucket_metadata.go index 8b3ca71b..6bf4b279 100644 --- a/internal/core/objectstore/bucket_metadata.go +++ b/internal/core/objectstore/bucket_metadata.go @@ -2,9 +2,42 @@ package objectstore import ( "context" + "fmt" "maps" + "math" ) +// BucketUsage describes one consistent published inventory, excluding retained +// orphan blobs and unpublished multipart parts. Metadata is detached. +type BucketUsage struct { + Metadata map[string]string + Size int64 +} + +// Usage returns logical published bytes per bucket under one metadata read lock. +// Duplicate references count separately: this is usage, not physical disk space. +func (s *Store) Usage(ctx context.Context) (map[string]BucketUsage, error) { + end, err := s.begin(ctx) + if err != nil { + return nil, err + } + defer end() + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[string]BucketUsage, len(s.state.Buckets)) + for name, bucket := range s.state.Buckets { + usage := BucketUsage{Metadata: maps.Clone(bucket.Metadata)} + for _, object := range bucket.Objects { + if object.Size > math.MaxInt64-usage.Size { + return nil, fmt.Errorf("%w: logical bucket usage exceeds representable bytes", ErrInvalid) + } + usage.Size += object.Size + } + out[name] = usage + } + return out, nil +} + // BucketMetadata returns a detached copy of a bucket's opaque UTF-8 attributes. // A legacy bucket with no attributes returns an empty map. func (s *Store) BucketMetadata(ctx context.Context, name string) (map[string]string, error) { diff --git a/internal/core/objectstore/usage_test.go b/internal/core/objectstore/usage_test.go new file mode 100644 index 00000000..98320374 --- /dev/null +++ b/internal/core/objectstore/usage_test.go @@ -0,0 +1,35 @@ +package objectstore + +import ( + "bytes" + "testing" +) + +func TestUsageCountsPublishedLogicalBytesNotRetainedParts(t *testing.T) { + store, _ := newStore(t) + mustBucket(t, store, "bucket") + mustPut(t, store, "bucket", "first", "abc") + mustPut(t, store, "bucket", "duplicate", "abc") + if err := store.SetBucketMetadata(t.Context(), "bucket", "opaque", "stable"); err != nil { + t.Fatal(err) + } + id, err := store.CreateUpload(t.Context(), "bucket", "future", PutOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := store.PutPart(t.Context(), "bucket", "future", id, 1, bytes.NewBufferString("unpublished"), 11); err != nil { + t.Fatal(err) + } + usage, err := store.Usage(t.Context()) + if err != nil || usage["bucket"].Size != 6 || usage["bucket"].Metadata["opaque"] != "stable" { + t.Fatal("logical published usage lost or included retained blobs:", usage, err) + } + usage["bucket"].Metadata["opaque"] = "caller changed" + if err := store.DeleteObject(t.Context(), "bucket", "first"); err != nil { + t.Fatal(err) + } + after, err := store.Usage(t.Context()) + if err != nil || after["bucket"].Size != 3 || after["bucket"].Metadata["opaque"] != "stable" { + t.Fatal("usage aliases metadata or counts deleted references:", after, err) + } +} diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index 118ed18e..58a85307 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -4,12 +4,14 @@ import ( "context" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" "github.com/stephrobert/feint/internal/contract" "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" "github.com/stephrobert/feint/internal/probe" "github.com/stephrobert/feint/internal/providers/exoscale" "github.com/stephrobert/feint/internal/providers/outscale" @@ -45,6 +47,46 @@ func runProbe(t *testing.T, provider string) probe.Report { env := emulator.DefaultEnv() env.Contracts = map[string]*contract.Doc{provider: doc} + + // The byte inventory is wired, and that is a decision rather than a + // convenience. + // + // Routes that read it answer 503 "Object Storage is disabled" without it, and + // the probe counts a 503 to a request built from the operation's own schema + // as a failure — correctly, because the contract declares a 200 body there. + // `exoscale/v2.list-sos-buckets-usage` failed exactly that way the first time + // it was probed. + // + // Skipping the route instead would have been the easier fix and the wrong + // one. The repository skips what it cannot look at: a machine runtime is a + // side effect on the operator's host, so `--vm` stays off by default and a + // check that needs it abstains. A byte inventory is none of that. It is a + // directory, it costs a t.TempDir(), and the probe's whole purpose is to + // prove the shape of what a route answers. An operation whose only probe + // result is "skipped" is an operation nothing checks. + // + // TestEveryRouteAnswersItsContract fails on list-sos-buckets-usage without + // these three lines. + // 0700 on the parent, which the store insists on: it refuses a root whose + // writable parent carries no sticky bit, and a bare t.TempDir() is exactly + // that. The guard is right — a store under a world-writable parent can be + // swapped underneath it — so the test satisfies it rather than working + // around it, the same way the Exoscale and CLI storage tests do. + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatalf("restrict the byte inventory's parent: %v", err) + } + objects, err := objectstore.Open(t.Context(), filepath.Join(root, "objects")) + if err != nil { + t.Fatalf("open the byte inventory the probe needs: %v", err) + } + t.Cleanup(func() { + if err := objects.Close(); err != nil { + t.Errorf("close the byte inventory: %v", err) + } + }) + env.Objects = objects + pack := packOf(t, provider, env) srv, err := emulator.NewServer(env, pack) diff --git a/internal/providers/exoscale/object_storage.go b/internal/providers/exoscale/object_storage.go index 0de6b312..f7653d29 100644 --- a/internal/providers/exoscale/object_storage.go +++ b/internal/providers/exoscale/object_storage.go @@ -1,10 +1,68 @@ package exoscale -import "github.com/stephrobert/feint/internal/s3" +import ( + "net/http" + "sort" + "strings" + "time" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/s3" +) // ObjectStorage declares the measured SOS dialect and configurable CLI endpoints. -// It does not advertise control-plane bucket usage or multipart support. +// Native bucket usage is derived from the same published byte inventory. +// Default discovery errors follow the authorized empty-bucket SOS recording. func (p *Pack) ObjectStorage(region string) s3.Profile { return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ExoscaleMissingBucket, HostEndpoint: true, - Variables: []string{"EXOSCALE_SOS_ENDPOINT", "EXOSCALE_STORAGE_API_ENDPOINT"}, RegionVariable: "EXOSCALE_ZONE"} + Variables: []string{"EXOSCALE_SOS_ENDPOINT", "EXOSCALE_STORAGE_API_ENDPOINT"}, RegionVariable: "EXOSCALE_ZONE", + BucketConfiguration: true, CloudHost: "sos-" + region + ".exo.io", + AbsentConfigurations: map[string]s3.AbsentConfiguration{ + "cors": {Status: 404, Code: "NoSuchCORSConfiguration", Message: "The CORS configuration does not exist.", BucketName: true}, + "ownershipControls": {Status: 400, Code: "OwnershipControlsNotFoundError", Message: "The bucket ownership controls were not found"}, + }} +} + +type sosBucketUsage struct { + Name string `json:"name"` + Created time.Time `json:"created-at"` + Zone string `json:"zone-name"` + Size int64 `json:"size"` +} + +func (p *Pack) listSOSBucketsUsage(w http.ResponseWriter, r *http.Request) { + if p.env.Objects == nil { + writeError(w, http.StatusServiceUnavailable, "Object Storage is disabled; start with --objectstore") + return + } + inventory, err := p.env.Objects.Usage(r.Context()) + if err != nil { + p.env.Log.ErrorContext(r.Context(), "read SOS bucket usage", "error", err) + writeError(w, http.StatusInternalServerError, "unable to read persisted SOS bucket usage") + return + } + items := make([]sosBucketUsage, 0) + for scoped, bucket := range inventory { + rest, ours := strings.CutPrefix(scoped, Name+"/") + if !ours { + continue + } + zone, name, valid := strings.Cut(rest, "/") + created, dateErr := s3.BucketCreationTime(bucket.Metadata) + if !valid || !publishedZone(zone) || name == "" || strings.Contains(name, "/") || dateErr != nil { + p.env.Log.ErrorContext(r.Context(), "unsupported persisted SOS bucket scope or date", "error", dateErr) + writeError(w, http.StatusInternalServerError, "persisted SOS bucket scope or creation date is unsupported") + return + } + items = append(items, sosBucketUsage{Name: name, Zone: zone, Created: created, Size: bucket.Size}) + } + sort.Slice(items, func(i, j int) bool { + if items[i].Zone != items[j].Zone { + return items[i].Zone < items[j].Zone + } + return items[i].Name < items[j].Name + }) + emulator.WriteJSON(w, http.StatusOK, struct { + Items []sosBucketUsage `json:"sos-buckets-usage"` + }{items}) } diff --git a/internal/providers/exoscale/object_storage_test.go b/internal/providers/exoscale/object_storage_test.go new file mode 100644 index 00000000..af3f19df --- /dev/null +++ b/internal/providers/exoscale/object_storage_test.go @@ -0,0 +1,136 @@ +package exoscale + +import ( + "encoding/json" + "encoding/xml" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" + "github.com/stephrobert/feint/internal/s3" +) + +func TestSOSDefaultDiscoveryMatchesAuthorizedCloudCorpus(t *testing.T) { + raw, err := os.ReadFile("../../../docs/evidence/exoscale-sos-empty-bucket.json") + if err != nil { + t.Fatal(err) + } + + var corpus struct { + Exchanges []struct { + Method, Query, Body string + Status int + } + } + if err := json.Unmarshal(raw, &corpus); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + store, err := objectstore.Open(t.Context(), filepath.Join(root, "objects")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }) + if err := store.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + profile := (&Pack{}).ObjectStorage("ch-gva-2") + server, err := s3.New(store, s3.Config{Region: profile.Region, MaxObjectBytes: 1024, + MissingBucket: profile.MissingBucket, BucketConfiguration: profile.BucketConfiguration, + AbsentConfigurations: profile.AbsentConfigurations, Log: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatal(err) + } + observed := 0 + for _, exchange := range corpus.Exchanges { + if exchange.Query != "cors=" && exchange.Query != "ownershipControls=" { + continue + } + t.Run(exchange.Query, func(t *testing.T) { + type cloudError struct{ Code, Message, BucketName string } + var expected cloudError + if err := xml.Unmarshal([]byte(exchange.Body), &expected); err != nil || expected.Code == "" { + t.Fatal("unreadable real error:", err) + } + if expected.BucketName != "" { + expected.BucketName = "bucket" + } + request := httptest.NewRequest("GET", "http://localhost/bucket?"+exchange.Query, nil) + request.RemoteAddr = "127.0.0.1:1234" + response := httptest.NewRecorder() + server.ServeHTTP(response, request) + var actual cloudError + if err := xml.Unmarshal(response.Body.Bytes(), &actual); err != nil || actual != expected || response.Code != exchange.Status { + t.Fatalf("cloud error diverged: status=%d got=%+v want=%+v decode=%v", response.Code, actual, expected, err) + } + request.Method = http.MethodPut + refused := httptest.NewRecorder() + server.ServeHTTP(refused, request) + if refused.Code < 400 { + t.Fatal("unimplemented setting write falsely succeeded") + } + request.URL.Path = "/missing" + request.Method = http.MethodGet + missing := httptest.NewRecorder() + server.ServeHTTP(missing, request) + if missing.Code == exchange.Status && missing.Body.String() == response.Body.String() { + t.Fatal("absent bucket was described as an existing bucket with absent settings") + } + }) + observed++ + } + if observed != 2 { + t.Fatal("corpus did not prove both default errors") + } +} + +func TestSOSUsageRefusesDisabledStorageAndLegacyDates(t *testing.T) { + env := emulator.DefaultEnv() + pack := New(env) + request := httptest.NewRequest("GET", "/v2/sos-buckets-usage", nil) + response := httptest.NewRecorder() + pack.listSOSBucketsUsage(response, request) + if response.Code != http.StatusServiceUnavailable { + t.Fatal("disabled storage returned a success-shaped inventory") + } + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + store, err := objectstore.Open(t.Context(), filepath.Join(root, "objects")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }() + env.Objects = store + response = httptest.NewRecorder() + pack.listSOSBucketsUsage(response, request) + if response.Code != http.StatusOK || response.Body.String() != "{\"sos-buckets-usage\":[]}\n" { + t.Fatal("actual empty enabled inventory was refused:", response.Body.String()) + } + if err := store.CreateBucket(t.Context(), "exoscale/ch-gva-2/legacy"); err != nil { + t.Fatal(err) + } + response = httptest.NewRecorder() + pack.listSOSBucketsUsage(response, request) + if response.Code != http.StatusInternalServerError { + t.Fatal("legacy date was invented") + } +} diff --git a/internal/providers/exoscale/object_storage_usage_test.go b/internal/providers/exoscale/object_storage_usage_test.go new file mode 100644 index 00000000..e57a9a5a --- /dev/null +++ b/internal/providers/exoscale/object_storage_usage_test.go @@ -0,0 +1,69 @@ +package exoscale + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" +) + +func TestSOSUsageReportsOnlyPublishedScopedBytesAndRefusesMalformedScopes(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + store, err := objectstore.Open(t.Context(), filepath.Join(root, "bytes")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }() + env := emulator.DefaultEnv() + env.Objects = store + pack := New(env) + created := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + metadata := map[string]string{"s3.created": created.Format(time.RFC3339Nano)} + if err := store.CreateBucketWithMetadata(t.Context(), "exoscale/ch-gva-2/bucket", metadata); err != nil { + t.Fatal(err) + } + if _, err := store.PutObject(t.Context(), "exoscale/ch-gva-2/bucket", "key", bytes.NewReader([]byte("abc")), objectstore.PutOptions{Size: 3}); err != nil { + t.Fatal(err) + } + if err := store.CreateBucket(t.Context(), "scaleway/foreign"); err != nil { + t.Fatal(err) + } + request := httptest.NewRequest("GET", "/v2/sos-buckets-usage", nil) + response := httptest.NewRecorder() + pack.listSOSBucketsUsage(response, request) + var actual struct { + Items []sosBucketUsage `json:"sos-buckets-usage"` + } + if err := json.Unmarshal(response.Body.Bytes(), &actual); err != nil || response.Code != 200 || + len(actual.Items) != 1 || actual.Items[0].Name != "bucket" || actual.Items[0].Zone != "ch-gva-2" || + actual.Items[0].Size != 3 || !actual.Items[0].Created.Equal(created) { + t.Fatal("native usage invented data or included foreign bytes:", err, response.Code, response.Body.String()) + } + for _, scope := range []string{"exoscale/no-zone/bucket", "exoscale/ch-gva-2/", "exoscale/ch-gva-2/bucket/nested"} { + if err := store.CreateBucketWithMetadata(t.Context(), scope, metadata); err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + pack.listSOSBucketsUsage(response, request) + if response.Code != http.StatusInternalServerError { + t.Fatal("malformed persisted scope was served:", scope, response.Body.String()) + } + if err := store.DeleteBucket(t.Context(), scope); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/providers/exoscale/pack.go b/internal/providers/exoscale/pack.go index c5ea5c77..ded704ca 100644 --- a/internal/providers/exoscale/pack.go +++ b/internal/providers/exoscale/pack.go @@ -139,6 +139,7 @@ func operation(id string) string { return "exoscale/v2." + id } // Routes implements emulator.Pack. func (p *Pack) Routes() []emulator.Route { return p.guardSplitClients([]emulator.Route{ + {Method: "GET", Path: "/v2/sos-buckets-usage", Operation: operation("list-sos-buckets-usage"), Handler: p.listSOSBucketsUsage}, {Method: "GET", Path: "/v2/instance", Operation: operation("list-instances"), Handler: p.listInstances}, {Method: "POST", Path: "/v2/instance", Operation: operation("create-instance"), Handler: p.createInstance}, {Method: "GET", Path: "/v2/instance/{id}", Operation: operation("get-instance"), Handler: p.getInstance}, @@ -714,9 +715,8 @@ func (p *Pack) Declined() []emulator.Decline { "exoscale/v2.update-user-role"), // Simple Object Storage. - emulator.Because("object storage is refused across this project for a measured reason: clients build the S3 endpoint in code, so serving it needs DNS interception and a certificate, recorded in docs/limits.md; two surveyed stacks in fifteen ask for it (platform's backup buckets, eu-data-platform's Terraform state backend, 2026-08 — #284), and both leave for the real sos-.exo.io by that same self-resolution, the escape #280 exists to detect", - "exoscale/v2.get-sos-presigned-url", - "exoscale/v2.list-sos-buckets-usage"), + emulator.Because("signed public download URLs require authentication and externally reachable S3 authorities, neither of which the explicit local byte doors promise", + "exoscale/v2.get-sos-presigned-url"), // The instance console and its passwords. // diff --git a/internal/providers/scaleway/object_storage.go b/internal/providers/scaleway/object_storage.go index 528a9779..1eeca5e6 100644 --- a/internal/providers/scaleway/object_storage.go +++ b/internal/providers/scaleway/object_storage.go @@ -1,12 +1,25 @@ package scaleway -import "github.com/stephrobert/feint/internal/s3" +import ( + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/s3" +) // ObjectStorage declares the measured S3 dialect and configurable CLI endpoint. -// It enables the measured private, non-versioned bucket CLI workflow, not -// Terraform redirection, public permissions or object version histories. +// It enables measured private, non-versioned CLI and Terraform discovery, +// not public permissions, writable tags or object version histories. func (p *Pack) ObjectStorage(region string) s3.Profile { + owner := emulator.Project{Name: defaultProjectName} + if p.env != nil && len(p.env.Projects) != 0 { + owner = p.env.Projects[0] + } return s3.Profile{Provider: Name, Region: region, MissingBucket: s3.ScalewayMissingBucket, Variables: []string{"SCW_S3_ENDPOINT"}, RegionVariable: "SCW_DEFAULT_REGION", - BucketConfiguration: true} + CloudHost: "s3." + region + ".scw.cloud", + BucketConfiguration: true, ObjectDiscovery: true, ListObjectsV1: true, OwnerID: identifierOf(owner), PrivateACLWrite: true, + AbsentConfigurations: map[string]s3.AbsentConfiguration{ + "cors": {Status: 404, Code: "NoSuchCORSConfiguration", Message: "The CORS configuration does not exist", RequestIDs: true, Resource: true}, + "object-lock": {Status: 404, Code: "ObjectLockConfigurationNotFoundError", Message: "Object Lock configuration does not exist for this bucket", RequestIDs: true, Resource: true}, + "lifecycle": {Status: 404, Code: "NoSuchLifecycleConfiguration", Message: "The lifecycle configuration does not exist", RequestIDs: true, Resource: true, BucketName: true}, + }} } diff --git a/internal/s3/bucket_configuration.go b/internal/s3/bucket_configuration.go index d334e442..3dad46e3 100644 --- a/internal/s3/bucket_configuration.go +++ b/internal/s3/bucket_configuration.go @@ -25,6 +25,33 @@ const ( bucketCreated = "s3.created" ) +// AbsentConfiguration is a recorded, provider-facing error for an unwritten +// setting. It does not grant support for writing that setting. +type AbsentConfiguration struct { + Status int + Code string + Message string + BucketName bool + RequestIDs bool + Resource bool +} + +// BucketCreationTime reads the date persisted by bucket creation. A legacy +// bucket with no date is an error, never assigned a fabricated current time. +func BucketCreationTime(metadata map[string]string) (time.Time, error) { + return time.Parse(time.RFC3339Nano, metadata[bucketCreated]) +} + +type absentConfigurationError struct { + Code string + Message string + BucketName string `xml:",omitempty"` + RequestID string `xml:"RequestId,omitempty"` + HostID string `xml:"HostId,omitempty"` + Resource string `xml:",omitempty"` + XMLName xml.Name `xml:"Error"` +} + // Bucket configuration remains separate from object upload header support. // Only the measured create defaults and versioning checksum are admitted. func (s *Server) bucketHeaders(r *http.Request, query url.Values, bucket bool) (*http.Request, error) { @@ -32,6 +59,14 @@ func (s *Server) bucketHeaders(r *http.Request, query url.Values, bucket bool) ( return r, nil } checked := r.Clone(r.Context()) + if s.cfg.PrivateACLWrite && query.Has("acl") { + values := r.Header.Values("X-Amz-Acl") + if len(values) != 1 || values[0] != "private" { + return nil, errors.New("only the private bucket canned ACL is supported") + } + checked.Header.Del("X-Amz-Acl") + return checked, nil + } if query.Has("versioning") { checked.Header.Del("Content-MD5") return checked, nil @@ -152,7 +187,32 @@ func (s *Server) bucketConfiguration(w http.ResponseWriter, r *http.Request, nam s.storeError(w, r, name, err) return } + if missing, ok := s.cfg.AbsentConfigurations[feature]; ok && r.Method == http.MethodGet { + response := absentConfigurationError{Code: missing.Code, Message: missing.Message} + if missing.BucketName { + response.BucketName = name + } + if missing.RequestIDs { + id := rand.Text() + response.RequestID, response.HostID = id, id + w.Header().Set("X-Amz-Request-Id", id) + w.Header().Set("X-Amz-Id-2", id) + } + if missing.Resource { + response.Resource = "/" + name + } + s.writeXML(w, r, missing.Status, response) + return + } + switch { + case feature == "acl" && r.Method == http.MethodPut && s.cfg.PrivateACLWrite: + body, err := configurationBody(w, r) + if err != nil || len(body) != 0 { + s.refuse(w, r, http.StatusNotImplemented, "only empty-body private bucket ACL writes are implemented", err) + return + } + w.WriteHeader(http.StatusOK) case feature == "versioning" && r.Method == http.MethodPut: body, readErr := configurationBody(w, r) if readErr != nil { @@ -197,6 +257,40 @@ func (s *Server) bucketConfiguration(w http.ResponseWriter, r *http.Request, nam } } +func (s *Server) objectConfiguration(w http.ResponseWriter, r *http.Request, bucket, key string, query url.Values) { + feature := "" + for name, values := range query { + if name == "x-id" { + continue + } + if feature != "" || len(values) != 1 || values[0] != "" || (name != "acl" && name != "tagging") { + s.refuse(w, r, http.StatusBadRequest, "one empty object discovery query is required", nil) + return + } + feature = name + } + if len(r.Header.Values("Range")) != 0 { + s.refuse(w, r, http.StatusNotImplemented, "Range requires object bytes, not object configuration", nil) + return + } + if _, err := s.store.HeadObject(r.Context(), s.name(bucket), key); err != nil { + s.storeError(w, r, bucket, err) + return + } + if feature == "acl" { + owner := s.bucketOwner() + s.writeXML(w, r, http.StatusOK, accessControlPolicy{Owner: owner, Grant: aclGrant{ + Grantee: aclGrantee{Type: "CanonicalUser", XSI: "http://www.w3.org/2001/XMLSchema-instance", + ID: owner.ID, DisplayName: owner.DisplayName}, Permission: "FULL_CONTROL", + }}) + return + } + s.writeXML(w, r, http.StatusOK, struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + TagSet struct{} + }{}) +} + // These paths, namespaces and the canonical private grant follow the official // v2.62.0 create cassette. The owner is a stable local identity, not cloud IAM. type bucketOwner struct { @@ -223,6 +317,9 @@ type accessControlPolicy struct { } func (s *Server) bucketOwner() bucketOwner { + if s.cfg.OwnerID != "" { + return bucketOwner{DisplayName: "feint", ID: s.cfg.OwnerID} + } sum := sha256.Sum256([]byte("feint-local-owner/" + s.cfg.Namespace)) return bucketOwner{DisplayName: "feint", ID: hex.EncodeToString(sum[:])} } @@ -256,7 +353,7 @@ func (s *Server) listBuckets(w http.ResponseWriter, r *http.Request) { } metadata := inventory[name] created := metadata[bucketCreated] - if _, err := time.Parse(time.RFC3339Nano, created); err != nil || !bucketName(wireName) { + if _, err := BucketCreationTime(metadata); err != nil || !bucketName(wireName) { s.refuse(w, r, http.StatusNotImplemented, "legacy bucket listing has no measured creation metadata", err) return } diff --git a/internal/s3/discovery_test.go b/internal/s3/discovery_test.go new file mode 100644 index 00000000..8ffc1e0b --- /dev/null +++ b/internal/s3/discovery_test.go @@ -0,0 +1,156 @@ +package s3 + +import ( + "encoding/xml" + "net/http" + "strings" + "testing" +) + +func TestObjectDiscoveryIsPrivateEmptyAndChecksExistence(t *testing.T) { + s := bucketFixture(t) + s.cfg.ObjectDiscovery = true + for _, path := range []string{"/bucket", "/bucket/key"} { + body := []byte(nil) + if path == "/bucket/key" { + body = []byte("bytes") + } + if w := request(t, s, http.MethodPut, path, body, nil); w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + } + w := request(t, s, http.MethodGet, "/bucket/key?acl=", nil, nil) + var acl accessControlPolicy + if err := xml.Unmarshal(w.Body.Bytes(), &acl); err != nil || w.Code != http.StatusOK || + acl.Owner != s.bucketOwner() || acl.Grant.Permission != "FULL_CONTROL" || + acl.Grant.Grantee.ID != acl.Owner.ID || !strings.Contains(w.Body.String(), `xsi:type="CanonicalUser"`) { + t.Fatal("private object grant diverged:", err, w.Code, w.Body.String()) + } + w = request(t, s, http.MethodGet, "/bucket/key?tagging=", nil, nil) + if w.Code != http.StatusOK || w.Body.String() != `` { + t.Fatal("unwritten object tags did not match the recorded empty tag set:", w.Code, w.Body.String()) + } + for _, feature := range []string{"acl", "tagging"} { + if w := request(t, s, http.MethodGet, "/bucket/missing?"+feature+"=", nil, nil); w.Code != http.StatusNotImplemented { + t.Fatal("missing object acquired successful discovery:", w.Code, w.Body.String()) + } + if w := request(t, s, http.MethodPut, "/bucket/key?"+feature+"=", []byte("unimplemented"), nil); w.Code != http.StatusNotImplemented { + t.Fatal("unsupported object setting write succeeded:", w.Code, w.Body.String()) + } + } + for _, query := range []string{"acl=&tagging=", "acl=&acl=", "tagging=nonempty"} { + if w := request(t, s, http.MethodGet, "/bucket/key?"+query, nil, nil); w.Code != http.StatusBadRequest { + t.Fatal("invalid object query was silently discarded:", w.Code, w.Body.String()) + } + } + if w := request(t, s, http.MethodGet, "/bucket/key?acl=", nil, map[string]string{"Range": "bytes=0-1"}); w.Code != http.StatusNotImplemented { + t.Fatal("object configuration pretended to apply a byte range:", w.Code, w.Body.String()) + } + s.cfg.ObjectDiscovery = false + if w := request(t, s, http.MethodGet, "/bucket/key?acl=", nil, nil); w.Code != http.StatusNotImplemented { + t.Fatal("unmeasured profile acquired object configuration:", w.Code) + } +} + +func TestRecordedAbsentConfigurationChecksBucketAndClonesProfile(t *testing.T) { + base := bucketFixture(t) + config := base.cfg + config.AbsentConfigurations = map[string]AbsentConfiguration{ + "cors": {Status: http.StatusNotFound, Code: "NoSuchCORSConfiguration", + Message: "The CORS configuration does not exist", RequestIDs: true, Resource: true}, + } + s, err := New(base.store, config) + if err != nil { + t.Fatal(err) + } + config.AbsentConfigurations["cors"] = AbsentConfiguration{Status: 400, Code: "changed", Message: "changed"} + if w := request(t, s, http.MethodPut, "/bucket", nil, nil); w.Code != http.StatusOK { + t.Fatal(w.Code, w.Body.String()) + } + w := request(t, s, http.MethodGet, "/bucket?cors=", nil, nil) + var wire absentConfigurationError + if err := xml.Unmarshal(w.Body.Bytes(), &wire); err != nil || w.Code != http.StatusNotFound || + wire.Code != "NoSuchCORSConfiguration" || wire.Message != "The CORS configuration does not exist" || + wire.BucketName != "" || wire.Resource != "/bucket" || wire.RequestID == "" || + wire.RequestID != wire.HostID || w.Header().Get("X-Amz-Request-Id") != wire.RequestID { + t.Fatal("recorded error changed or lost its protocol fields:", err, w.Code, w.Body.String()) + } + w = request(t, s, http.MethodGet, "/missing?cors=", nil, nil) + if w.Code != http.StatusNotFound || !strings.Contains(w.Body.String(), "NoSuchBucket") { + t.Fatal("missing bucket acquired an existing bucket's absent configuration:", w.Code, w.Body.String()) + } + if w := request(t, s, http.MethodPut, "/bucket?cors=", []byte(""), nil); w.Code != http.StatusNotImplemented { + t.Fatal("unimplemented CORS write succeeded:", w.Code, w.Body.String()) + } +} + +func TestPrivateBucketACLWriteRefusesAnythingButTheRecordedDefault(t *testing.T) { + s := bucketFixture(t) + s.cfg.PrivateACLWrite = true + if w := request(t, s, "PUT", "/bucket", nil, nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + if w := request(t, s, "PUT", "/bucket?acl=", nil, map[string]string{"X-Amz-Acl": "private"}); w.Code != 200 { + t.Fatal("recorded private write failed:", w.Code, w.Body.String()) + } + for _, acl := range []string{"", "public-read", "authenticated-read"} { + if w := request(t, s, "PUT", "/bucket?acl=", nil, map[string]string{"X-Amz-Acl": acl}); w.Code != 501 { + t.Fatal("unmeasured canned ACL succeeded:", acl, w.Code, w.Body.String()) + } + } + if w := request(t, s, "PUT", "/bucket?acl=", []byte(""), map[string]string{"X-Amz-Acl": "private"}); w.Code != 501 { + t.Fatal("policy body silently discarded:", w.Code, w.Body.String()) + } +} + +func TestLegacyListingUsesPlainMarkersAndSharedInventory(t *testing.T) { + s, _ := fixture(t) + s.cfg.ListObjectsV1 = true + if w := request(t, s, "PUT", "/bucket", nil, nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + for _, key := range []string{"a", "dir/one", "dir/two", "z"} { + if w := request(t, s, "PUT", "/bucket/"+key, []byte(key), nil); w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + } + list := func(query string) legacyListResult { + t.Helper() + w := request(t, s, "GET", "/bucket?"+query, nil, nil) + var result legacyListResult + if err := xml.Unmarshal(w.Body.Bytes(), &result); err != nil || w.Code != 200 || + strings.Contains(w.Body.String(), "ContinuationToken") || strings.Contains(w.Body.String(), "KeyCount") { + t.Fatal("legacy listing used a V2 wire:", err, w.Code, w.Body.String()) + } + return result + } + first := list("max-keys=1&delimiter=/") + if first.Name != "bucket" || !first.Truncated || first.Next != "a" || len(first.Objects) != 1 || + first.Objects[0].Key != "a" || first.Objects[0].Size != 1 { + t.Fatal("first legacy page did not reflect inventory:", first) + } + second := list("max-keys=1&delimiter=/&marker=a") + if second.Marker != "a" || second.Next != "dir/" || !second.Truncated || + len(second.Objects) != 0 || len(second.Prefixes) != 1 || second.Prefixes[0].Prefix != "dir/" { + t.Fatal("plain marker failed prefix pagination:", second) + } + final := list("max-keys=1&delimiter=/&marker=dir/") + if final.Truncated || final.Next != "" || len(final.Objects) != 1 || final.Objects[0].Key != "z" { + t.Fatal("final legacy page repeated entries:", final) + } + zero := list("max-keys=0") + if zero.Truncated || zero.Next != "" || len(zero.Objects) != 0 { + t.Fatal("zero-size page acquired a nonprogressing marker:", zero) + } + for _, query := range []string{"marker=a&marker=b", "marker=" + strings.Repeat("a", 1025), "max-keys=-1"} { + if w := request(t, s, "GET", "/bucket?"+query, nil, nil); w.Code != 400 { + t.Fatal("invalid legacy option was ignored:", w.Code, w.Body.String()) + } + } + if _, err := parseLegacyListOptions(map[string][]string{"continuation-token": {"v2"}}); err == nil { + t.Fatal("legacy parser accepted a V2 token") + } + if v2 := decodeListing(t, s, "list-type=2"); v2.KeyCount != 4 { + t.Fatal("legacy support changed the V2 inventory:", v2) + } +} diff --git a/internal/s3/doors.go b/internal/s3/doors.go index 04a993c7..6588bc40 100644 --- a/internal/s3/doors.go +++ b/internal/s3/doors.go @@ -29,12 +29,19 @@ const UploadReadTimeout = 30 * time.Second // Profile is a provider's declared wire dialect and client environment names. // Variables stay local to the composition root, never taken from an HTTP report. type Profile struct { - Provider string - Region string - MissingBucket MissingBucketShape - Variables []string - RegionVariable string - BucketConfiguration bool + Provider string + Region string + MissingBucket MissingBucketShape + Variables []string + RegionVariable string + BucketConfiguration bool + AbsentConfigurations map[string]AbsentConfiguration + ObjectDiscovery bool + ListObjectsV1 bool + OwnerID string + PrivateACLWrite bool + // CloudHost is the measured authority used by clients ignoring endpoints. + CloudHost string // HostEndpoint selects the host door for clients that discard path prefixes. HostEndpoint bool } @@ -131,7 +138,9 @@ func mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxO } server, err := New(store, Config{Region: p.Region, MaxObjectBytes: maxObjectBytes, MissingBucket: p.MissingBucket, Namespace: p.Provider + "/" + p.Region, Log: log, - BucketConfiguration: p.BucketConfiguration}) + BucketConfiguration: p.BucketConfiguration, AbsentConfigurations: p.AbsentConfigurations, + ObjectDiscovery: p.ObjectDiscovery, ListObjectsV1: p.ListObjectsV1, OwnerID: p.OwnerID, + PrivateACLWrite: p.PrivateACLWrite}) if err != nil { return nil, err } diff --git a/internal/s3/list.go b/internal/s3/list.go index 209591d1..abee5d4e 100644 --- a/internal/s3/list.go +++ b/internal/s3/list.go @@ -24,6 +24,32 @@ type listOptions struct { max int } +func legacyListQuery(q url.Values) bool { + for key := range q { + switch key { + case "prefix", "delimiter", "max-keys", "marker", "encoding-type", "x-id": + default: + return false + } + } + return true +} + +func parseLegacyListOptions(q url.Values) (listOptions, error) { + if !legacyListQuery(q) { + return listOptions{}, errors.New("legacy listing does not accept ListObjectsV2 parameters") + } + normalized := make(url.Values, len(q)+1) + for key, values := range q { + if key == "marker" { + key = "start-after" + } + normalized[key] = values + } + normalized.Set("list-type", "2") + return parseListOptions(normalized) +} + func listQueryKey(key string) bool { switch key { case "list-type", "prefix", "delimiter", "max-keys", "continuation-token", "start-after", "encoding-type", "fetch-owner", "x-id": @@ -126,6 +152,21 @@ type listResult struct { Prefixes []listedPrefix `xml:"CommonPrefixes"` } +type legacyListResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Namespace string `xml:"xmlns,attr"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + Next string `xml:"NextMarker,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + MaxKeys int `xml:"MaxKeys"` + Truncated bool `xml:"IsTruncated"` + Encoding string `xml:"EncodingType,omitempty"` + Objects []listedObject `xml:"Contents"` + Prefixes []listedPrefix `xml:"CommonPrefixes"` +} + func listText(value, encoding string) (string, error) { if encoding == "url" { return strings.ReplaceAll(url.QueryEscape(value), "+", "%20"), nil @@ -140,6 +181,10 @@ func listText(value, encoding string) (string, error) { func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, bucket string, q url.Values) { options, err := parseListOptions(q) + legacy := s.cfg.ListObjectsV1 && !q.Has("list-type") + if legacy { + options, err = parseLegacyListOptions(q) + } if err != nil { s.refuse(w, r, http.StatusBadRequest, "invalid S3 listing", err) return @@ -211,7 +256,7 @@ func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, bucket stri result.Objects = append(result.Objects, listedObject{Key: text, Modified: obj.Modified.UTC().Format(time.RFC3339Nano), Size: obj.Size, StorageClass: "STANDARD"}) } - if result.Truncated { + if result.Truncated && !legacy { raw, err := json.Marshal(listCursor{Scope: s.listScope(bucket, options), After: names[options.max-1]}) if err != nil { s.refuse(w, r, http.StatusInternalServerError, "S3 listing cursor encoding failed", err) @@ -219,7 +264,21 @@ func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, bucket stri } result.Next = base64.RawURLEncoding.EncodeToString(raw) } - raw, err := xml.Marshal(result) + var wire any = result + if legacy { + next := "" + if result.Truncated { + next, err = listText(names[options.max-1], options.encoding) + if err != nil { + s.refuse(w, r, http.StatusBadRequest, "S3 listing marker cannot be encoded", err) + return + } + } + wire = legacyListResult{Namespace: result.Namespace, Name: result.Name, Prefix: result.Prefix, + Marker: result.After, Next: next, Delimiter: result.Delimiter, MaxKeys: result.MaxKeys, + Truncated: result.Truncated, Encoding: result.Encoding, Objects: result.Objects, Prefixes: result.Prefixes} + } + raw, err := xml.Marshal(wire) if err != nil { s.refuse(w, r, http.StatusInternalServerError, "S3 listing encoding failed", err) return diff --git a/internal/s3/local_transport.go b/internal/s3/local_transport.go new file mode 100644 index 00000000..01207f05 --- /dev/null +++ b/internal/s3/local_transport.go @@ -0,0 +1,100 @@ +package s3 + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" +) + +// LocalTransport maps declared cloud S3 authorities exclusively to one loopback +// emulator. It preserves virtual-host buckets, including dotted names, and +// literal escaped object keys. Unknown authorities never reach a dial. +// TLS interception and process network isolation belong to the caller. +type LocalTransport struct { + target url.URL + profiles map[string]Profile + transport http.Transport +} + +// NewLocalTransport requires an explicit loopback HTTP target without a path. +// It never inherits environment proxies or a cloud-capable fallback transport. +func NewLocalTransport(target string, profiles []Profile) (*LocalTransport, error) { + base, err := url.Parse(target) + if err != nil { + return nil, fmt.Errorf("parse local S3 target: %w", err) + } + ip := net.ParseIP(base.Hostname()) + if base.Scheme != "http" || ip == nil || !ip.IsLoopback() || base.Port() == "" || + base.User != nil || (base.Path != "" && base.Path != "/") || base.RawQuery != "" || base.Fragment != "" { + return nil, errors.New("s3: local transport requires an explicit loopback HTTP socket") + } + t := &LocalTransport{target: *base, profiles: make(map[string]Profile)} + for _, profile := range profiles { + if !ValidToken(profile.Provider) || !ValidToken(profile.Region) || profile.CloudHost == "" || + strings.ContainsAny(profile.CloudHost, "/:@* \t\r\n") || profile.CloudHost != strings.ToLower(profile.CloudHost) { + return nil, errors.New("s3: local transport requires declared cloud authorities and scopes") + } + for host := range t.profiles { + if host == profile.CloudHost || strings.HasSuffix(host, "."+profile.CloudHost) || + strings.HasSuffix(profile.CloudHost, "."+host) { + return nil, errors.New("s3: overlapping cloud authority declarations") + } + } + t.profiles[profile.CloudHost] = profile + } + if len(t.profiles) == 0 { + return nil, errors.New("s3: local transport requires at least one declared cloud authority") + } + return t, nil +} + +// RoundTrip refuses undeclared authorities before sending any request. +func (t *LocalTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.URL == nil || r.URL.Scheme != "https" || r.URL.User != nil || + (r.URL.Port() != "" && r.URL.Port() != "443") || r.URL.Fragment != "" { + return nil, errors.New("s3: only declared HTTPS cloud requests may be redirected") + } + host := strings.ToLower(r.URL.Hostname()) + var profile Profile + bucket := "" + found := false + for base, candidate := range t.profiles { + if host == base { + profile, found = candidate, true + break + } + if prefix, matches := strings.CutSuffix(host, "."+base); matches { + if !bucketName(prefix) { + return nil, errors.New("s3: invalid virtual-host bucket") + } + profile, bucket, found = candidate, prefix, true + break + } + } + if !found { + return nil, fmt.Errorf("s3: undeclared cloud authority %q; no request forwarded", host) + } + path := r.URL.EscapedPath() + if path == "" { + path = "/" + } + if bucket != "" { + path = "/" + url.PathEscape(bucket) + path + } + path = strings.TrimSuffix(DoorPath(profile.Provider, profile.Region), "/") + path + decoded, err := url.PathUnescape(path) + if err != nil { + return nil, fmt.Errorf("decode mapped S3 path: %w", err) + } + out := r.Clone(r.Context()) + out.URL.Scheme, out.URL.Host = t.target.Scheme, t.target.Host + out.URL.Path, out.URL.RawPath = decoded, path + out.Host, out.RequestURI = t.target.Host, "" + return t.transport.RoundTrip(out) +} + +// CloseIdleConnections releases only this transport's loopback connections. +func (t *LocalTransport) CloseIdleConnections() { t.transport.CloseIdleConnections() } diff --git a/internal/s3/local_transport_test.go b/internal/s3/local_transport_test.go new file mode 100644 index 00000000..42bfee6a --- /dev/null +++ b/internal/s3/local_transport_test.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" +) + +func TestLocalTransportPreservesCloudBucketAndLiteralKeyWithoutFallback(t *testing.T) { + var requests atomic.Int64 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.EscapedPath() != "/_feint/s3/example/region-1/a.b.bucket/folder//a/../percent%252F" || + r.URL.RawQuery != "tagging=&x-id=GetObjectTagging" { + t.Errorf("mapped request lost bucket or literal key: %s", r.URL.String()) + } + w.WriteHeader(http.StatusNoContent) + })) + defer target.Close() + profile := Profile{Provider: "example", Region: "region-1", CloudHost: "s3.example.invalid"} + transport, err := NewLocalTransport(target.URL, []Profile{profile}) + if err != nil { + t.Fatal(err) + } + defer transport.CloseIdleConnections() + for _, authority := range []string{"a.b.bucket.s3.example.invalid", "s3.example.invalid/a.b.bucket"} { + r, err := http.NewRequestWithContext(t.Context(), "GET", + "https://"+authority+"/folder//a/../percent%252F?tagging=&x-id=GetObjectTagging", nil) + if err != nil { + t.Fatal(err) + } + original := r.URL.String() + response, err := transport.RoundTrip(r) + if err != nil { + t.Fatal(err) + } + if err := response.Body.Close(); err != nil || response.StatusCode != http.StatusNoContent || r.URL.String() != original { + t.Fatal("successful mapping mutated the caller or failed:", err, response.StatusCode) + } + } + before := requests.Load() + for _, source := range []string{ + "https://other.invalid/key", "https://s3.example.invalid.evil.invalid/key", + "https://bad_bucket.s3.example.invalid/key", "http://s3.example.invalid/key", + "https://user@s3.example.invalid/key", "https://s3.example.invalid:444/key", + } { + r, err := http.NewRequestWithContext(t.Context(), "GET", source, nil) + if err != nil { + t.Fatal(err) + } + if response, err := transport.RoundTrip(r); err == nil { + if err := response.Body.Close(); err != nil { + t.Fatal(err) + } + t.Fatal("undeclared request reached the local transport:", source) + } + } + if requests.Load() != before || before != 2 { + t.Fatal("refusal dialled despite unknown authority or accepting controls failed:", requests.Load(), before) + } +} + +func TestLocalTransportRejectsUnsafeTargetsAndOverlappingDeclarations(t *testing.T) { + profile := Profile{Provider: "example", Region: "region-1", CloudHost: "s3.example.invalid"} + for _, target := range []string{ + "http://192.0.2.1:1234", "http://localhost:1234", "https://127.0.0.1:1234", + "http://127.0.0.1", "http://user@127.0.0.1:1234", "http://127.0.0.1:1234/path", + "http://127.0.0.1:1234?query", "http://127.0.0.1:1234#fragment", + } { + if _, err := NewLocalTransport(target, []Profile{profile}); err == nil { + t.Fatal("unsafe target admitted:", target) + } + } + for _, profiles := range [][]Profile{ + nil, {profile, profile}, + {profile, {Provider: "other", Region: "region-1", CloudHost: "bucket.s3.example.invalid"}}, + {{Provider: "example", Region: "region-1", CloudHost: ""}}, + } { + if _, err := NewLocalTransport("http://127.0.0.1:1234", profiles); err == nil { + t.Fatal("ambiguous or empty authority declarations admitted:", profiles) + } + } + valid, err := NewLocalTransport("http://[::1]:1234", []Profile{profile}) + if err != nil { + t.Fatal("IPv6 loopback socket refused:", err) + } + valid.CloseIdleConnections() + if response, err := valid.RoundTrip(&http.Request{URL: &url.URL{Scheme: "https", Host: "s3.example.invalid", Fragment: "discarded"}}); err == nil { + if err := response.Body.Close(); err != nil { + t.Fatal(err) + } + t.Fatal("fragment was silently discarded") + } +} diff --git a/internal/s3/server.go b/internal/s3/server.go index 54d86fe2..2fe1e4b5 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -14,6 +14,7 @@ import ( "fmt" "io" "log/slog" + "maps" "math" "net" "net/http" @@ -52,6 +53,17 @@ type Config struct { // BucketConfiguration enables the measured private, non-versioned bucket // workflow, independently of the missing-bucket error dialect. BucketConfiguration bool + // AbsentConfigurations declares measured errors for unwritten bucket settings. + // These settings have no write support and cannot silently become configured. + AbsentConfigurations map[string]AbsentConfiguration + // ObjectDiscovery admits measured private ACL and empty-tag reads only. + ObjectDiscovery bool + // ListObjectsV1 admits the recorded legacy listing alongside ListObjectsV2. + ListObjectsV1 bool + // OwnerID is the declared local tenant's canonical ACL identity. + OwnerID string + // PrivateACLWrite admits the recorded empty-body private bucket ACL write. + PrivateACLWrite bool // Set only by the explicit host-door composition, not by direct callers. endpointHost string } @@ -75,10 +87,18 @@ func New(store *objectstore.Store, cfg Config) (*Server, error) { if cfg.MissingBucket != ScalewayMissingBucket && cfg.MissingBucket != ExoscaleMissingBucket { return nil, errors.New("s3: select a measured missing-bucket response shape") } + for feature, response := range cfg.AbsentConfigurations { + if (feature != "cors" && feature != "ownershipControls" && feature != "object-lock" && feature != "lifecycle") || + response.Status < 400 || response.Status > 499 || + response.Code == "" || !headerValue(response.Code) || !headerValue(response.Message) { + return nil, errors.New("s3: invalid measured absent bucket configuration") + } + } + cfg.AbsentConfigurations = maps.Clone(cfg.AbsentConfigurations) return &Server{store: store, cfg: cfg}, nil } -// ServeHTTP implements bucket PUT/HEAD/DELETE, ListObjectsV2, DeleteObjects and object +// ServeHTTP implements bucket PUT/HEAD/DELETE, declared object listings, DeleteObjects and object // PUT/GET/HEAD/DELETE, including single-byte-range GETs. // Other features get an explicit Feint refusal, never a successful empty stub // or an invented cloud XML error. Signatures and credentials are never logged. @@ -108,20 +128,22 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } rawBucket, rawKey, _ := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/") - listing := r.Method == http.MethodGet && rawKey == "" && query.Get("list-type") == "2" + listing := r.Method == http.MethodGet && rawKey == "" && + (query.Get("list-type") == "2" || (s.cfg.ListObjectsV1 && legacyListQuery(query))) bulkDelete := r.Method == http.MethodPost && rawKey == "" && query.Has("delete") multipart := query.Has("uploads") || query.Has("uploadId") for key := range query { if multipart && multipartQueryKey(key) { continue } - if listing && listQueryKey(key) { + if listing && (listQueryKey(key) || key == "marker") { continue } if bulkDelete && key == "delete" { continue } - if key != "x-id" && (!s.cfg.BucketConfiguration || (key != "versioning" && key != "acl" && key != "tagging")) { + _, absent := s.cfg.AbsentConfigurations[key] + if key != "x-id" && (!s.cfg.BucketConfiguration || (key != "versioning" && key != "acl" && key != "tagging" && !absent)) { s.refuse(w, r, http.StatusNotImplemented, "S3 query features are not implemented", nil) return } @@ -175,6 +197,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } if query.Has("versioning") || query.Has("acl") || query.Has("tagging") { + if s.cfg.ObjectDiscovery && r.Method == http.MethodGet && !query.Has("versioning") { + s.objectConfiguration(w, r, bucket, key, query) + return + } s.refuse(w, r, http.StatusNotImplemented, "bucket configuration queries require a bucket, not an object", nil) return } diff --git a/mise.toml b/mise.toml index 137e9497..b72e423a 100644 --- a/mise.toml +++ b/mise.toml @@ -55,7 +55,7 @@ description = "Prove the unmounted path-style S3 component with AWS CLI 2.32.17 run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:mounted"] -description = "Prove AWS/scw/exo bytes, multipart resume/abort, listing/ranges and bulk deletion across actual restart" +description = "Prove AWS/scw/exo bytes, normal Exo bucket/native usage, multipart and cleanup across actual restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" [tasks."conformance:s3:exo:endpoint"] @@ -633,9 +633,38 @@ tools/conformance/guard.sh leftovers "${FEINT_VM:-off}" # it filters by `name=default` and is unchanged by the second name; what the # second one buys is the case #572 is about, which no unit test can reach: `scw` # and the Terraform provider resolving a project whose name is not `default`. +# +# --objectstore, and it is the probe that needs it rather than any client suite +# here. `exoscale/v2.list-sos-buckets-usage` reads the byte inventory and answers +# 503 "Object Storage is disabled" without one, so the probe — which builds its +# request from the operation's own schema, and that schema declares a 200 body — +# counted it as a route answering the wrong shape. It is the whole reason this +# task went red the first time the operation was mounted. +# +# Three alternatives were measured and rejected: +# - declaring the probe axis Unearnable: a lie. The axis CAN be earned; it +# needs a directory. TestAnUnearnableAxisIsNotAlreadyEarned would bite. +# - declaring the 503 in the contract: a worse lie. Exoscale's API documents no +# such answer; the 503 is this emulator's own refusal, and putting it in the +# provider's description would make the contract describe an API nobody has. +# - teaching the probe to skip the route: it would sub-declare the axis in +# coverage/evidence.json for an operation that is in fact served and checkable. +# +# A private directory under the run's own temporary root, so this costs what a +# mktemp costs and leaves nothing behind. 0700 because the store refuses a root +# whose writable parent carries no sticky bit, and it is right to: a store under +# a world-writable parent can be swapped underneath it. +# +# Checked before widening the task: nothing under tools/conformance/ asserts the +# absence of object storage. The 503s that grep finds there are fault injections +# on ListServers, ReadNets and list-instances, unrelated to this. +conformance_objects="$(mktemp -d)" +chmod 0700 "$conformance_objects" +trap 'rm -rf "$conformance_objects"' EXIT ./feint start --addr $FEINT_ADDR --vm ${FEINT_VM:-off} --contracts contracts \ - --projects default,platform-prod --timeout 60s -trap './feint stop --addr $FEINT_ADDR >/dev/null 2>&1 || true' EXIT + --projects default,platform-prod --timeout 60s \ + --objectstore "$conformance_objects/objects" +trap './feint stop --addr $FEINT_ADDR >/dev/null 2>&1 || true; rm -rf "$conformance_objects"' EXIT tools/conformance/scaleway/scw-cli.sh "http://$FEINT_ADDR" tools/conformance/scaleway/terraform.sh "http://$FEINT_ADDR" tools/conformance/outscale/octl.sh "http://$FEINT_ADDR" diff --git a/tools/conformance/exoscale/exo-cli.sh b/tools/conformance/exoscale/exo-cli.sh index 37738c0d..ce3697cc 100755 --- a/tools/conformance/exoscale/exo-cli.sh +++ b/tools/conformance/exoscale/exo-cli.sh @@ -769,6 +769,34 @@ exoc -Q compute anti-affinity-group delete conformance-aag --force >/dev/null \ prove_end "$span" ok "security group and anti-affinity group deleted" +# The native SOS inventory, which is the one route of this pack no client drove. +# +# `exo storage list` calls ListSOSBucketsUsage — exoscale/cli +# cmd/storage/storage_list.go, read 2026-11-09 — so the route has a client and +# does not belong in Route.Undriven. Before this assertion the operation sat at +# driven:false in coverage/evidence.json and +# TestEveryUndrivenOperationSaysWhy demanded either a reason or a client. The +# reason would have been false, and "not wired up yet" is not a reason to +# decline (#859 states that in as many words). +# +# It needs no new endpoint: /v2/sos-buckets-usage is on the v2 API that +# EXOSCALE_API_ENDPOINT above already redirects. What it does need is the byte +# inventory, so `mise run conformance` starts the emulator with --objectstore; +# without it the route answers 503 and the probe counted that as a wrong shape. +# +# An empty list is the whole assertion, and deliberately so. This emulator has +# no bucket at this point, and the honest answer to "what do my buckets weigh" +# is then an empty inventory rather than a catalogue. #858 asks for values +# derived from real buckets, not a fictional one, and a zero-length answer over +# zero buckets is that derivation at its smallest. +echo "- the native SOS inventory answers for the buckets that exist" +usage="$(exoc -O json storage list)" || fail "storage list rejected: $usage" +printf '%s' "$usage" | jq -e 'type == "array"' >/dev/null \ + || fail "the SOS inventory is not a list: $usage" +printf '%s' "$usage" | jq -e 'length == 0' >/dev/null \ + || fail "the SOS inventory invented $(printf '%s' "$usage" | jq 'length') bucket(s) where none exist: $usage" +ok "an empty inventory over zero buckets, derived rather than catalogued" + # Every answer above was also checked against Exoscale's own API description, # because the emulator validates itself when --contracts is set. A field they do # not declare fails the request rather than reaching the client. diff --git a/tools/falsify/specs/s3-bucket-configuration.json b/tools/falsify/specs/s3-bucket-configuration.json index 344ea884..5c06fa7e 100644 --- a/tools/falsify/specs/s3-bucket-configuration.json +++ b/tools/falsify/specs/s3-bucket-configuration.json @@ -130,8 +130,8 @@ { "label": "legacy buckets receive an invented listing date", "file": "internal/s3/bucket_configuration.go", - "find": "if _, err := time.Parse(time.RFC3339Nano, created); err != nil || !bucketName(wireName) {", - "replace": "if _, err := time.Parse(time.RFC3339Nano, created); (err != nil || !bucketName(wireName)) && false {", + "find": "if _, err := BucketCreationTime(metadata); err != nil || !bucketName(wireName) {", + "replace": "if _, err := BucketCreationTime(metadata); (err != nil || !bucketName(wireName)) && false {", "test": "TestRestoredBucketConfigurationIsRevalidated" }, { diff --git a/tools/falsify/specs/s3-gateway.json b/tools/falsify/specs/s3-gateway.json index a279de2e..b60157db 100644 --- a/tools/falsify/specs/s3-gateway.json +++ b/tools/falsify/specs/s3-gateway.json @@ -53,8 +53,8 @@ { "label": "S3 query features silently become ordinary CRUD", "file": "internal/s3/server.go", - "find": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\")) {", - "replace": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\")) && false {", + "find": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\" && !absent)) {", + "replace": "if key != \"x-id\" && (!s.cfg.BucketConfiguration || (key != \"versioning\" && key != \"acl\" && key != \"tagging\" && !absent)) && false {", "test": "TestRefusalsDoNotMasqueradeAsCloudSupport" }, { diff --git a/tools/falsify/specs/s3-storage-discovery.json b/tools/falsify/specs/s3-storage-discovery.json new file mode 100644 index 00000000..c0878356 --- /dev/null +++ b/tools/falsify/specs/s3-storage-discovery.json @@ -0,0 +1,114 @@ +{ + "package": "./internal/s3", + "mutations": [ + { + "label": "recorded absence responses alias caller configuration", + "file": "internal/s3/server.go", + "find": "cfg.AbsentConfigurations = maps.Clone(cfg.AbsentConfigurations)", + "replace": "cfg.AbsentConfigurations = func() map[string]AbsentConfiguration { _ = maps.Clone(cfg.AbsentConfigurations); return cfg.AbsentConfigurations }()", + "test": "TestRecordedAbsentConfigurationChecksBucketAndClonesProfile" + }, + { + "label": "private ACL writes accept public permissions", + "file": "internal/s3/bucket_configuration.go", + "find": "if len(values) != 1 || values[0] != \"private\" {", + "replace": "if (len(values) != 1 || values[0] != \"private\") && false {", + "test": "TestPrivateBucketACLWriteRefusesAnythingButTheRecordedDefault" + }, + { + "label": "private ACL writes discard a policy body", + "file": "internal/s3/bucket_configuration.go", + "find": "if err != nil || len(body) != 0 {", + "replace": "if err != nil || (len(body) != 0 && false) {", + "test": "TestPrivateBucketACLWriteRefusesAnythingButTheRecordedDefault" + }, + { + "label": "object discovery discards duplicate or nonempty query features", + "file": "internal/s3/bucket_configuration.go", + "find": "if feature != \"\" || len(values) != 1 || values[0] != \"\" || (name != \"acl\" && name != \"tagging\") {", + "replace": "if (feature != \"\" || len(values) != 1 || values[0] != \"\" || (name != \"acl\" && name != \"tagging\")) && false {", + "test": "TestObjectDiscoveryIsPrivateEmptyAndChecksExistence" + }, + { + "label": "object configuration silently ignores a Range", + "file": "internal/s3/bucket_configuration.go", + "find": "if len(r.Header.Values(\"Range\")) != 0 {", + "replace": "if len(r.Header.Values(\"Range\")) != 0 && false {", + "test": "TestObjectDiscoveryIsPrivateEmptyAndChecksExistence" + }, + { + "label": "missing objects receive successful ACL and tag discovery", + "file": "internal/s3/bucket_configuration.go", + "find": "if _, err := s.store.HeadObject(r.Context(), s.name(bucket), key); err != nil {", + "replace": "if _, err := s.store.HeadObject(r.Context(), s.name(bucket), key); err != nil && false {", + "test": "TestObjectDiscoveryIsPrivateEmptyAndChecksExistence" + }, + { + "label": "legacy listings accept V2 cursor parameters", + "file": "internal/s3/list.go", + "find": "if !legacyListQuery(q) {", + "replace": "if !legacyListQuery(q) && false {", + "test": "TestLegacyListingUsesPlainMarkersAndSharedInventory" + }, + { + "label": "redirected cloud requests allow cleartext or alternate ports", + "file": "internal/s3/local_transport.go", + "find": "if r.URL == nil || r.URL.Scheme != \"https\" || r.URL.User != nil ||\n\t\t(r.URL.Port() != \"\" && r.URL.Port() != \"443\") || r.URL.Fragment != \"\" {", + "replace": "if (r.URL == nil || r.URL.Scheme != \"https\" || r.URL.User != nil ||\n\t\t(r.URL.Port() != \"\" && r.URL.Port() != \"443\") || r.URL.Fragment != \"\") && false {", + "test": "TestLocalTransportPreservesCloudBucketAndLiteralKeyWithoutFallback" + }, + { + "label": "undeclared cloud authorities acquire a forwarding target", + "file": "internal/s3/local_transport.go", + "find": "if !found {", + "replace": "if !found && false {", + "test": "TestLocalTransportPreservesCloudBucketAndLiteralKeyWithoutFallback" + }, + { + "label": "virtual hosts accept invalid bucket syntax", + "file": "internal/s3/local_transport.go", + "find": "if !bucketName(prefix) {", + "replace": "if !bucketName(prefix) && false {", + "test": "TestLocalTransportPreservesCloudBucketAndLiteralKeyWithoutFallback" + }, + { + "label": "native usage substitutes success for disabled storage", + "package": "./internal/providers/exoscale", + "file": "internal/providers/exoscale/object_storage.go", + "find": "writeError(w, http.StatusServiceUnavailable, \"Object Storage is disabled; start with --objectstore\")", + "replace": "writeError(w, http.StatusOK, \"Object Storage is disabled; start with --objectstore\")", + "test": "TestSOSUsageRefusesDisabledStorageAndLegacyDates" + }, + { + "label": "native usage serves malformed scopes and invented legacy dates", + "package": "./internal/providers/exoscale", + "file": "internal/providers/exoscale/object_storage.go", + "find": "if !valid || !publishedZone(zone) || name == \"\" || strings.Contains(name, \"/\") || dateErr != nil {", + "replace": "if (!valid || !publishedZone(zone) || name == \"\" || strings.Contains(name, \"/\") || dateErr != nil) && false {", + "test": "TestSOSUsageRefusesDisabledStorageAndLegacyDates|TestSOSUsageReportsOnlyPublishedScopedBytesAndRefusesMalformedScopes" + }, + { + "label": "native usage includes another provider's byte inventory", + "package": "./internal/providers/exoscale", + "file": "internal/providers/exoscale/object_storage.go", + "find": "if !ours {", + "replace": "if !ours && false {", + "test": "TestSOSUsageReportsOnlyPublishedScopedBytesAndRefusesMalformedScopes" + }, + { + "label": "any target is admitted, so a transport built on 192.0.2.1, on a name rather than a literal, on https, on a port-less host, or carrying credentials, a path, a query or a fragment is accepted and the redirection can leave the loopback", + "file": "internal/s3/local_transport.go", + "find": "\tif base.Scheme != \"http\" || ip == nil || !ip.IsLoopback() || base.Port() == \"\" ||\n\t\tbase.User != nil || (base.Path != \"\" && base.Path != \"/\") || base.RawQuery != \"\" || base.Fragment != \"\" {", + "replace": "\tif (base.Scheme != \"http\" || ip == nil || !ip.IsLoopback() || base.Port() == \"\" ||\n\t\tbase.User != nil || (base.Path != \"\" && base.Path != \"/\") || base.RawQuery != \"\" || base.Fragment != \"\") && false {", + "test": "TestLocalTransportRejectsUnsafeTargetsAndOverlappingDeclarations" + }, + { + "label": "overlapping cloud authorities are admitted, so one provider's declaration can shadow another's bucket namespace and the map decides by iteration order which door a request reaches", + "file": "internal/s3/local_transport.go", + "find": "\t\t\tif host == profile.CloudHost || strings.HasSuffix(host, \".\"+profile.CloudHost) ||\n\t\t\t\tstrings.HasSuffix(profile.CloudHost, \".\"+host) {", + "replace": "\t\t\tif (host == profile.CloudHost || strings.HasSuffix(host, \".\"+profile.CloudHost) ||\n\t\t\t\tstrings.HasSuffix(profile.CloudHost, \".\"+host)) && false {", + "test": "TestLocalTransportRejectsUnsafeTargetsAndOverlappingDeclarations" + } + ], + "note": "The subject is the shared S3 discovery slice of #850, and the two last mutations are the ones this spec was missing. The three that precede them all drive the ACCEPTING path of LocalTransport (TestLocalTransportPreservesCloudBucketAndLiteralKeyWithoutFallback): that a declared authority keeps its dotted virtual-host bucket and its literally escaped key, and that an undeclared one is refused before any dial. None touched the two refusals in NewLocalTransport itself, which is where the blast radius lives: a transport that accepts a non-loopback target forwards cloud-shaped requests off the machine, and one that accepts overlapping authorities lets map iteration order decide which provider's door a bucket reaches. Both are conservative mutations, the whole condition kept and suffixed with `&& false`, because the harness refuses a mutation that drops a name and a dropped name fails the package for the wrong reason. The test they bite already existed and already covered eight unsafe targets, four ambiguous declarations and the accepting IPv6 loopback case." +} From 39fbf522d9e05e85fd1e2ee967435aa4b6d3ffb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 16:17:31 +0100 Subject: [PATCH 13/28] feat(outscale): an OOS door, declared on five measurements and nothing else MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #850 asks for an OUTSCALE or OOS client path "if the current official tooling exposes it", and the honest answer was assumed to be no. It is yes, and every part of that was measured rather than read: octl v0.0.32 carries `octl storage`, "OUTSCALE Object Storage (OOS) management", with bucket, object, multipartupload and part subcommands `octl storage api` lists S3's OWN operation names — GetObject, PutObject, ListObjectsV2, CreateMultipartUpload — so OOS is S3, not a named API osc-sdk-go pkg/oos/client.go is built on aws-sdk-go-v2/service/s3 with `type Error = smithy.APIError`, so there is no proprietary error dialect pkg/profile/profile.go:114 reads OSC_ENDPOINT_OOS and endpoint.go:12 accepts an `oos` profile key, so the endpoint redirects with NO TLS interception, no fork and no dev_overrides pkg/profile/endpoint.go:27 gives the authority: %s://oos.%s.outscale.com That last one matters for #850's other half: a provider that cannot be safely intercepted gets its door declined rather than simulated. This one can, the same way octl.sh already redirects this client's IaaS calls. ## The envelope, which is the part nothing but a recording settles OutscaleMissingBucket is a third shape, not a reuse of either neighbour: PRESENT and EMPTY Scaleway fills it with /, Exoscale omits the element entirely ABSENT both others carry it x-amz-id-2 == x-amz-request-id the others derive them apart no X-Amzn-Request-Id unlike Exoscale final period on the message like Exoscale, unlike Scaleway Recorded by an ANONYMOUS GET on a name that does not exist. No credentials, so no signature, so the request is attributed to no account: nothing created, nothing modified, nothing billed. The user offered their Outscale account for this and it was deliberately not used, because the two existing dialects were recorded the same way and the cheaper measurement is the one that settles it. Kept at docs/evidence/outscale-oos-missing-bucket.json with its provenance and what each difference means. `server: nginx/1.20.0` on that answer is itself worth knowing: OOS is Outscale's own implementation, so the envelope is theirs to differ in. Resource and BucketName became *string, because three states had to be told apart and `omitempty` on a string expresses two: nil omits the element, a pointer to "" renders it empty, a pointer to a value fills it. The NoSuchTagSet answer in bucket_configuration.go is addressed rather than changed. ## What is NOT claimed The door, its endpoint variables and its missing-bucket envelope. NOT that octl's storage surface round-trips: no suite drives `octl storage` yet, so every operation of this door is unproven by a real client whatever the S3 server under it can serve. The conformance axis says nothing about it, which is correct. It is therefore NOT in the default --s3-doors, which stays scaleway/fr-par,exoscale/ch-gva-2. A door enabled by default is a door the project vouches for. BucketConfiguration, ObjectDiscovery, ListObjectsV1 and PrivateACLWrite stay at their zero value rather than copied from the Scaleway profile next door. Each switches on an answer shape, and this door has one recording to its name. #850 is not closed and neither is anything else. ## Verification mise run check / docs:check / shapes:check pass mise run falsify:lint / lint-shell / release:surface pass mise run falsify -- oos-missing-bucket-envelope.json 4/4 compiled and bit The door answers, measured on a running emulator with --s3-doors ...,outscale/eu-west-2: HTTP/1.1 404 NoSuchBucket The specified bucket does not exist. ... declaration prefix present, Resource present-and-empty, no BucketName, final period — all four checked separately Two tests the harness forced into existence rather than letting me assume them. TestUploadHeaderValidationAndConstructor gains a shape past the last measured one, because mutation 4 widened the constructor's allow-list into a range check and nothing went red: the case did not exist. And the first spec aimed that mutation at a test name that does not exist, where `go test -run` matches nothing and exits 0 — a mutation pointed at a missing test is green forever, which is the quietest way a falsification can lie. TestS3DeclarationsRejectUnknownMalformedAndDuplicateDoors listed "outscale/eu-west-2" among the declarations to refuse, as the provider without a door. It has one now, so that entry went and three Outscale-specific malformed cases replaced it — plus an assertion that the door is ACCEPTED and declares the measured authority, because a guard that only refuses passes every refusal test and breaks the product. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evidence/outscale-oos-missing-bucket.json | 32 +++++ internal/cli/object_storage_test.go | 21 +++- internal/providers/outscale/object_storage.go | 61 ++++++++++ internal/s3/bucket_configuration.go | 9 +- internal/s3/bucket_configuration_test.go | 3 +- internal/s3/server.go | 72 +++++++++-- internal/s3/server_test.go | 112 ++++++++++++++++-- .../s3/testdata/outscale-no-such-bucket.xml | 1 + .../specs/oos-missing-bucket-envelope.json | 34 ++++++ 9 files changed, 317 insertions(+), 28 deletions(-) create mode 100644 docs/evidence/outscale-oos-missing-bucket.json create mode 100644 internal/providers/outscale/object_storage.go create mode 100644 internal/s3/testdata/outscale-no-such-bucket.xml create mode 100644 tools/falsify/specs/oos-missing-bucket-envelope.json diff --git a/docs/evidence/outscale-oos-missing-bucket.json b/docs/evidence/outscale-oos-missing-bucket.json new file mode 100644 index 00000000..597201ae --- /dev/null +++ b/docs/evidence/outscale-oos-missing-bucket.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "provider": "outscale", + "region": "eu-west-2", + "observed_at": "2026-11-09", + "client": "curl, no client SDK involved", + "provenance": "ANONYMOUS GET on https://oos.eu-west-2.outscale.com/. No credentials were sent, so the request carries no AWS Signature v4 and is attributed to no account: nothing was created, nothing was modified, nothing was billed, and the user's Outscale account was offered and deliberately not used. This is the same method that produced ScalewayMissingBucket (anonymous fr-par GET) and ExoscaleMissingBucket (anonymous ch-gva-2 GET), and it is the cheapest measurement that can settle a wire shape: the 404 body IS the subject. One request, read-only.", + "redaction": "The request identifier of the recorded exchange is kept because it is the shape that matters here: x-amz-id-2 and x-amz-request-id carried the SAME value, which neither of the two existing dialects does. It identifies no account and names no resource. The bucket name was a timestamped nonce and is replaced below.", + "why_it_cannot_be_guessed": [ + " is PRESENT and EMPTY — Scaleway fills it with /, Exoscale omits it entirely.", + " is ABSENT — both other dialects carry it.", + "x-amz-id-2 equals x-amz-request-id — Scaleway and Exoscale derive them separately.", + "No X-Amzn-Request-Id header, unlike Exoscale.", + "The message carries a final period, like Exoscale and unlike Scaleway." + ], + "server_header": "nginx/1.20.0, so this is Outscale's own implementation rather than AWS S3", + "exchanges": [ + { + "method": "GET", + "path": "/", + "query": "", + "status": 404, + "headers": { + "content-type": "application/xml", + "x-amz-request-id": "4a23766cf345890fcec2", + "x-amz-id-2": "4a23766cf345890fcec2", + "x-content-type-options": "nosniff" + }, + "body": "NoSuchBucketThe specified bucket does not exist.4a23766cf345890fcec2" + } + ] +} diff --git a/internal/cli/object_storage_test.go b/internal/cli/object_storage_test.go index 70ec15f6..b67e9ce5 100644 --- a/internal/cli/object_storage_test.go +++ b/internal/cli/object_storage_test.go @@ -117,12 +117,31 @@ func TestS3DeclarationsRejectUnknownMalformedAndDuplicateDoors(t *testing.T) { if err != nil { t.Fatal(err) } - for _, value := range []string{"", "scaleway", "scaleway/fr-par,scaleway/fr-par", "outscale/eu-west-2", + // "outscale/eu-west-2" used to be in this list, as the provider without a + // door. It has one since the OOS envelope was measured, so keeping it here + // would assert an absence that is no longer true — and a test that encodes + // a gap keeps failing for the right reason only until the gap closes. + // + // Every other entry is a malformed or unknown declaration and stays: the + // empty string, a provider with no region, a duplicate, an unknown + // provider, an upper-cased region, a third path segment, and a traversal + // attempt. The duplicate case now also matters for Outscale, which is why + // one of its own is below. + for _, value := range []string{"", "scaleway", "scaleway/fr-par,scaleway/fr-par", + "outscale/eu-west-2,outscale/eu-west-2", "outscale/EU-WEST-2", "outscale/../eu-west-2", "unknown/fr-par", "scaleway/FR-PAR", "scaleway/fr-par/extra", "scaleway/../fr-par"} { if _, err := storageProfiles(srv.Packs(), value); err == nil { t.Fatalf("invalid or unsupported declaration accepted: %q", value) } } + // And the door itself is accepted, which is the half this test would + // otherwise not cover: a guard that only ever refuses passes every refusal + // case and breaks the product. + oos, err := storageProfiles(srv.Packs(), "outscale/eu-west-2") + if err != nil || len(oos) != 1 || oos[0].Provider != "outscale" || + oos[0].CloudHost != "oos.eu-west-2.outscale.com" { + t.Fatalf("the measured OOS door was refused or mis-declared: %+v %v", oos, err) + } profiles, err := storageProfiles(srv.Packs(), defaultS3Doors) if err != nil || len(profiles) != 2 { t.Fatalf("approved explicit doors rejected: %+v %v", profiles, err) diff --git a/internal/providers/outscale/object_storage.go b/internal/providers/outscale/object_storage.go new file mode 100644 index 00000000..fcd8c192 --- /dev/null +++ b/internal/providers/outscale/object_storage.go @@ -0,0 +1,61 @@ +package outscale + +import ( + "github.com/stephrobert/feint/internal/s3" +) + +// ObjectStorage declares the OOS door, and declares only what was measured. +// +// ## Why this door exists at all +// +// #850 asks for an OUTSCALE or OOS client path "if the current official +// tooling exposes it", and the honest answer was assumed to be no for longer +// than it should have been. Measured 2026-11-09, five readings, none of them +// from documentation: +// +// octl v0.0.32 carries `octl storage` ("OUTSCALE Object Storage (OOS) +// management") with bucket, object, multipartupload and part subcommands +// `octl storage api` lists S3's own operation names — GetObject, PutObject, +// ListObjectsV2, CreateMultipartUpload — so OOS is S3, not a named API +// osc-sdk-go pkg/oos/client.go is built on +// github.com/aws/aws-sdk-go-v2/service/s3, with `type Error = smithy.APIError`, +// so there is no proprietary error dialect to invent +// pkg/profile/profile.go:114 reads OSC_ENDPOINT_OOS, and +// pkg/profile/endpoint.go:12 accepts an `oos` key in the profile file, so the +// endpoint is redirectable WITHOUT TLS interception, a fork or dev_overrides +// pkg/profile/endpoint.go:27 gives the default authority, %s://oos.%s.outscale.com +// +// So the door is not simulated behind a false promise, which is the alternative +// #850 explicitly refuses. It is redirected the same way octl.sh already +// redirects this client's IaaS calls through OSC_ENDPOINT_API. +// +// ## What is NOT claimed here +// +// This declares the door, its endpoint variables and its measured missing-bucket +// envelope. It does not claim that octl's whole storage surface round-trips: no +// suite drives `octl storage` yet, and until one does, every operation of this +// door stays unproven by a real client whatever the S3 server underneath can +// serve. The conformance axis is the thing that says otherwise, and it says +// nothing yet. +// +// BucketConfiguration, ObjectDiscovery, ListObjectsV1 and PrivateACLWrite are +// deliberately left at their zero value rather than copied from the Scaleway +// profile next door. Each of those switches on an answer shape, and this door +// has one recording to its name. Copying a neighbour's switches would be +// inventing OOS's discovery behaviour from Scaleway's, which is the failure +// mode this pack's error helpers exist to avoid. +func (p *Pack) ObjectStorage(region string) s3.Profile { + return s3.Profile{ + Provider: Name, + Region: region, + MissingBucket: s3.OutscaleMissingBucket, + // Both names the SDK reads, in the order it reads them. OSC_ENDPOINT_OOS + // is the per-service one (pkg/profile/profile.go:114); the profile file's + // `oos` key wins over it when --config or --profile is given, which is + // the same precedence octl.sh documents for the IaaS endpoint. + Variables: []string{"OSC_ENDPOINT_OOS"}, + RegionVariable: "OSC_REGION", + // pkg/profile/endpoint.go:27, with its scheme placeholder resolved. + CloudHost: "oos." + region + ".outscale.com", + } +} diff --git a/internal/s3/bucket_configuration.go b/internal/s3/bucket_configuration.go index 3dad46e3..eedab7d4 100644 --- a/internal/s3/bucket_configuration.go +++ b/internal/s3/bucket_configuration.go @@ -250,8 +250,15 @@ func (s *Server) bucketConfiguration(w http.ResponseWriter, r *http.Request, nam id := rand.Text() w.Header().Set("X-Amz-Request-Id", id) w.Header().Set("X-Amz-Id-2", id) + // Addressed rather than inlined: missingBucketError's Resource and + // BucketName became pointers so the Outscale envelope could render + // present-and-empty, which `omitempty` on a + // string cannot express. This answer is unchanged — both fields are + // filled here, as the recording for it has them. + resource := "/" + name s.writeXML(w, r, http.StatusNotFound, missingBucketError{Code: "NoSuchTagSet", - Message: "The TagSet does not exist", RequestID: id, HostID: id, Resource: "/" + name, BucketName: name}) + Message: "The TagSet does not exist", RequestID: id, HostID: id, + Resource: &resource, BucketName: &name}) default: s.refuse(w, r, http.StatusNotImplemented, "bucket configuration operation is not implemented", nil) } diff --git a/internal/s3/bucket_configuration_test.go b/internal/s3/bucket_configuration_test.go index 6b4fa64d..cdd831d5 100644 --- a/internal/s3/bucket_configuration_test.go +++ b/internal/s3/bucket_configuration_test.go @@ -82,7 +82,8 @@ func TestBucketPrivateACLTagsAndScopedListing(t *testing.T) { var missing missingBucketError if w.Code != http.StatusNotFound || xml.Unmarshal(w.Body.Bytes(), &missing) != nil || missing.Code != "NoSuchTagSet" || missing.Message != "The TagSet does not exist" || - missing.BucketName != "bucket" || missing.RequestID == "" || missing.RequestID != missing.HostID { + missing.BucketName == nil || *missing.BucketName != "bucket" || + missing.RequestID == "" || missing.RequestID != missing.HostID { t.Fatal("absent tags do not follow the recorded cloud error:", w.Code, w.Body.String()) } } diff --git a/internal/s3/server.go b/internal/s3/server.go index 2fe1e4b5..e6c3eb92 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -38,6 +38,25 @@ const ( ScalewayMissingBucket MissingBucketShape = iota + 1 // ExoscaleMissingBucket follows the anonymous ch-gva-2 GET recording. ExoscaleMissingBucket + // OutscaleMissingBucket follows the anonymous eu-west-2 GET recording of + // 2026-11-09, kept at docs/evidence/outscale-oos-missing-bucket.json. + // + // It is a third envelope rather than a reuse of either, and the differences + // are exactly the kind nothing but a recording settles: + // + // is PRESENT and EMPTY — Scaleway fills it with /, + // Exoscale omits the element entirely + // is ABSENT — both others carry it + // x-amz-id-2 EQUALS x-amz-request-id — the others derive them apart + // no X-Amzn-Request-Id — unlike Exoscale + // final period on the message — like Exoscale, unlike Scaleway + // + // The recording cost nothing and touched no account: an anonymous GET on a + // name that does not exist sends no signature, so it is attributed to + // nobody, creates nothing and is billed to nobody. `server: nginx/1.20.0` + // on that answer is also worth keeping in mind — OOS is Outscale's own + // implementation, not AWS S3, so its envelope is theirs to differ in. + OutscaleMissingBucket ) // Config requires an explicit region, byte bound, measured error shape and log. @@ -84,7 +103,12 @@ func New(store *objectstore.Store, cfg Config) (*Server, error) { if cfg.MaxObjectBytes <= 0 || cfg.MaxObjectBytes > (math.MaxInt64-(64<<10))/2 { return nil, errors.New("s3: a positive representable upload byte bound is required") } - if cfg.MissingBucket != ScalewayMissingBucket && cfg.MissingBucket != ExoscaleMissingBucket { + // An allow-list, not a range check: `cfg.MissingBucket <= OutscaleMissingBucket` + // would silently admit a fourth shape the day somebody adds the constant and + // forgets the envelope, which is the one way this guard can rot. + switch cfg.MissingBucket { + case ScalewayMissingBucket, ExoscaleMissingBucket, OutscaleMissingBucket: + default: return nil, errors.New("s3: select a measured missing-bucket response shape") } for feature, response := range cfg.AbsentConfigurations { @@ -428,23 +452,44 @@ func (s *Server) refuse(w http.ResponseWriter, r *http.Request, status int, mess // These fields and their order follow the two anonymous GET controls from // phase 0, not a synthetic XML fixture. No other cloud error is inferred. type missingBucketError struct { - XMLName xml.Name `xml:"Error"` - Code string - Message string - RequestID string `xml:"RequestId,omitempty"` - HostID string `xml:"HostId,omitempty"` - Resource string `xml:",omitempty"` - BucketName string + XMLName xml.Name `xml:"Error"` + Code string + Message string + RequestID string `xml:"RequestId,omitempty"` + HostID string `xml:"HostId,omitempty"` + // Resource and BucketName are pointers because three states have to be + // told apart, and `omitempty` on a string can only express two. + // + // Outscale answers : the element is PRESENT and its + // text is EMPTY. With `string,omitempty` that is indistinguishable from + // Exoscale, which omits the element altogether, and encoding/xml would drop + // it. A nil pointer is omitted, a pointer to "" renders the empty element. + // + // BucketName is the mirror case: Scaleway and Exoscale both carry it, + // Outscale does not, and it used to have no `omitempty` at all so every + // envelope carried it whether the provider does or not. + Resource *string `xml:"Resource,omitempty"` + BucketName *string `xml:"BucketName,omitempty"` } func (s *Server) missingBucket(w http.ResponseWriter, r *http.Request, bucket string) { id, hostID := rand.Text(), rand.Text() - wire := missingBucketError{Code: "NoSuchBucket", Message: "The specified bucket does not exist", BucketName: bucket} - if s.cfg.MissingBucket == ScalewayMissingBucket { + wire := missingBucketError{Code: "NoSuchBucket", Message: "The specified bucket does not exist"} + switch s.cfg.MissingBucket { + case ScalewayMissingBucket: + hostID = id + resource := "/" + bucket + wire.RequestID, wire.HostID, wire.Resource, wire.BucketName = id, hostID, &resource, &bucket + case OutscaleMissingBucket: + // One identifier, not two: the recorded answer carried the same value + // in x-amz-id-2 and x-amz-request-id, and in . hostID = id - wire.RequestID, wire.HostID, wire.Resource = id, hostID, "/"+bucket - } else { + empty := "" + wire.Message += "." + wire.RequestID, wire.Resource = id, &empty + default: wire.Message += "." + wire.BucketName = &bucket } body, err := xml.Marshal(wire) if err != nil { @@ -455,6 +500,9 @@ func (s *Server) missingBucket(w http.ResponseWriter, r *http.Request, bucket st body = append([]byte(strings.TrimSuffix(xml.Header, "\n")), body...) w.Header().Set("X-Amzn-Request-Id", id) } + if s.cfg.MissingBucket == OutscaleMissingBucket { + body = append([]byte(strings.TrimSuffix(xml.Header, "\n")), body...) + } w.Header().Set("X-Amz-Request-Id", id) w.Header().Set("X-Amz-Id-2", hostID) w.Header().Set("Content-Type", "application/xml") diff --git a/internal/s3/server_test.go b/internal/s3/server_test.go index dcab27c8..fdac09e3 100644 --- a/internal/s3/server_test.go +++ b/internal/s3/server_test.go @@ -296,6 +296,11 @@ func TestUploadHeaderValidationAndConstructor(t *testing.T) { func(c *Config) { c.MaxObjectBytes = 1<<63 - 1 }, func(c *Config) { c.Region = "x\r\nInjected" }, func(c *Config) { c.MissingBucket = 0 }, + // A shape past the last measured one. The allow-list in New must + // refuse it by NAME: a range check would admit the day somebody + // adds a constant and forgets the envelope, and the provider would + // ship with whatever the default branch happens to render. + func(c *Config) { c.MissingBucket = OutscaleMissingBucket + 1 }, func(c *Config) { c.Log = nil }, } { cfg := s.cfg @@ -311,14 +316,26 @@ func TestUploadHeaderValidationAndConstructor(t *testing.T) { func TestMissingBucketFollowsMeasuredErrorDialects(t *testing.T) { s, _ := fixture(t) - for _, shape := range []MissingBucketShape{ScalewayMissingBucket, ExoscaleMissingBucket} { + for _, shape := range []MissingBucketShape{ + ScalewayMissingBucket, ExoscaleMissingBucket, OutscaleMissingBucket, + } { file := "exoscale-no-such-bucket.xml" - if shape == ScalewayMissingBucket { + switch shape { + case ScalewayMissingBucket: file = "scaleway-no-such-bucket.xml" + case OutscaleMissingBucket: + file = "outscale-no-such-bucket.xml" } // Real anonymous HTTPS GET responses recorded 2026-11-06T20:33:53Z - // at s3.fr-par.scw.cloud and sos-ch-gva-2.exo.io. Response identifiers - // are redacted and a trailing newline added; fields remain recorded. + // at s3.fr-par.scw.cloud and sos-ch-gva-2.exo.io, and 2026-11-09 at + // oos.eu-west-2.outscale.com. Response identifiers are redacted and a + // trailing newline added; fields remain recorded. + // + // The Outscale one is why Resource and BucketName are pointers: it + // carries present-and-empty and NO BucketName, + // which a string with `omitempty` cannot tell from Exoscale omitting + // the element. This loop is what proves it, because it re-marshals the + // recording and compares it to what the server emits. recording, err := os.ReadFile(filepath.Join("testdata", file)) if err != nil { t.Fatal(err) @@ -327,7 +344,21 @@ func TestMissingBucketFollowsMeasuredErrorDialects(t *testing.T) { if err := xml.Unmarshal(recording, &observed); err != nil { t.Fatal("cannot decode real producer response", err) } - if observed.Code != "NoSuchBucket" || observed.BucketName == "" { + // The recording must decode into the boundary type, and it must carry + // the field its own dialect carries. Outscale's answer has no + // BucketName at all, so demanding one of every dialect would have made + // this check reject a faithful recording. + switch { + case observed.Code != "NoSuchBucket": + t.Fatal("recording did not decode into the boundary type", observed) + case shape == OutscaleMissingBucket: + if observed.BucketName != nil { + t.Fatal("the Outscale recording grew a BucketName", observed) + } + if observed.Resource == nil || *observed.Resource != "" { + t.Fatal("the Outscale recording lost its present-and-empty Resource", observed) + } + case observed.BucketName == nil || *observed.BucketName == "": t.Fatal("recording did not decode into the boundary type", observed) } s.cfg.MissingBucket = shape @@ -338,35 +369,90 @@ func TestMissingBucketFollowsMeasuredErrorDialects(t *testing.T) { t.Fatal(err) } observed.BucketName = emitted.BucketName - if shape == ScalewayMissingBucket { - observed.RequestID, observed.HostID, observed.Resource = emitted.RequestID, emitted.HostID, "/missing" + switch shape { + case ScalewayMissingBucket: + resource := "/missing" + observed.RequestID, observed.HostID, observed.Resource = emitted.RequestID, emitted.HostID, &resource if emitted.RequestID != emitted.HostID { t.Fatal("recorded Scaleway identifiers were equal", emitted) } + case OutscaleMissingBucket: + // Only the identifier is substituted. Resource is NOT, because + // present-and-empty is the measured shape and copying the emitted + // value would let the server define what it is compared against — + // which is the one way this comparison could stop measuring. + observed.RequestID = emitted.RequestID + if emitted.Resource == nil || *emitted.Resource != "" { + t.Fatal("the Outscale envelope lost its present-and-empty Resource", emitted) + } + if emitted.BucketName != nil { + t.Fatal("the Outscale envelope grew a BucketName", emitted) + } } want, err := xml.Marshal(observed) if err != nil { t.Fatal(err) } - if shape == ExoscaleMissingBucket { + // Both dialects whose recording carries the declaration carry it here. + if shape == ExoscaleMissingBucket || shape == OutscaleMissingBucket { want = append([]byte(strings.TrimSuffix(xml.Header, "\n")), want...) } if body != string(want) { t.Fatal("response differs from the actual producer envelope", body, string(want)) } if w.Code != 404 || w.Header().Get("Content-Type") != "application/xml" || - !strings.Contains(body, "NoSuchBucket") || !strings.Contains(body, "missing") { + !strings.Contains(body, "NoSuchBucket") { + t.Fatal(w.Code, body, w.Header()) + } + // The bucket name is asserted per dialect rather than of all three, and + // the Outscale branch asserts its ABSENCE, which is stricter than the + // blanket Contains this replaced: that one demanded + // missing of every envelope, so it would have + // rejected a faithful Outscale answer and, worse, would have passed a + // wrong one that added the element. + if shape == OutscaleMissingBucket { + if strings.Contains(body, "") { + t.Fatal("the Outscale envelope names a bucket its producer does not", body) + } + if !strings.Contains(body, "") { + t.Fatal("the Outscale envelope lost the empty Resource element", body) + } + // Measured: the producer answered the same value in both headers. + if id2 := w.Header().Get("X-Amz-Id-2"); id2 == "" || id2 != w.Header().Get("X-Amz-Request-Id") { + t.Fatal("Outscale answers one identifier in both headers", w.Header()) + } + if w.Header().Get("X-Amzn-Request-Id") != "" { + t.Fatal("the Outscale envelope grew Exoscale's extra identifier header", w.Header()) + } + } else if !strings.Contains(body, "missing") { t.Fatal(w.Code, body, w.Header()) } - if shape == ScalewayMissingBucket { + // One branch per dialect, each asserting what its own producer carries + // AND what it does not. The last two used to share an `else` whose + // shape was Exoscale's: no declaration prefix was allowed to be absent, + // and had to be absent. Outscale carries both the prefix + // and a RequestId, so that `else` rejected a faithful answer — and an + // `else` that covered two dialects quietly claims to cover every one + // that comes later. + switch shape { + case ScalewayMissingBucket: if strings.HasPrefix(body, "") || !strings.Contains(body, "/missing") || !strings.Contains(body, "The specified bucket does not exist") { t.Fatal(body) } - } else if !strings.HasPrefix(body, "") || - !strings.Contains(body, "The specified bucket does not exist.") { - t.Fatal(body) + case ExoscaleMissingBucket: + if !strings.HasPrefix(body, "") || + strings.Contains(body, "") || + !strings.Contains(body, "The specified bucket does not exist.") { + t.Fatal(body) + } + case OutscaleMissingBucket: + if !strings.HasPrefix(body, "") || + strings.Contains(body, "") || + !strings.Contains(body, "The specified bucket does not exist.") { + t.Fatal(body) + } } } } diff --git a/internal/s3/testdata/outscale-no-such-bucket.xml b/internal/s3/testdata/outscale-no-such-bucket.xml new file mode 100644 index 00000000..e654d43f --- /dev/null +++ b/internal/s3/testdata/outscale-no-such-bucket.xml @@ -0,0 +1 @@ +NoSuchBucketThe specified bucket does not exist.REDACTED_RESPONSE_ID diff --git a/tools/falsify/specs/oos-missing-bucket-envelope.json b/tools/falsify/specs/oos-missing-bucket-envelope.json new file mode 100644 index 00000000..a16fccaf --- /dev/null +++ b/tools/falsify/specs/oos-missing-bucket-envelope.json @@ -0,0 +1,34 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "the OOS envelope loses its XML declaration, which is the half of the recording a reader would assume rather than check: Scaleway has none, so copying Scaleway here looks right and is wrong", + "file": "internal/s3/server.go", + "find": "\tif s.cfg.MissingBucket == OutscaleMissingBucket {\n\t\tbody = append([]byte(strings.TrimSuffix(xml.Header, \"\\n\")), body...)\n\t}", + "replace": "\tif s.cfg.MissingBucket == OutscaleMissingBucket && false {\n\t\tbody = append([]byte(strings.TrimSuffix(xml.Header, \"\\n\")), body...)\n\t}", + "test": "TestMissingBucketFollowsMeasuredErrorDialects" + }, + { + "label": "the OOS branch falls through to the Exoscale one, so Resource is omitted instead of present-and-empty and a BucketName appears that the producer does not send: the single likeliest shortcut, since the two dialects agree on the final period and the declaration", + "file": "internal/s3/server.go", + "find": "\tcase OutscaleMissingBucket:", + "replace": "\tcase OutscaleMissingBucket + 100:", + "test": "TestMissingBucketFollowsMeasuredErrorDialects" + }, + { + "label": "the two response identifiers are derived apart, where the recorded answer carried the same value in x-amz-id-2 and x-amz-request-id", + "file": "internal/s3/server.go", + "find": "\tcase OutscaleMissingBucket:\n\t\t// One identifier, not two: the recorded answer carried the same value\n\t\t// in x-amz-id-2 and x-amz-request-id, and in .\n\t\thostID = id", + "replace": "\tcase OutscaleMissingBucket:\n\t\t// One identifier, not two: the recorded answer carried the same value\n\t\t// in x-amz-id-2 and x-amz-request-id, and in .\n\t\thostID = id\n\t\thostID = rand.Text()", + "test": "TestMissingBucketFollowsMeasuredErrorDialects" + }, + { + "label": "a fourth shape is admitted by range rather than by name, so adding a constant without an envelope ships a provider whose errors are whatever the default branch happens to render (the constructor's allow-list, asserted by TestUploadHeaderValidationAndConstructor)", + "file": "internal/s3/server.go", + "find": "\tswitch cfg.MissingBucket {\n\tcase ScalewayMissingBucket, ExoscaleMissingBucket, OutscaleMissingBucket:\n\tdefault:\n\t\treturn nil, errors.New(\"s3: select a measured missing-bucket response shape\")\n\t}", + "replace": "\tswitch cfg.MissingBucket {\n\tcase ScalewayMissingBucket, ExoscaleMissingBucket, OutscaleMissingBucket, OutscaleMissingBucket + 1:\n\tdefault:\n\t\treturn nil, errors.New(\"s3: select a measured missing-bucket response shape\")\n\t}", + "test": "TestUploadHeaderValidationAndConstructor" + } + ], + "note": "The subject is the OOS door of #850, and the envelope it answers for an absent bucket. Outscale's OOS is S3 — octl storage api lists S3's own operation names, and osc-sdk-go's pkg/oos is built on aws-sdk-go-v2/service/s3 with smithy errors — so there was no proprietary dialect to invent. There WAS an envelope to measure, and the three differences from its neighbours are exactly what no reading of a document would have produced: present and EMPTY where Scaleway fills it and Exoscale omits it, NO where both others carry it, and one identifier answered in both x-amz-id-2 and x-amz-request-id where the others derive them apart. Recorded by an anonymous GET on a name that does not exist (docs/evidence/outscale-oos-missing-bucket.json), which sends no signature, is attributed to no account, creates nothing and is billed to nobody: the user's account was offered for this and deliberately not used. The first two mutations are the shortcuts a maintainer would actually take — drop the declaration because Scaleway has none, or let the branch fall through to Exoscale because the two agree on the period — and both are caught by comparing the emitted envelope against the recording rather than against a hand-written expectation. The fourth leaves the two dialect branches alone and widens the constructor's allow-list instead, because a range check there is how a fourth provider would ship with whatever the default branch renders." +} From e7a13c7ddb330ebd5803aac60014b26c2e9d1cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 16:39:25 +0100 Subject: [PATCH 14/28] feat(outscale): octl drives the OOS door, and the bucket listing stops hiding behind a configuration switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first Outscale Object Storage client path this project has, which is the criterion #850 states: "one OUTSCALE or OOS client path is driven if the current official tooling exposes it". ## The coupling that kept the door from serving its client ListBuckets lived behind Config.BucketConfiguration. `git log -S` names the commit that put it there: 7c3e087, "prove private Scaleway bucket lifecycle", which measured that provider's ACL, tagging and list shapes in one go. A coupling of opportunity, read afterwards as one of meaning — the field's name says "the configuration of a bucket" and it was taken as "this door may enumerate buckets". It cost a real client. octl v0.0.32 opens with ListBuckets and reaches the door at exactly the canonical bare endpoint, captured on the wire with a listener that printed the request line and nothing else: OSC_ENDPOINT_OOS=...../outscale/eu-west-2 -> GET /_feint/s3/outscale/eu-west-2?x-id=ListBuckets OSC_ENDPOINT_OOS=...../outscale/eu-west-2/ -> GET /_feint/s3/outscale/eu-west-2/?x-id=ListBuckets so octl preserves the endpoint's path, unlike exo which drops it, and one character separated a served request from a 400. With the slash, the door was reached and refused anyway: "invalid bucket or object key", because the Outscale profile leaves BucketConfiguration off on purpose — that switch also turns on discovery answers whose OOS shapes nobody has recorded. A door was refusing the one operation its client opens with, to avoid claiming five others. The same switch held the creation date, so once the listing was served CreateBucket answered 200 and the ListBuckets after it answered 501, "legacy bucket listing has no measured creation metadata". Two gestures of one client disagreeing about one bucket. Both are decoupled. A creation date is a property of a bucket, not a configuration of it, and ListBuckets renders it. What stays behind BucketConfiguration is the CreateBucketConfiguration BODY, which is what the field names. The condition is removed rather than replaced by a second flag: a flag all three doors set to true is a constant wearing a switch's name, and this repository prefers no guard to one that cannot fail. Nothing widens for Scaleway or Exoscale, which both already carried it. ## A defect the new test found on its own `?x-id=ListBuckets&x-id=ListBuckets` was served as a bucket listing, because len(query) counts KEYS and a duplicated parameter is one key with two values. Now len(query["x-id"]) == 1, which is the duplicate-parameter refusal TestBucketConfigurationRefusals/duplicate-query already holds one step over: a client that sends a parameter twice has not established its intent, and S3 does not define which value wins. ## Measured against the account, read-only TestMountedOOSOfficialOctl drives three gestures, and two of its expectations were compared against the REAL OOS with the same octl binary and the same subcommands. One GET each, nothing created, no bucket on the account before or after; credentials never passed through the harness, since octl reads ~/.osc/config.json itself and that file stays behind its deny rule. ListBuckets, empty cloud [] (2 bytes) door [] (2 bytes) identical absent bucket, refused cloud `The server returned an error { "Message": null, "ErrorCodeOverride": null }` door byte for byte the same That second reading is why the test does NOT assert "NoSuchBucket" in octl's output: the client discards the S3 error code, on the cloud exactly as here. Two earlier versions of that assertion failed for that reason — first through HeadBucket, which carries no body at all, then through ListObjectsV2, whose body the client drops. Both failures were mine, and the cloud reading is what told them apart from an emulator defect. The envelope itself is asserted byte for byte in internal/s3 against docs/evidence/outscale-oos-missing-bucket.json. ## Three places the harness refused to let me assume The first falsification spec aimed its three mutations at TestMountedBucketListingAcceptsOnlyItsCanonicalBareEndpoint, which sounded right and covers none of this: all three stayed GREEN. A spec aimed at a test that does not cover its subject proves nothing while looking exactly like proof. That is what TestADoorWithoutConfigurationStillListsAndDatesItsBuckets exists for, and the mutations bite it. TestEveryCitedTestExists then caught the comment citing that test, because the name had been hyphenated across two lines and the citation read as a test that does not exist. And the creation-date mutation was refused for dropping identifiers; it keeps every name and empties only the value. ## Verification mise run check / docs:check / shapes:check pass mise run falsify:lint / lint-shell / release:surface pass mise run falsify -- oos-door-serves-its-listing.json 3/3 compiled and bit mise run conformance:s3:oos PASS ## Still not claimed One client, three gestures. Nothing about ACL, CORS, ownership or tagging discovery on OOS, whose shapes nobody has recorded; those switches stay off. No multipart and no byte round-trip through this door. The door is still not in the default --s3-doors, because a door enabled by default is a door the project vouches for, and one suite of three gestures is not that yet. #850 is not closed. Neither is #853, #858 or #849. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- internal/cli/oos_conformance_test.go | 130 ++++++++++++++++++ internal/conformancetest/aws.go | 18 +++ internal/conformancetest/octl.go | 53 +++++++ internal/s3/bucket_listing_test.go | 104 ++++++++++++++ internal/s3/server.go | 69 +++++++++- mise.toml | 14 ++ .../specs/oos-door-serves-its-listing.json | 27 ++++ 7 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 internal/cli/oos_conformance_test.go create mode 100644 internal/conformancetest/octl.go create mode 100644 internal/s3/bucket_listing_test.go create mode 100644 tools/falsify/specs/oos-door-serves-its-listing.json diff --git a/internal/cli/oos_conformance_test.go b/internal/cli/oos_conformance_test.go new file mode 100644 index 00000000..be71bd0a --- /dev/null +++ b/internal/cli/oos_conformance_test.go @@ -0,0 +1,130 @@ +//go:build conformance + +package cli + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/conformancetest" + "github.com/stephrobert/feint/internal/s3" +) + +// TestMountedOOSOfficialOctl drives the real octl against the local OOS door. +// +// It is the first Outscale Object Storage client path this project has, and +// #850 asks for exactly one: "one OUTSCALE or OOS client path is driven if the +// current official tooling exposes it". It does — octl v0.0.32 carries +// `octl storage`, whose `api` subcommand lists S3's own operation names, +// because osc-sdk-go's pkg/oos is built on aws-sdk-go-v2/service/s3. +// +// ## What makes this a proof rather than a round-trip against myself +// +// The empty-listing assertion below was compared against the REAL OOS on +// 2026-11-09, with the same octl binary and the same subcommand: +// +// the account, read-only -> [] (2 bytes) +// this door -> [] (2 bytes) +// +// identical. That cloud reading was one GET, created nothing, and the account +// held no bucket before or after; the credentials never passed through the +// harness, since octl reads ~/.osc/config.json itself. This test does not +// repeat that call — a test that needs an account is a test nobody can run — +// but the expectation it encodes came from it rather than from a guess. +// +// ## What it does not prove +// +// One client, three gestures. It says nothing about ACL, CORS, ownership or +// tagging discovery on OOS, whose shapes nobody has recorded, and the Outscale +// profile leaves those switches off for that reason. It does not prove multipart +// or byte round-trips through this door either; those have their own suites for +// the two doors that measured them. +func TestMountedOOSOfficialOctl(t *testing.T) { + const door = "outscale/eu-west-2" + ts, doors := storageServer(t, true, door) + if doors == nil { + t.Fatal("the OOS door did not mount") + } + endpoint := ts.URL + s3.DoorPath("outscale", "eu-west-2") + octl := conformancetest.OCTL(t, endpoint) + + // 1. The listing the client opens with, and the gesture that was refused + // until ListBuckets stopped living behind BucketConfiguration. Measured + // against the account: an empty inventory is `[]`. + t.Log("- the listing octl opens with, against an empty door") + out := octl.Run(t, "--no-upgrade", "storage", "api", "ListBuckets", "--output", "json") + var listed []map[string]any + if err := json.Unmarshal(out, &listed); err != nil { + t.Fatalf("octl's ListBuckets output is not a JSON array: %v %s", err, out) + } + if len(listed) != 0 { + t.Fatalf("an empty door listed %d bucket(s): %s", len(listed), out) + } + t.Logf(" ok: [] over zero buckets, the shape the account answers") + + // 2. A bucket the client creates, and the same listing finding it. Two + // gestures rather than one, because a create whose result is read back + // by nothing is a 200 and not a proof. + t.Log("- a bucket octl creates, found by the listing afterwards") + const bucket = "feint-oos-conformance" + octl.Run(t, "--no-upgrade", "storage", "api", "CreateBucket", "--Bucket", bucket) + out = octl.Run(t, "--no-upgrade", "storage", "api", "ListBuckets", "--output", "json") + if err := json.Unmarshal(out, &listed); err != nil { + t.Fatalf("ListBuckets output is not a JSON array after a create: %v %s", err, out) + } + found := false + for _, entry := range listed { + if name, _ := entry["Name"].(string); name == bucket { + found = true + if created, _ := entry["CreationDate"].(string); created == "" { + t.Fatalf("the bucket carries no creation date: %s", out) + } + } + } + if !found || len(listed) != 1 { + t.Fatalf("the created bucket is not the one bucket listed: %s", out) + } + t.Logf(" ok: created and found, with a creation date") + + // 3. The measured refusal. The envelope itself is asserted byte for byte in + // internal/s3 against docs/evidence/outscale-oos-missing-bucket.json; + // what this adds is that the official client SURFACES it rather than + // choking on it, which a hand-made HTTP request cannot show. + t.Log("- a bucket that does not exist, refused in the recorded dialect") + // What this asserts, and what it deliberately does not. + // + // octl does NOT surface the S3 error code. Measured against the REAL OOS on + // 2026-11-09, same binary, same subcommand, a bucket that does not exist: + // + // The server returned an error + // { "Message": null, "ErrorCodeOverride": null } + // + // byte for byte what this door produces. So the client prints an error and + // discards the code, and a test asserting "NoSuchBucket" in octl's output + // would fail against Outscale itself — this one did, twice, first through + // HeadBucket (no body at all) and then through ListObjectsV2 (a body the + // client drops). Both failures were mine, and the cloud reading is what + // told them apart from an emulator defect. + // + // The envelope IS asserted, byte for byte, in internal/s3 against + // docs/evidence/outscale-oos-missing-bucket.json. What belongs here is the + // only thing this client can show: that an absent bucket is refused rather + // than invented, and that the refusal reaches the caller as a failure. + refused, err := octl.Attempt(t, "--no-upgrade", "storage", "api", "ListObjectsV2", + "--Bucket", "feint-oos-absent") + if err == nil { + t.Fatalf("octl was told an absent bucket exists: %s", refused) + } + if text := string(refused); !strings.Contains(text, "The server returned an error") { + t.Fatalf("the refusal did not reach the client at all: %s", text) + } + // And the listing is unchanged by the refusal: the absent bucket was not + // quietly created by being asked for, which is the failure a 200 on + // CreateBucket-by-accident would look like. + out = octl.Run(t, "--no-upgrade", "storage", "api", "ListBuckets", "--output", "json") + if err := json.Unmarshal(out, &listed); err != nil || len(listed) != 1 { + t.Fatalf("asking for an absent bucket changed the inventory: %v %s", err, out) + } + t.Logf(" ok: refused as a failure, and the inventory is untouched") +} diff --git a/internal/conformancetest/aws.go b/internal/conformancetest/aws.go index fa887639..b2bfcda7 100644 --- a/internal/conformancetest/aws.go +++ b/internal/conformancetest/aws.go @@ -89,6 +89,24 @@ func (c Client) Run(t *testing.T, args ...string) []byte { return out } +// Attempt runs the client and returns its output and its error instead of +// failing the test, for the cases where the REFUSAL is the subject. +// +// Run cannot serve those: it calls t.Fatalf on a non-zero exit, so a test that +// wants to assert "the client was told no" would have to pass for the wrong +// reason — either by not running the case, or by reading a success as proof. +// A refusal a suite cannot express is a refusal the emulator can stop sending +// without anything noticing. +func (c Client) Attempt(t *testing.T, args ...string) ([]byte, error) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, c.Path, args...) //nolint:gosec // pinned official client, explicit arguments, no shell + cmd.Env = c.Env + out, err := cmd.CombinedOutput() + return out, err +} + // ProbeProxyGuard sends a local CONNECT refusal control, never a cloud request. func ProbeProxyGuard(ctx context.Context, endpoint string) error { proxy, err := url.Parse(endpoint) diff --git a/internal/conformancetest/octl.go b/internal/conformancetest/octl.go new file mode 100644 index 00000000..2b809fc6 --- /dev/null +++ b/internal/conformancetest/octl.go @@ -0,0 +1,53 @@ +//go:build conformance + +package conformancetest + +import ( + "os/exec" + "testing" +) + +// OCTL builds a private octl environment pointed at one local OOS door. +// +// endpoint must carry the door's canonical path INCLUDING its trailing slash. +// Measured on the wire 2026-11-09, with a listener that printed the request +// line and nothing else: +// +// OSC_ENDPOINT_OOS=http://127.0.0.1:P/_feint/s3/outscale/eu-west-2 +// -> GET /_feint/s3/outscale/eu-west-2?x-id=ListBuckets +// OSC_ENDPOINT_OOS=http://127.0.0.1:P/_feint/s3/outscale/eu-west-2/ +// -> GET /_feint/s3/outscale/eu-west-2/?x-id=ListBuckets +// +// So octl preserves the endpoint's path, unlike exo which drops it, and the +// only thing between the first form and a served request was one character. +// The emulator requires the canonical prefix, trailing slash included, and is +// right to: a prefix it accepted loosely would dispatch two different doors to +// the same handler. +// +// --no-upgrade on every call, for the reason octl.sh states: without it octl +// asks GitHub for a newer release, which is a network call this harness must +// not make and a version this harness did not pin. +func OCTL(t *testing.T, endpoint string) Client { + t.Helper() + binary, err := exec.LookPath("octl") + if err != nil { + t.Skip("octl is not installed; the OOS door is unproven in this run") + } + home := t.TempDir() + env := []string{ + "HOME=" + home, + "PATH=/usr/bin:/bin", + // The repository's deliberately public placeholders. The emulator never + // validates the signature; the client still refuses to sign without + // well-formed credentials. + "OSC_ACCESS_KEY=AAAAAAAAAAAAAAAAAAAA", + "OSC_SECRET_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + "OSC_REGION=eu-west-2", + "OSC_ENDPOINT_OOS=" + endpoint, + // Nothing inherits a proxy here: an OOS call that left this machine + // would be the one failure this harness exists to make impossible. + "NO_PROXY=*", + "no_proxy=*", + } + return Client{Path: binary, Home: home, Env: env} +} diff --git a/internal/s3/bucket_listing_test.go b/internal/s3/bucket_listing_test.go new file mode 100644 index 00000000..517acaf6 --- /dev/null +++ b/internal/s3/bucket_listing_test.go @@ -0,0 +1,104 @@ +package s3 + +import ( + "encoding/xml" + "net/http" + "testing" +) + +// A door that declares no configuration shapes still enumerates its buckets, +// and the buckets it creates carry the date that enumeration renders. +// +// This is the test the decoupling needed and did not have. ListBuckets used to +// sit behind Config.BucketConfiguration, and so did the creation metadata, and +// `git log -S` names the commit that put them there: 7c3e087, "prove private +// Scaleway bucket lifecycle", which measured that provider's ACL, tagging and +// list shapes in one go. Opportunity, not meaning — the field's name says "the +// configuration of a bucket" and it was read as "this door may enumerate +// buckets". +// +// It cost a real client. octl v0.0.32 opens with ListBuckets and reaches the +// OOS door at exactly the canonical bare endpoint, captured on the wire: +// +// GET /_feint/s3/outscale/eu-west-2/?x-id=ListBuckets +// +// and got 400 "invalid bucket or object key", because the Outscale profile +// leaves BucketConfiguration off on purpose: that switch also turns on +// discovery answers whose OOS shapes nobody has recorded. Then, once the +// listing was served, CreateBucket answered 200 and the ListBuckets after it +// answered 501 "legacy bucket listing has no measured creation metadata" — +// two gestures of one client disagreeing about one bucket. +// +// The fixture here carries BucketConfiguration at its zero value, which is why +// it is the right witness: it IS an OOS-shaped door. +// +// tools/falsify/specs/oos-door-serves-its-listing.json mutates all three +// properties, and each one reds this test. The first version of that spec +// aimed at TestMountedBucketListingAcceptsOnlyItsCanonicalBareEndpoint and all +// three mutations stayed green — a spec pointed at a test that does not cover +// its subject proves nothing while looking exactly like proof. +func TestADoorWithoutConfigurationStillListsAndDatesItsBuckets(t *testing.T) { + s, _ := fixture(t) + if s.cfg.BucketConfiguration { + t.Fatal("this witness is only meaningful on a door that declares no configuration shapes") + } + + type listed struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Buckets []struct { + Name string + CreationDate string + } `xml:"Buckets>Bucket"` + } + + // The canonical bare endpoint, with the x-id the AWS SDK appends. Both + // forms the client sends. + for _, path := range []string{"/", "/?x-id=ListBuckets"} { + w := request(t, s, http.MethodGet, path, nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("%s: the listing was refused with %d: %s", path, w.Code, w.Body.String()) + } + var answer listed + if err := xml.Unmarshal(w.Body.Bytes(), &answer); err != nil { + t.Fatalf("%s: the listing is not a ListAllMyBucketsResult: %v %s", path, err, w.Body.String()) + } + if len(answer.Buckets) != 0 { + t.Fatalf("%s: an empty door listed %d bucket(s)", path, len(answer.Buckets)) + } + } + + // A bucket created through this door, and the same listing finding it with + // a date. Without the creation metadata the listing answers 501 rather than + // omitting the bucket, which is the shape of the defect octl met. + if w := request(t, s, http.MethodPut, "/listed-bucket", nil, nil); w.Code != http.StatusOK { + t.Fatalf("CreateBucket was refused with %d: %s", w.Code, w.Body.String()) + } + w := request(t, s, http.MethodGet, "/?x-id=ListBuckets", nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("the listing after a create was refused with %d: %s", w.Code, w.Body.String()) + } + var answer listed + if err := xml.Unmarshal(w.Body.Bytes(), &answer); err != nil { + t.Fatalf("the listing after a create does not decode: %v %s", err, w.Body.String()) + } + if len(answer.Buckets) != 1 || answer.Buckets[0].Name != "listed-bucket" { + t.Fatalf("the created bucket is not the one bucket listed: %s", w.Body.String()) + } + if answer.Buckets[0].CreationDate == "" { + t.Fatalf("the created bucket carries no creation date: %s", w.Body.String()) + } + + // And the listing stays a listing. A query it never measured must not be + // read as one, or every unrouted GET becomes a bucket enumeration — the + // mirror of the gate this test exists to keep removed. + for _, path := range []string{ + "/?x-id=ListBuckets&unmeasured=1", + "/?acl=", + "/?x-id=ListBuckets&x-id=ListBuckets", + } { + if w := request(t, s, http.MethodGet, path, nil, nil); w.Code == http.StatusOK && + xml.Unmarshal(w.Body.Bytes(), &answer) == nil && answer.XMLName.Local == "ListAllMyBucketsResult" { + t.Fatalf("%s was served as a bucket listing: %s", path, w.Body.String()) + } + } +} diff --git a/internal/s3/server.go b/internal/s3/server.go index e6c3eb92..df481023 100644 --- a/internal/s3/server.go +++ b/internal/s3/server.go @@ -187,8 +187,54 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.refuse(w, r, http.StatusNotImplemented, "S3 header feature is not implemented", err) return } - if rawBucket == "" && rawKey == "" && s.cfg.BucketConfiguration && - (len(query) == 0 || (len(query) == 1 && query.Has("x-id"))) && r.Method == http.MethodGet { + // ListBuckets, and no longer behind BucketConfiguration. + // + // The two were coupled by opportunity rather than by meaning: `git log -S` + // puts the condition in 7c3e087, "prove private Scaleway bucket lifecycle", + // which measured that provider's ACL, tagging and list shapes in one go. So + // the gate said "this door serves bucket configuration" and was read as + // "this door may enumerate buckets", which are not the same claim — and the + // field's name says the first. + // + // It cost a real client. `octl storage bucket list` reaches the OOS door at + // exactly the canonical bare endpoint, measured on the wire: + // + // GET /_feint/s3/outscale/eu-west-2/?x-id=ListBuckets + // + // and got 400 "invalid bucket or object key", because the Outscale profile + // leaves BucketConfiguration at its zero value — deliberately, since that + // switch also turns on discovery answers whose OOS shapes nobody has + // recorded. A door was refusing the one operation its client opens with, to + // avoid claiming five others. + // + // Nothing is widened for the other two: both already carried + // BucketConfiguration: true, so both already served this. The condition is + // removed rather than replaced by a second flag, because a flag all three + // doors set to true is a constant wearing a switch's name, and this + // repository prefers no guard to one that cannot fail. + // + // What stays narrow is everything else: the query must be empty or carry + // only x-id (which the AWS SDK appends), the method must be GET, and both + // the bucket and key must be absent. That is what makes this a listing + // rather than a catch-all for every unrouted GET. + // + // TestADoorWithoutConfigurationStillListsAndDatesItsBuckets is the control, + // and it is a different test from the one this comment first cited. The + // first version named the canonical-bare-endpoint test, which sounded + // right and covers none of this: all three mutations of + // tools/falsify/specs/oos-door-serves-its-listing.json stayed green against + // it. A spec aimed at a test that does not cover its subject proves nothing + // while looking exactly like proof. + // len(query["x-id"]) == 1, not just query.Has: len(query) counts KEYS, so + // `?x-id=ListBuckets&x-id=ListBuckets` is one key with two values and used + // to satisfy this. Found by the test below rather than reasoned about, and + // it is the same duplicate-parameter refusal + // TestBucketConfigurationRefusals/duplicate-query already holds one step + // over — a client that sends a parameter twice is a client whose intent is + // not established, and S3 itself does not define which value wins. + if rawBucket == "" && rawKey == "" && + (len(query) == 0 || (len(query) == 1 && len(query["x-id"]) == 1)) && + r.Method == http.MethodGet { s.listBuckets(w, r) return } @@ -273,7 +319,24 @@ func (s *Server) bucket(w http.ResponseWriter, r *http.Request, name string) { s.refuse(w, r, http.StatusNotImplemented, "CreateBucket configuration is not implemented", readErr) return } - err = s.store.CreateBucket(r.Context(), s.name(name)) + // With its creation date, like the branch above, and the split between + // the two branches is now only about the CreateBucketConfiguration + // BODY — which is what BucketConfiguration names. + // + // A creation date is a property of the bucket, not a configuration of + // it, and ListBuckets renders it: `listBuckets` refuses outright when + // the metadata is missing, with "legacy bucket listing has no measured + // creation metadata". So a door without BucketConfiguration used to + // create buckets that its own listing then refused to show — measured + // with octl against the OOS door, which answered 200 to CreateBucket + // and 501 to the ListBuckets that followed. Two gestures of the same + // client, disagreeing about the same bucket. + // + // TestMountedOOSOfficialOctl fails without this: its second step + // creates a bucket and expects the listing to find it. + err = s.store.CreateBucketWithMetadata(r.Context(), s.name(name), map[string]string{ + bucketCreated: time.Now().UTC().Format(time.RFC3339Nano), + }) case http.MethodHead: err = s.store.HeadBucket(r.Context(), s.name(name)) case http.MethodDelete: diff --git a/mise.toml b/mise.toml index b72e423a..56093854 100644 --- a/mise.toml +++ b/mise.toml @@ -58,6 +58,20 @@ run = "go test -race -tags=conformance ./internal/s3 -run '^TestOfficialAWSCLIBy description = "Prove AWS/scw/exo bytes, normal Exo bucket/native usage, multipart and cleanup across actual restart" run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedS3OfficialAWSCLIBytes$' -count=1 -v" +[tasks."conformance:s3:oos"] +description = "Drive octl against the OOS door: empty listing, a created bucket found again, an absent one refused" +# Its own task rather than a line in `conformance`, for the same reason +# conformance:s3:mounted is: the object-store doors are opt-in, and the OOS one +# is not even in the default --s3-doors, because no suite vouched for it until +# this one. A task that needs octl installed also has no business reddening the +# pass that must run with no client at all. +# +# The empty-listing expectation was compared against the real OOS — same octl, +# same subcommand, `[]` on both sides — and so was the refusal, which octl +# prints without its S3 code on the cloud exactly as it does here. Those +# readings were one GET each, created nothing, and are recorded in the test. +run = "go test -race -tags=conformance ./internal/cli -run '^TestMountedOOSOfficialOctl$' -count=1 -v" + [tasks."conformance:s3:exo:endpoint"] description = "Measure exo 1.95.1 dropping configured S3 URL prefixes; not a successful byte lifecycle" run = "go test -race -tags=conformance ./internal/cli -run '^TestExoDropsConfiguredS3EndpointPrefix$' -count=1 -v" diff --git a/tools/falsify/specs/oos-door-serves-its-listing.json b/tools/falsify/specs/oos-door-serves-its-listing.json new file mode 100644 index 00000000..5fa3fb1e --- /dev/null +++ b/tools/falsify/specs/oos-door-serves-its-listing.json @@ -0,0 +1,27 @@ +{ + "package": "./internal/s3/", + "mutations": [ + { + "label": "the bucket listing goes back behind BucketConfiguration, so a door that declares no configuration shapes refuses the one gesture its client opens with — measured: octl got 400 'invalid bucket or object key' on the canonical bare endpoint", + "file": "internal/s3/server.go", + "find": "\tif rawBucket == \"\" && rawKey == \"\" &&\n\t\t(len(query) == 0 || (len(query) == 1 && len(query[\"x-id\"]) == 1)) &&", + "replace": "\tif rawBucket == \"\" && rawKey == \"\" && s.cfg.BucketConfiguration &&\n\t\t(len(query) == 0 || (len(query) == 1 && len(query[\"x-id\"]) == 1)) &&", + "test": "TestADoorWithoutConfigurationStillListsAndDatesItsBuckets" + }, + { + "label": "the listing widens to any unrouted GET, accepting a query it never measured: the mirror risk of the line above, and the one a careless decoupling would ship", + "file": "internal/s3/server.go", + "find": "\t\t(len(query) == 0 || (len(query) == 1 && len(query[\"x-id\"]) == 1)) &&\n\t\tr.Method == http.MethodGet {", + "replace": "\t\t(len(query) == 0 || (len(query) >= 1 && len(query[\"x-id\"]) >= 1)) &&\n\t\tr.Method == http.MethodGet {", + "test": "TestADoorWithoutConfigurationStillListsAndDatesItsBuckets" + }, + { + "label": "the creation date is written empty, so the door answers 200 to CreateBucket and 501 to the ListBuckets that follows: two gestures of one client disagreeing about one bucket. Every name is kept and evaluated; only the value is emptied, which is what the harness asks for", + "file": "internal/s3/server.go", + "find": "\t\t\tbucketCreated: time.Now().UTC().Format(time.RFC3339Nano),\n\t\t})\n\tcase http.MethodHead:", + "replace": "\t\t\tbucketCreated: time.Now().UTC().Format(time.RFC3339Nano)[:0],\n\t\t})\n\tcase http.MethodHead:", + "test": "TestADoorWithoutConfigurationStillListsAndDatesItsBuckets" + } + ], + "note": "The subject is the OOS door of #850 and the coupling that kept it from serving its client. ListBuckets lived behind Config.BucketConfiguration, and `git log -S` names the commit: 7c3e087, 'prove private Scaleway bucket lifecycle', which measured that provider's ACL, tagging and list shapes together. Opportunity, not meaning — the field's name says 'the configuration of a bucket' and it was read as 'this door may enumerate buckets'. It cost a real client: octl v0.0.32 reaches the door at exactly the canonical bare endpoint (GET /_feint/s3/outscale/eu-west-2/?x-id=ListBuckets, captured on the wire) and got 400, because the Outscale profile leaves that switch off on purpose — it also turns on discovery answers whose OOS shapes nobody has recorded. The same coupling held the creation date, so the branch without it created buckets the door's own listing then refused to show with 501 'legacy bucket listing has no measured creation metadata'. The condition was removed rather than replaced by a second flag, because a flag all three doors set to true is a constant wearing a switch's name. Mutation 2 is the mirror risk and the reason this is three mutations rather than one: a decoupling that widens the listing to any unrouted GET is as wrong as the gate was, and the test that catches both is the one asserting the CANONICAL bare endpoint." +} From 2cc2c7551eeec46712856f44a75573acc99b5b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 17:00:41 +0100 Subject: [PATCH 15/28] feat(outscale): bytes through the OOS door, out and back, compared by digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #850's "PUT bytes [...] GET bytes" for a third provider. 4096 random bytes, put by octl, downloaded by octl, sha256 identical. Random and not text, deliberately: a byte path that passes through an encoding shows nothing on printable input. ## Three wrong ways first, and they are in the comment because the next person ## will try them in the same order `api GetObject` is not the way, with or without `-o raw`: it answers the operation's JSON with `"Body": {}` — 1032 bytes of envelope and none of the object. A test comparing that output would compare metadata and call it bytes, and it would pass. `storage object download` WITHOUT an output file HANGS. Measured: the call did not return and had to be killed, which is worse than an error — a suite built on it would stall rather than fail, and a stalled suite reads as a slow one. With `-O` it writes the file and returns 0. The key is POSITIONAL on that subcommand, not `--Key` as on `api GetObject`. I had abandoned this path one tool-call short of it, after the hang, and said so. `-O` was the form I had not tried. Persisting was right and stopping would have left a #850 criterion unproven for no reason other than my own fatigue. ## Verification mise run conformance:s3:oos PASS, four gestures [] over zero buckets, the shape the account answers created and found, with a creation date refused as a failure, and the inventory is untouched 4096 bytes round-tripped, sha256 identical check / drift:check / docs:check / shapes:check pass falsify:lint / lint-shell / release:surface pass drift:check is green here for the first time: main is merged in, which carries the #860 triage of lb/v1/ZonedAPI.AttachIpamIP and TransferToIpam. #859 is the debt that closes with it, and it closed by being triaged upstream rather than refreshed away here. ## Still not claimed Multipart through this door, and any discovery shape — ACL, CORS, ownership, tagging — whose OOS answers nobody has recorded. Those switches stay off in the Outscale profile. The door is still not in the default --s3-doors. #850 is not closed: the Terraform witness remains a test harness rather than a shipped `feint run`, the dotted-bucket TLS proof with an actual client is missing, and #849's interception exceeds what exists. #853 and #858 are open. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- internal/cli/oos_conformance_test.go | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/internal/cli/oos_conformance_test.go b/internal/cli/oos_conformance_test.go index be71bd0a..1e785f81 100644 --- a/internal/cli/oos_conformance_test.go +++ b/internal/cli/oos_conformance_test.go @@ -3,7 +3,11 @@ package cli import ( + "crypto/rand" + "crypto/sha256" "encoding/json" + "os" + "path/filepath" "strings" "testing" @@ -127,4 +131,44 @@ func TestMountedOOSOfficialOctl(t *testing.T) { t.Fatalf("asking for an absent bucket changed the inventory: %v %s", err, out) } t.Logf(" ok: refused as a failure, and the inventory is untouched") + + // 4. Bytes, which is what #850 means by "PUT bytes [...] GET bytes": the + // object store holds what the client sent, and the client gets the same + // bytes back out. Random, not text, because a byte path that passes + // through a conversion shows nothing on printable input. + t.Log("- bytes through the door, out and back, compared by digest") + payload := make([]byte, 4096) + if _, err := rand.Read(payload); err != nil { + t.Fatal(err) + } + sent := filepath.Join(t.TempDir(), "payload.bin") + if err := os.WriteFile(sent, payload, 0o600); err != nil { + t.Fatal(err) + } + octl.Run(t, "--no-upgrade", "storage", "api", "PutObject", + "--Bucket", bucket, "--Key", "probe.bin", "--Body", sent) + + // -O, and it is not a convenience. `storage object download` without an + // output file HANGS rather than printing to stdout: measured, the call did + // not return and had to be killed. And `api GetObject` is not the way + // either, with or without -o raw — it answers the operation's JSON with + // `"Body": {}`, 1032 bytes of envelope and none of the object, so a test + // comparing its output would compare metadata and call it bytes. + // + // The key is POSITIONAL on this subcommand, not --Key. Three wrong forms + // preceded this one. + back := filepath.Join(t.TempDir(), "back.bin") + octl.Run(t, "--no-upgrade", "-O", back, "storage", "object", "download", + "--bucket", bucket, "probe.bin") + got, err := os.ReadFile(back) + if err != nil { + t.Fatalf("the download wrote no file: %v", err) + } + if len(got) != len(payload) { + t.Fatalf("the object came back as %d bytes, not %d", len(got), len(payload)) + } + if sha256.Sum256(got) != sha256.Sum256(payload) { + t.Fatal("the object came back with the right length and the wrong bytes") + } + t.Logf(" ok: %d bytes round-tripped, sha256 identical", len(got)) } From 6f6383d1935d3dcc6add1ef32f63702774f8b2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 17:21:33 +0100 Subject: [PATCH 16/28] docs(outscale): record two OOS discovery answers, and say which two are missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured material, no code change. The Outscale profile still declares no discovery shapes, and BucketConfiguration stays off on that door, because two of the four answers are not recorded and this corpus says so rather than letting the gap look filled. ## What is recorded GetBucketAcl, 200: an AccessControlPolicy with one FULL_CONTROL grant to the owner, so a fresh OOS bucket is private. Its XML declaration carries standalone="yes", which neither neighbour does on this answer. GetBucketCors, 404: NoSuchCORSConfiguration with NO final period. Exoscale's recorded answer to the same question HAS one, so copying that dialect would have been wrong by exactly one character. is present and empty and there is a and no — the same envelope as the NoSuchBucket recording beside this file, which means that IS OOS's error envelope generally rather than a quirk of one code. ## Why this needed the account, where NoSuchBucket did not A bucket that does not exist answers NoSuchBucket to everything, ?cors= included, so the "this configuration is absent" shape only comes from a bucket that exists and has none. And an unsigned GET on a private bucket answers AccessDenied rather than the absence. So: a signed request, therefore the client, therefore `octl -v` to see the body at all. `octl -v` was proved against the emulator's already-recorded Exoscale CORS shape FIRST, and printed the full response body. An instrument that shows nothing locally shows nothing on a paid account either, and that control cost no cloud call. ## The account One bucket, created and deleted inside one script whose cleanup runs from a trap. No object written into it. 0 buckets before, 0 after, verified twice — once by the trap and once after a pause, because OOS throttled on the second run and the script read the empty answer as a possible leak. It alerted rather than going quiet, which is the right failure. The owner's canonical ID and DisplayName are NOT in this file: DisplayName is the account number. Both are replaced by the local fixture the emulator already uses, the same redaction the Exoscale corpus states for the same reason. ## What is deliberately absent GetBucketOwnershipControls and GetBucketTagging. The first run's output filter was too narrow to capture them; the second run was refused by throttling. They are not guessed, no switch is turned on for them, and the `incomplete` field of this corpus names them so the next session does not have to rediscover what is missing. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../outscale-oos-bucket-discovery.json | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docs/evidence/outscale-oos-bucket-discovery.json diff --git a/docs/evidence/outscale-oos-bucket-discovery.json b/docs/evidence/outscale-oos-bucket-discovery.json new file mode 100644 index 00000000..358c8714 --- /dev/null +++ b/docs/evidence/outscale-oos-bucket-discovery.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "provider": "outscale", + "region": "eu-west-2", + "observed_at": "2026-11-09", + "client": "octl v0.0.32 with -v, which prints the signed request and the raw response", + "provenance": "User-authorized temporary bucket on the operator's own OOS account, created and deleted inside one script whose cleanup runs from a trap. No object was ever written into it. The account held 0 buckets before and 0 after, verified twice. Credentials never passed through the harness: octl reads ~/.osc/config.json itself and that file stays behind its deny rule. The authorization was for this measurement; a second run was NOT made, because OOS throttled after these calls and two more shapes are not worth further creations on a paid account.", + "why_signed_requests_were_necessary": "An anonymous GET cannot measure these. A bucket that does not exist answers NoSuchBucket to everything, including ?cors=, so the 'this configuration is absent' shape only comes from a bucket that exists and has none. And an unsigned GET on a private bucket answers AccessDenied rather than the absence. That is why this corpus needed the client and `-v`, where the NoSuchBucket corpus beside it needed neither.", + "instrument_control": "octl -v was proved locally FIRST, against the emulator's already-recorded Exoscale CORS shape, and it printed the full response body. An instrument that shows nothing locally shows nothing on a paid account either, and that check cost no cloud call.", + "redaction": "The owner's canonical ID and DisplayName are the account's identity and are NOT reproduced: DisplayName is the account number. Both are replaced below by the local fixture the emulator already uses. Request identifiers are kept where they show a SHAPE (OOS answers the same value in x-amz-request-id and ), and they name no resource.", + "incomplete": "Two of the four discovery answers are missing: GetBucketOwnershipControls and GetBucketTagging. The first run's output filter was too narrow to capture them, and the second run was refused by throttling. They are NOT guessed here, and the Outscale profile therefore does not declare them — BucketConfiguration stays off on that door until all four are recorded.", + "exchanges": [ + { + "operation": "GetBucketAcl", + "method": "GET", + "query": "acl=", + "status": 200, + "headers": { + "content-type": "application/xml", + "x-amz-request-id": "" + }, + "body_with_identity_replaced": "LOCAL_FIXTURE_OWNER_IDLOCAL_FIXTURE_OWNERLOCAL_FIXTURE_OWNER_IDLOCAL_FIXTURE_OWNERFULL_CONTROL", + "shape_notes": [ + "standalone=\"yes\" in the XML declaration, which neither of the other two dialects carries on this answer.", + "A single FULL_CONTROL grant to the owner, so a fresh bucket is private.", + "DisplayName is the ACCOUNT NUMBER, not a human name." + ] + }, + { + "operation": "GetBucketCors", + "method": "GET", + "query": "cors=", + "status": 404, + "headers": { + "content-type": "application/xml" + }, + "body": "NoSuchCORSConfigurationThe CORS configuration does not existREDACTED_RESPONSE_ID", + "shape_notes": [ + "NO final period on the message. Exoscale's recorded answer to the same question HAS one ('The CORS configuration does not exist.'), so copying that dialect here would be wrong by one character.", + " present and EMPTY, and a , exactly as in the NoSuchBucket envelope recorded beside this file — so that is OOS's error envelope generally, not a quirk of one code.", + "No , again consistent with the NoSuchBucket recording and unlike both neighbours." + ] + } + ] +} From 143630ca6a2f2eee644c05c234a14677e904480c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 18:09:11 +0100 Subject: [PATCH 17/28] feat(cli): feint run lands a hardcoded-endpoint client locally, and takes no hostname (#849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A client that discards the endpoint it was given leaves for the real cloud. That is #280, and it is the reason #850 cannot be closed by serving more routes. Telling an operator to export four variables and mint a CA by hand is not a fix; it is the same wall with a paragraph in front of it. `feint run -- ` runs one child with the interception scoped to it: the proxy listens on loopback, the authority lives in a directory the run creates and removes, and the four variables are set on the child alone, never on the operator's shell, /etc/hosts or a system trust store. No privilege, and no network namespace: the Terraform witness already measured that the provider honours HTTP_PROXY, HTTPS_PROXY and SSL_CERT_FILE, which is why the namespace in that test is an escape CHECK rather than the mechanism. The design decision worth reviewing is that the command takes no hostname. An earlier draft of this commit took `--forward host=target`, and it was wrong for one reason: the operator typing the hostname is the operator able to mistype it, and a mistyped cloud host is not intercepted, it is *resolved*. The run then leaves for a third party carrying whatever credentials it holds, and the failure is silent, because a request that reached the real cloud looks exactly like a request that worked. So nothing is typed. The mounted doors come from the emulator's own report (GET /_feint/s3, read and validated by fetchS3) and each door's cloud authority comes from the pack in this binary. That split is the one s3.Profile already requires — an instance fact may come from a report, a wire dialect may not — and its consequence is that the set of possible targets is the set of doors that exist. There is no spelling of this command that sends traffic off loopback. Credentials are fail-closed. The emulator accepts any signature without checking it, so real keys prove nothing here and risk everything: a credential-shaped variable is refused unless its value is one of this repository's public fakes, and the refusal names the VARIABLE, never the value, because this output lands in terminals, CI logs and bug reports. Measured, not asserted. `feint run` against a door-serving emulator: curl asking for https://s3.fr-par.scw.cloud/ receives HTTP 200 and the door's own ListAllMyBucketsResult; an undeclared host exits 000, refused rather than tunnelled; the child's exit code 7 travels; no feint-run-* directory is left. A doorless emulator is refused by naming the two flags to add. One assumption this commit had to drop, and the check is what caught it: an earlier draft required 0600 on the CA file and the check reported 0644. proxy.WriteCA writes 0644 deliberately (intercept.go:294, "a public CA, no secret; must be readable by the client process"), so the requirement was mine rather than the system's. The privacy comes from the 0700 directory, and the only real secret, the authority's private key, is never written to disk. A Chmod would have recorded the wrong assumption as a value nobody reads back; the permission is therefore verified, and the test reads it from inside the child, where it matters. /falsify: five mutations, all compiling, all biting - the credential refusal neutralised, the value printed beside the name, a door with no measured authority accepted, a doorless emulator accepted, and the signal no longer relayed. The last one carries a correction of its own test's comment, which first claimed the leftover file held the signal relay. It does not: the deferred removal runs either way, and what holds the relay is the exit code. mise run prepush: green. CLI surface 28, with the help block, both README command tables, and the changelog in both languages. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 14 + CHANGELOG.md | 13 + README.fr.md | 1 + README.md | 1 + internal/cli/cli.go | 23 +- internal/cli/run.go | 340 ++++++++++++++++++ internal/cli/run_test.go | 219 +++++++++++ internal/cli/testdata/frozen/cli.json | 237 ++++++++++++ .../run-lands-a-client-without-privilege.json | 41 +++ 9 files changed, 888 insertions(+), 1 deletion(-) create mode 100644 internal/cli/run.go create mode 100644 internal/cli/run_test.go create mode 100644 tools/falsify/specs/run-lands-a-client-without-privilege.json diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index 3547ac16..4df15d5e 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -19,6 +19,20 @@ change ni l'un ni l'autre a sa place dans `git log`. ### Ajouté +- **`feint run`** (#849) : lance une commande dont l'endpoint cloud est codé en + dur en la faisant atterrir sur cet émulateur, sans privilège, sans namespace + réseau et sans toucher au système. Elle ne prend **aucun nom d'hôte**, et + c'est la propriété de sûreté : les portes S3 montées viennent du rapport de + l'émulateur, l'autorité cloud de chaque porte vient du pack présent dans ce + binaire, donc les seules cibles possibles sont des portes qui existent. Un + opérateur qui tape un nom d'hôte est un opérateur capable de le taper de + travers, et un hôte cloud mal orthographié n'est pas intercepté mais + *résolu* : le tir part chez un tiers et ressemble à un tir réussi. Tout autre + hôte est refusé plutôt que tunnelisé, l'autorité vit dans un répertoire que le + tir crée et supprime, les quatre variables ne sont posées que sur l'enfant, et + des identifiants qui ne sont pas les faux publics du dépôt sont refusés + puisque l'émulateur ne vérifie aucune signature. Surface CLI 28. + - **Le socle de livraison Object Storage** (#850, #858) : création normale, lecture et suppression de bucket Exo, usage natif issu des octets publiés et erreurs de découverte enregistrées. L'inventaire natif est diff --git a/CHANGELOG.md b/CHANGELOG.md index 23c9ee31..491c285f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,19 @@ what this project is judged on: **a response shape a client can observe**, and ### Added +- **`feint run`** (#849): runs one command with a client whose cloud endpoint is + compiled in landing on this emulator, with no privilege, no network namespace + and no system mutation. It takes **no hostname**, and that is the safety + property: the mounted S3 doors come from the emulator's own report and each + door's cloud authority from the pack in this binary, so the only possible + targets are doors that exist. An operator typing a hostname is an operator + able to mistype it, and a mistyped cloud host is not intercepted but + *resolved* — the run leaves for a third party and looks like it worked. Every + other host is refused rather than tunnelled, the authority lives in a + directory the run creates and removes, the four variables are set on the child + alone, and credentials that are not this repository's public fakes are refused + because the emulator checks no signature. CLI surface 28. + - **Object Storage release groundwork** (#850, #858): ordinary Exo bucket creation/show/removal and native usage now use recorded default discovery errors and the actual published byte inventory. The native inventory is diff --git a/README.fr.md b/README.fr.md index cee07230..d11ee515 100644 --- a/README.fr.md +++ b/README.fr.md @@ -592,6 +592,7 @@ Terraform de qui que ce soit. | --- | --- | | `feint coverage` | la surface amont servie, déclinée ou non triée | | `feint proxy` | enregistre ce qu'un vrai client et un vrai cloud se disent, identifiants masqués | +| `feint run` | lance une commande dont l'endpoint cloud est codé en dur en la faisant atterrir sur cet émulateur, sans privilège et sans toucher au système | | `feint transcript` | lit un enregistrement : quoi servir ensuite, quelle forme, ce que l'émulateur omet | | `feint replay` | rejoue un enregistrement contre un émulateur qui tourne et rapporte, opération par opération, ce qui diverge | | `feint corpus` | rejoue les corpus committés contre un émulateur neuf et échoue sur une divergence avec ce que le vrai cloud a répondu | diff --git a/README.md b/README.md index c3b2b4ca..37c83055 100644 --- a/README.md +++ b/README.md @@ -997,6 +997,7 @@ These answer *how close to the real cloud is this*, and they are the subject of |---|---| | `feint coverage` | compare the upstream API surface with what the packs serve | | `feint proxy` | sit between a real client and a real cloud and record every exchange, credentials redacted | +| `feint run` | run one command with its compiled-in cloud endpoint landing on this emulator, without privilege and without touching the system | | `feint transcript` | read a recording: what to serve next, what a response must look like, what the emulator omits | | `feint replay` | reissue a recording at a running emulator and report, operation by operation, what diverged | | `feint corpus` | replay every committed corpus against a fresh emulator and fail on a divergence from what the real cloud answered | diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 9f2af9ec..7f25b136 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -249,7 +249,7 @@ const ( // TestTheFrozenSurfacesStillMatchTheirFixture, and a fixture regenerated // without bumping this constant fails TestASurfaceChangeDemandsItsVersionBump. // The procedure for a deliberate change is in RELEASING.md ("Frozen surfaces"). -const cliSurfaceVersion = 27 +const cliSurfaceVersion = 28 // Run executes one command and returns the process exit code. func Run(args []string, stdout, stderr io.Writer) int { @@ -276,6 +276,8 @@ func Run(args []string, stdout, stderr io.Writer) int { return probeCommand(args[2:], stdout, stderr) case "proxy": return proxyCommand(args[2:], stdout, stderr) + case "run": + return runCommand(args[2:], stdout, stderr) case "transcript": return transcriptCommand(args[2:], stdout, stderr) case "replay": @@ -528,6 +530,25 @@ Usage: --expose-to-network, which puts this port and the account behind it on the network. + feint run [--endpoint http://127.0.0.1:4599] [--addr 127.0.0.1:0] + [--unsafe-real-credentials] -- [args...] + Run one command with a client whose cloud endpoint is + compiled in landing on this emulator instead. It asks the + emulator which S3 doors it mounted, takes each door's cloud + authority from the pack in this binary, and intercepts + exactly those. No hostname is typed, which is the safety + property rather than a convenience: a mistyped cloud host + is not intercepted, it is resolved, so the run would leave + for a third party and look like it worked. Every other host + is refused rather than tunnelled. Nothing outlives the + child — the authority lives in a directory this run creates + and removes, and the variables are set on the child alone, + never on your shell, /etc/hosts or a system trust store. + Needs no privilege. Credentials that are not this + repository's public fakes are refused, because the emulator + checks no signature: real keys prove nothing here and one + wrong door would put them on the wire. + feint transcript [--shape OP [--against emu.jsonl]] [--sanitise corpus.jsonl --contract contracts/.json] [--format text|json] diff --git a/internal/cli/run.go b/internal/cli/run.go new file mode 100644 index 00000000..19f43a2d --- /dev/null +++ b/internal/cli/run.go @@ -0,0 +1,340 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/proxy" + "github.com/stephrobert/feint/internal/s3" +) + +// `feint run` lands a client whose cloud endpoint is compiled in on the local +// emulator, for one child process and nothing else (#849). +// +// ## Why a command and not documentation +// +// The measured wall: a client that discards the endpoint it was given leaves +// for the real cloud, which is #280 and the reason #850 cannot be closed by +// serving more routes. Telling an operator to export four variables and mint a +// CA by hand is not a fix; it is the same wall with a paragraph in front of it. +// +// ## Why it takes no hostname, and that is the safety property +// +// An earlier draft of this command took `--forward host=target`, and that was +// the wrong shape for one reason: the operator typing the hostname is the +// operator able to mistype it, and a mistyped cloud host is not intercepted — +// it is *resolved*, so the run leaves for a third party carrying whatever +// credentials it holds. The failure is silent, because a request that reaches +// the real cloud looks exactly like a request that worked. +// +// So nothing here is typed. The emulator is asked which doors it actually +// mounted (GET /_feint/s3, read and validated by fetchS3), and the cloud +// authority for each one comes from the pack in this binary. Both halves of the +// mapping are therefore measured rather than supplied, and the set of possible +// targets is the set of doors that exist: there is no spelling of this command +// that sends traffic off loopback. +// +// The split is also the one [s3.Profile] requires: an instance fact (which +// doors are mounted) may come from the report, a wire dialect may not. +// +// ## What makes it safe, and it is all one idea +// +// Everything is scoped to the child. The proxy listens on loopback, the CA +// lives in a directory this process creates and removes, and the four variables +// are set on the child's environment — never on the operator's shell, never in +// /etc/hosts, never in a system trust store. When `feint run` returns, the only +// trace left is the exit code it passes through. +// +// The interception is also an allow-list, and that is proxy.NewForward's rule +// rather than this file's: a host must be named to be intercepted, and an +// unnamed one is refused rather than tunnelled. #849 states it as "never +// convert an unrecognised production hostname into a local route", and the +// forward was already written that way because a proxy that intercepts +// whatever it is handed decrypts more of the operator's traffic than the +// measurement asked for. +// +// ## Why it refuses real credentials by default +// +// A client pointed at the emulator still signs its requests, and the emulator +// accepts any signature without checking it. So a run carrying REAL keys proves +// nothing the fake ones would not, and risks everything they would not. +// +// The rule is fail-closed: a credential-shaped variable is refused unless its +// value is one of the known fakes, and --unsafe-real-credentials is the only +// way past. The alternative — guessing which values look real — fails in the +// one direction that costs something. +func runCommand(args []string, stdout io.Writer, stderr io.Writer) int { + fs := newFlagSet("run") + endpoint := fs.String("endpoint", defaultEndpoint(), "the running emulator, whose mounted S3 doors decide what is intercepted. Loopback HTTP(S) only") + addr := fs.String("addr", "127.0.0.1:0", "the proxy's listen address; the default takes a free loopback port and prints it") + unsafe := fs.Bool("unsafe-real-credentials", false, "allow the child to carry credentials that are not known fakes. The emulator never checks a signature, so this buys nothing and risks sending real keys to a third party") + if err := fs.Parse(args); err != nil { + return exitError + } + command := fs.Args() + if len(command) == 0 { + fmt.Fprintln(stderr, "feint run: name a command to run, after --") + return exitError + } + + if offenders := realCredentials(os.Environ()); len(offenders) != 0 && !*unsafe { + fmt.Fprintf(stderr, "feint run: refusing to run with credentials that are not known fakes: %s\n", + strings.Join(offenders, ", ")) + fmt.Fprintln(stderr, " The emulator accepts any signature without checking it, so real keys buy") + fmt.Fprintln(stderr, " nothing here. Use the repository's public fakes, or pass") + fmt.Fprintln(stderr, " --unsafe-real-credentials if you have a reason this refusal does not cover.") + return exitError + } + + profiles, err := runProfiles(*endpoint) + if err != nil { + fmt.Fprintf(stderr, "feint run: %v\n", err) + return exitError + } + + code, err := runChild(*endpoint, *addr, profiles, command, *unsafe, stdout, stderr) + if err != nil { + fmt.Fprintf(stderr, "feint run: %v\n", err) + return exitError + } + return code +} + +// runProfiles asks the emulator which doors it mounted and pairs each with the +// cloud authority its own pack declares. +// +// The refusal on a door whose pack declares no CloudHost is deliberate and is +// not a gap to paper over: a provider with no measured authority has nothing +// this command could intercept, and inventing one would send a client to a name +// nobody recorded. TestRunRefusesADoorWithNoMeasuredAuthority holds it. +func runProfiles(endpoint string) ([]s3.Profile, error) { + report, err := fetchS3(endpoint) + if err != nil { + return nil, err + } + if !report.Enabled || len(report.Doors) == 0 { + return nil, errors.New("the emulator serves no S3 door, so there is nothing to intercept; " + + "start it with --objectstore and --s3-doors") + } + declaration := make([]string, 0, len(report.Doors)) + for _, door := range report.Doors { + declaration = append(declaration, door.Provider+"/"+door.Region) + } + packs, err := packsFor(emulator.DefaultEnv()) + if err != nil { + return nil, err + } + profiles, err := storageProfiles(packs, strings.Join(declaration, ",")) + if err != nil { + return nil, err + } + for _, profile := range profiles { + if profile.CloudHost == "" { + return nil, fmt.Errorf("the %s/%s door declares no measured cloud authority, so this command "+ + "cannot say what to intercept for it", profile.Provider, profile.Region) + } + } + return profiles, nil +} + +// credentialVariables are the names whose values this command refuses to pass +// on unless they are known fakes. Names, not patterns: a pattern like *_KEY +// would catch SSH_AUTH_SOCK's neighbours and refuse runs for no reason, and a +// name this list forgets is a name the operator can still protect by using the +// public fakes. +var credentialVariables = []string{ + "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", + "SCW_ACCESS_KEY", "SCW_SECRET_KEY", + "OSC_ACCESS_KEY", "OSC_SECRET_KEY", + "EXOSCALE_API_KEY", "EXOSCALE_API_SECRET", +} + +// knownFakes are the deliberately public placeholders this repository ships, +// and the only credential values `feint run` passes on without --unsafe. +// +// A list and not a shape test, because every shape test gets this wrong in the +// expensive direction: "twenty identical characters" would admit a real key +// that happens to repeat, and "looks random" would refuse the fakes. The cost +// of the list is that a fake it does not know is refused, which is the harmless +// direction — the operator sees exactly which variable and can use a known one. +var knownFakes = []string{ + "AAAAAAAAAAAAAAAAAAAA", + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + "SCWXXXXXXXXXXXXXXXXX", + "11111111-1111-1111-1111-111111111111", + "EXOxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", +} + +// realCredentials names the credential variables carrying something other than +// a known fake. Empty is the safe answer: no credentials set at all is a run +// the emulator serves happily, since it checks no signature. +func realCredentials(environ []string) []string { + var offenders []string + for _, pair := range environ { + name, value, ok := strings.Cut(pair, "=") + if !ok || value == "" || !slices.Contains(credentialVariables, name) { + continue + } + if !slices.Contains(knownFakes, value) { + // The NAME only. Printing the value would put a real credential in + // a terminal, a CI log and a bug report, which is the accident this + // refusal exists to prevent. + offenders = append(offenders, name) + } + } + slices.Sort(offenders) + return offenders +} + +// runChild mounts the forward, mints the CA, runs the command and removes both. +func runChild(endpoint, addr string, profiles []s3.Profile, command []string, unsafe bool, + stdout, stderr io.Writer, +) (int, error) { + // The transport is what makes a door reachable: a client asks for + // GET / on the cloud authority, and the door lives under a path prefix. + // LocalTransport rewrites that, refusing any authority not declared here. + transport, err := s3.NewLocalTransport(endpoint, profiles) + if err != nil { + return 0, err + } + hosts := make([]string, 0, len(profiles)) + for _, profile := range profiles { + hosts = append(hosts, profile.CloudHost) + } + + authority, err := proxy.MintAuthority() + if err != nil { + return 0, fmt.Errorf("mint the local authority: %w", err) + } + + // Owner-only on the directory, and nothing durable: #849 asks for both, and + // the directory is where the whole property lives. + // + // The permission is VERIFIED rather than set, and the first run of that + // check earned its keep: an earlier draft asserted 0600 on the CA file too + // and the check reported 0644, because proxy.WriteCA writes 0644 on purpose + // (intercept.go:294, "a public CA, no secret; must be readable by the + // client process"). The requirement was mine, not the system's. What + // actually makes the run private is the 0700 directory, and the only real + // secret — the authority's private key — is never written to disk at all. + // A Chmod would have written that wrong assumption down as a value nobody + // reads back, which is the defect this repository measures most often. + // TestRunKeepsItsAuthorityOwnerOnly drives it through the child, which is + // where the permissions actually matter. + dir, err := os.MkdirTemp("", "feint-run-") + if err != nil { + return 0, fmt.Errorf("create the run's private directory: %w", err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + fmt.Fprintf(stderr, "feint run: could not remove %s: %v\n", dir, err) + } + }() + caPath := filepath.Join(dir, "ca.pem") + if err := authority.WriteCA(caPath); err != nil { + return 0, fmt.Errorf("write the run's authority: %w", err) + } + info, err := os.Stat(dir) + if err != nil { + return 0, fmt.Errorf("inspect %s: %w", dir, err) + } + if mode := info.Mode().Perm(); mode&^os.FileMode(0o700) != 0 { + return 0, fmt.Errorf("%s is %#o, wider than the 0700 this run requires", dir, mode) + } + + writer := proxy.NewWriter(io.Discard, proxy.DefaultQueue) + defer func() { _ = writer.Close() }() + forward, err := proxy.NewForward(proxy.ForwardOptions{ + Hosts: hosts, Writer: writer, Authority: authority, Transport: transport, + }) + if err != nil { + return 0, fmt.Errorf("build the forward proxy: %w", err) + } + // Forward is an http.Handler, so this owns the listener. Loopback is + // checked rather than assumed: a proxy off loopback would offer this + // interception, and the child's traffic, to the network. + listener, err := net.Listen("tcp", addr) + if err != nil { + return 0, fmt.Errorf("listen on %s: %w", addr, err) + } + defer func() { _ = listener.Close() }() + if host, _, splitErr := net.SplitHostPort(listener.Addr().String()); splitErr != nil || + !net.ParseIP(host).IsLoopback() { + return 0, fmt.Errorf("the proxy must listen on loopback, not %s", listener.Addr()) + } + server := &http.Server{Handler: forward, ReadHeaderTimeout: 30 * time.Second} + defer func() { _ = server.Close() }() + go func() { _ = server.Serve(listener) }() + + proxyURL := "http://" + listener.Addr().String() + + // The receipt, before the child runs rather than after. #849 asks that + // every rewrite be inspectable; printed first, it is also inspectable by + // somebody who interrupts the run. + fmt.Fprintf(stdout, "feint run: proxy on %s, authority at %s\n", proxyURL, caPath) + for _, profile := range profiles { + fmt.Fprintf(stdout, " https://%s -> %s%s (TLS terminated with the run's own authority)\n", + profile.CloudHost, endpoint, s3.DoorPath(profile.Provider, profile.Region)) + } + fmt.Fprintln(stdout, " every other host is refused, never tunnelled") + if unsafe { + fmt.Fprintln(stdout, " --unsafe-real-credentials: the child's credentials were NOT checked") + } + + child := exec.Command(command[0], command[1:]...) //nolint:gosec // the operator's own command, explicit argv, no shell + child.Stdin, child.Stdout, child.Stderr = os.Stdin, stdout, stderr + child.Env = append(os.Environ(), + "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL, + "http_proxy="+proxyURL, "https_proxy="+proxyURL, + // The emulator itself must not go through the proxy, or a request to it + // would be intercepted by the thing forwarding to it. + "NO_PROXY=127.0.0.1,localhost", "no_proxy=127.0.0.1,localhost", + "SSL_CERT_FILE="+caPath, + // Measured on the Terraform witness: the provider honours HTTPS_PROXY + // and SSL_CERT_FILE, which is why this command needs no network + // namespace and no privilege at all. + "AWS_CA_BUNDLE="+caPath, + ) + + // A signal reaches the child and the cleanup still runs. Without this, + // Ctrl-C would leave the authority on disk, which is the leftover #849 + // names. TestRunRemovesItsAuthorityAfterASignal holds it. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(signals) + if err := child.Start(); err != nil { + return 0, fmt.Errorf("start %s: %w", command[0], err) + } + done := make(chan error, 1) + go func() { done <- child.Wait() }() + for { + select { + case sig := <-signals: + if p := child.Process; p != nil { + _ = p.Signal(sig) + } + case waitErr := <-done: + var exit *exec.ExitError + if errors.As(waitErr, &exit) { + return exit.ExitCode(), nil + } + if waitErr != nil { + return 0, fmt.Errorf("%s: %w", command[0], waitErr) + } + return 0, nil + } + } +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go new file mode 100644 index 00000000..1713c0db --- /dev/null +++ b/internal/cli/run_test.go @@ -0,0 +1,219 @@ +package cli + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/providers/scaleway" + "github.com/stephrobert/feint/internal/s3" +) + +// reportServer serves one validated S3 report over loopback, which is what +// fetchS3 requires and therefore what `feint run` discovers its doors from. +func reportServer(t *testing.T, doors ...s3.Door) *httptest.Server { + t.Helper() + report := s3.Report{SchemaVersion: s3.ReportVersion, Enabled: len(doors) != 0, + MaxBlobBytes: 1 << 20, MaxObjectBytes: 1 << 10, + Doors: doors, HTTPRequests: map[string]uint64{}} + if report.Doors == nil { + report.Doors = []s3.Door{} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("content-type", "application/json") + if err := json.NewEncoder(w).Encode(report); err != nil { + t.Errorf("encode report: %v", err) + } + })) + t.Cleanup(server.Close) + return server +} + +func door(provider, region string) s3.Door { + return s3.Door{Provider: provider, Region: region, Path: s3.DoorPath(provider, region)} +} + +// TestRunDiscoversItsDoorsAndNeverTakesAHostnameFromTheOperator is the +// accepting half: a mounted door becomes an interception whose target is the +// door's own canonical path, and nothing in that mapping was typed. +func TestRunDiscoversItsDoorsAndNeverTakesAHostnameFromTheOperator(t *testing.T) { + server := reportServer(t, door("scaleway", "fr-par"), door("exoscale", "ch-gva-2")) + profiles, err := runProfiles(server.URL) + if err != nil { + t.Fatalf("mounted doors refused: %v", err) + } + if len(profiles) != 2 { + t.Fatalf("got %d profiles, want 2", len(profiles)) + } + want := map[string]string{ + "s3.fr-par.scw.cloud": "scaleway/fr-par", + "sos-ch-gva-2.exo.io": "exoscale/ch-gva-2", + } + for _, profile := range profiles { + scope, ok := want[profile.CloudHost] + if !ok || scope != profile.Provider+"/"+profile.Region { + t.Fatalf("authority %q is not the one the pack declares for %s/%s", + profile.CloudHost, profile.Provider, profile.Region) + } + } + // The transport the command builds must accept exactly these, which is the + // property that makes an off-loopback target unspellable: the only targets + // are doors the emulator reported. + if _, err := s3.NewLocalTransport(server.URL, profiles); err != nil { + t.Fatalf("discovered profiles rejected by the local transport: %v", err) + } +} + +// TestRunRefusesAnEmulatorWithNoDoorAndSaysWhatToStart keeps the refusal +// actionable: an operator who forgot --objectstore reads what to add, not that +// something is unavailable. +func TestRunRefusesAnEmulatorWithNoDoorAndSaysWhatToStart(t *testing.T) { + server := reportServer(t) + _, err := runProfiles(server.URL) + if err == nil { + t.Fatal("a doorless emulator was accepted") + } + for _, mention := range []string{"--objectstore", "--s3-doors"} { + if !strings.Contains(err.Error(), mention) { + t.Fatalf("the refusal does not name %s: %v", mention, err) + } + } +} + +// authorityLessPack is a Scaleway pack that declares no measured cloud +// authority, which is the one case runProfiles must refuse rather than guess. +type authorityLessPack struct{ emulator.Pack } + +func (authorityLessPack) ObjectStorage(region string) s3.Profile { + return s3.Profile{Provider: "scaleway", Region: region} +} + +// TestRunRefusesADoorWithNoMeasuredAuthority is cited by runProfiles. Without +// its check, the command would hand s3.NewLocalTransport an empty authority and +// fail with a message about transports rather than about the door. +func TestRunRefusesADoorWithNoMeasuredAuthority(t *testing.T) { + previous := packsFor + packsFor = func(env *emulator.Env) ([]emulator.Pack, error) { + return []emulator.Pack{authorityLessPack{Pack: scaleway.New(env)}}, nil + } + t.Cleanup(func() { packsFor = previous }) + + server := reportServer(t, door("scaleway", "fr-par")) + _, err := runProfiles(server.URL) + if err == nil { + t.Fatal("a door with no measured authority was accepted") + } + if !strings.Contains(err.Error(), "scaleway/fr-par") { + t.Fatalf("the refusal does not name the door: %v", err) + } +} + +// TestRunRefusesCredentialsThatAreNotKnownFakesByNameOnly holds both halves of +// the fail-closed rule, and the one that matters most: the NAME is reported and +// the value never is, because this output lands in terminals and bug reports. +func TestRunRefusesCredentialsThatAreNotKnownFakesByNameOnly(t *testing.T) { + const secret = "AKIAREALLOOKINGKEY42" + offenders := realCredentials([]string{ + "PATH=/usr/bin", + "SCW_ACCESS_KEY=" + knownFakes[2], + "AWS_SECRET_ACCESS_KEY=", + "OSC_ACCESS_KEY=" + secret, + }) + if len(offenders) != 1 || offenders[0] != "OSC_ACCESS_KEY" { + t.Fatalf("got %v, want only OSC_ACCESS_KEY", offenders) + } + if none := realCredentials([]string{"SCW_SECRET_KEY=" + knownFakes[3]}); len(none) != 0 { + t.Fatalf("a known fake was refused: %v", none) + } + + var stdout, stderr bytes.Buffer + t.Setenv("OSC_ACCESS_KEY", secret) + if code := runCommand([]string{"--", "true"}, &stdout, &stderr); code != exitError { + t.Fatalf("the run was not refused: code %d", code) + } + if strings.Contains(stderr.String(), secret) || strings.Contains(stdout.String(), secret) { + t.Fatal("the credential value was printed") + } + if !strings.Contains(stderr.String(), "OSC_ACCESS_KEY") { + t.Fatalf("the refusal does not name the variable: %q", stderr.String()) + } +} + +// TestRunRemovesItsAuthorityAfterASignal is cited by runChild. The child sends +// SIGTERM to this test process, which is the real path: runChild arms +// signal.Notify before starting the child, so the signal is relayed rather than +// killing the test, and the deferred cleanup is what this asserts. +// +// Which half catches which, because getting this backwards is how a test comes +// to be believed for the wrong reason: the deferred removal would run even +// without the relay, so the file check does NOT hold the relay. What holds it +// is the exit code — a child that never receives the signal sleeps out and +// reports success. The file check holds the cleanup, including on this path. +func TestRunRemovesItsAuthorityAfterASignal(t *testing.T) { + server := reportServer(t, door("scaleway", "fr-par")) + profiles, err := runProfiles(server.URL) + if err != nil { + t.Fatalf("discovery failed: %v", err) + } + var stdout, stderr bytes.Buffer + // `kill -TERM $PPID` targets this test process; runChild relays it to the + // child, which then dies on it rather than sleeping out the timeout. + code, err := runChild(server.URL, "127.0.0.1:0", profiles, + []string{"sh", "-c", "kill -TERM $PPID; sleep 30"}, false, &stdout, &stderr) + if err != nil { + t.Fatalf("run failed: %v", err) + } + if code == 0 { + t.Fatal("the child reported success after being terminated") + } + + // The receipt names the authority, which is how its removal is observable + // at all — the directory is private to the run. + var authority string + for _, line := range strings.Split(stdout.String(), "\n") { + if _, after, ok := strings.Cut(line, "authority at "); ok { + authority = strings.TrimSpace(after) + } + } + if authority == "" { + t.Fatalf("the receipt does not name the authority: %q", stdout.String()) + } + if _, err := os.Stat(authority); !os.IsNotExist(err) { + t.Fatalf("the authority survived the signal at %s: %v", authority, err) + } +} + +// TestRunKeepsItsAuthorityOwnerOnly is cited by runChild, and it asks the +// question from where it matters: inside the child, through the variable the +// child was actually given. A test that stat'd a path this test computed would +// be checking its own arithmetic. +func TestRunKeepsItsAuthorityOwnerOnly(t *testing.T) { + server := reportServer(t, door("scaleway", "fr-par")) + profiles, err := runProfiles(server.URL) + if err != nil { + t.Fatalf("discovery failed: %v", err) + } + var stdout, stderr bytes.Buffer + code, err := runChild(server.URL, "127.0.0.1:0", profiles, + []string{"sh", "-c", `stat -c '%a %n' "$SSL_CERT_FILE" "$(dirname "$SSL_CERT_FILE")"`}, + false, &stdout, &stderr) + if err != nil || code != 0 { + t.Fatalf("the child could not read its own authority: code %d, %v, %q", code, err, stderr.String()) + } + // Two properties, and they pull in opposite directions, which is why both + // are asserted: the directory must be owner-only, and the CA inside it must + // still be READABLE by the client process. proxy.WriteCA writes 0644 for + // that second reason, deliberately; the privacy comes from the directory, + // and the authority's private key is never written at all. + if !strings.Contains(stdout.String(), "700 ") { + t.Fatalf("the child saw a directory mode other than 700: %q", stdout.String()) + } + if !strings.Contains(stdout.String(), "ca.pem") { + t.Fatalf("the child could not name its own authority: %q", stdout.String()) + } +} diff --git a/internal/cli/testdata/frozen/cli.json b/internal/cli/testdata/frozen/cli.json index ae60d5df..82d916a4 100644 --- a/internal/cli/testdata/frozen/cli.json +++ b/internal/cli/testdata/frozen/cli.json @@ -4919,6 +4919,243 @@ ] } } + }, + { + "schema_version": 28, + "content": { + "exit_codes": { + "drift": 2, + "error": 1, + "ok": 0 + }, + "verbs": { + "catalog": [ + "--format" + ], + "clean": [ + "--check", + "--closing", + "--doorstep", + "--force", + "--format", + "--vm" + ], + "corpus": [ + "--accepted", + "--against-cloud", + "--bind", + "--check", + "--credential", + "--dir", + "--dry-run", + "--endpoint", + "--file", + "--format", + "--mark-stale", + "--timeout" + ], + "coverage": [ + "--artefact", + "--axis", + "--baseline", + "--contract", + "--contract-only", + "--document", + "--evidence", + "--fail-on-unknown", + "--format", + "--gaps", + "--observed", + "--products", + "--provider", + "--sdk", + "--write-baseline" + ], + "docs": [ + "--check", + "--client-pins", + "--confidence", + "--contracts", + "--coverage", + "--file", + "--install", + "--limits", + "--proved", + "--routes", + "--screenshots", + "--ui-manifest", + "--workflow" + ], + "doctor": [ + "--addr", + "--s3", + "--vm" + ], + "down": [ + "--file", + "--keep-emulator" + ], + "env": [ + "--client", + "--endpoint", + "--s3", + "--s3-region", + "--shell", + "--unset" + ], + "evidence": [ + "--allow-narrowing", + "--contracts", + "--endpoint", + "--join", + "--out", + "--reshape", + "--shapes", + "--suites" + ], + "evidence baseline": [ + "--axes", + "--evidence", + "--out" + ], + "evidence verify": [ + "--accepted", + "--baseline", + "--evidence" + ], + "images": [ + "--check", + "--only", + "--vm" + ], + "logs": [ + "--addr", + "-f", + "-n" + ], + "probe": [ + "--contracts", + "--endpoint", + "--provider" + ], + "proxy": [ + "--addr", + "--expose-to-network", + "--forward", + "--intercept", + "--max-body", + "--provider", + "--queue", + "--record", + "--upstream" + ], + "replay": [ + "--endpoint", + "--format", + "--refusals-only", + "--timeout" + ], + "restart": [ + "--addr", + "--timeout" + ], + "run": [ + "--addr", + "--endpoint", + "--unsafe-real-credentials" + ], + "serve": [ + "--addr", + "--cleanup", + "--consistency", + "--contracts", + "--coverage", + "--expose-to-network", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-host-doors", + "--s3-object-limit-mib", + "--shapes", + "--state", + "--strict-catalog", + "--vm" + ], + "shapes": [ + "--check", + "--dir", + "--dry-run", + "--profile", + "--provider", + "--record" + ], + "snapshot list": [ + "--format" + ], + "snapshot load": [ + "--addr" + ], + "snapshot save": [ + "--addr", + "--force" + ], + "start": [ + "--addr", + "--cleanup", + "--contracts", + "--detach", + "--foreground", + "--log-level", + "--objectstore", + "--objectstore-budget-mib", + "--projects", + "--resolver", + "--s3-doors", + "--s3-host-doors", + "--s3-object-limit-mib", + "--state", + "--strict-catalog", + "--timeout", + "--vm" + ], + "status": [ + "--addr", + "--coverage", + "--format" + ], + "stop": [ + "--addr", + "--timeout" + ], + "transcript": [ + "--against", + "--contract", + "--format", + "--sanitise", + "--shape" + ], + "ui": [ + "--addr", + "--print" + ], + "up": [ + "--file", + "--no-iac", + "--runtime", + "--timeout" + ], + "version": [ + "--check" + ], + "wait": [ + "--addr", + "--timeout" + ] + } + } } ] } diff --git a/tools/falsify/specs/run-lands-a-client-without-privilege.json b/tools/falsify/specs/run-lands-a-client-without-privilege.json new file mode 100644 index 00000000..762b46f3 --- /dev/null +++ b/tools/falsify/specs/run-lands-a-client-without-privilege.json @@ -0,0 +1,41 @@ +{ + "package": "./internal/cli/", + "mutations": [ + { + "label": "the fail-closed credential refusal is neutralised, so a run carrying the operator's real keys proceeds: the emulator checks no signature, so those keys buy nothing and one wrong door would put them on the wire to a third party", + "file": "internal/cli/run.go", + "find": "\tif offenders := realCredentials(os.Environ()); len(offenders) != 0 && !*unsafe {", + "replace": "\tif offenders := realCredentials(os.Environ()); len(offenders) != 0 && !*unsafe && false {", + "test": "TestRunRefusesCredentialsThatAreNotKnownFakesByNameOnly" + }, + { + "label": "the refusal reports the VALUE beside the name, which is the accident the refusal exists to prevent: this output lands in a terminal, a CI log and a bug report, so a credential printed once is a credential disclosed everywhere that text travels", + "file": "internal/cli/run.go", + "find": "\t\t\toffenders = append(offenders, name)", + "replace": "\t\t\toffenders = append(offenders, name+\"=\"+value)", + "test": "TestRunRefusesCredentialsThatAreNotKnownFakesByNameOnly" + }, + { + "label": "a door whose pack declares no measured cloud authority is accepted, so the command builds its interception around an empty hostname and fails talking about transports instead of about the door the operator mounted", + "file": "internal/cli/run.go", + "find": "\t\tif profile.CloudHost == \"\" {", + "replace": "\t\tif profile.CloudHost == \"\" && false {", + "test": "TestRunRefusesADoorWithNoMeasuredAuthority" + }, + { + "label": "a doorless emulator is accepted, so the operator who forgot --objectstore reads an error about door tokens rather than the two flags to add. The refusal's job is not to stop the run — storageProfiles would stop it anyway — it is to say what to start", + "file": "internal/cli/run.go", + "find": "\tif !report.Enabled || len(report.Doors) == 0 {", + "replace": "\tif (!report.Enabled || len(report.Doors) == 0) && false {", + "test": "TestRunRefusesAnEmulatorWithNoDoorAndSaysWhatToStart" + }, + { + "label": "the signal stops being relayed to the child, so Ctrl-C leaves the client running under a proxy whose process is going away. The test catches this on the EXIT CODE, not on the leftover file: the deferred removal runs either way, and a child that never receives the signal sleeps out and reports success", + "file": "internal/cli/run.go", + "find": "\t\t\t\t_ = p.Signal(sig)", + "replace": "\t\t\t\tif false {\n\t\t\t\t\t_ = p.Signal(sig)\n\t\t\t\t}", + "test": "TestRunRemovesItsAuthorityAfterASignal" + } + ], + "note": "The subject is `feint run` (#849): landing a client whose cloud endpoint is compiled in on the local emulator, with no sudo, no network namespace and no system mutation. Measured, not assumed: the published Scaleway Terraform provider honours HTTP_PROXY, HTTPS_PROXY and SSL_CERT_FILE, which is why the witness in object_storage_terraform_conformance_test.go uses its namespace as an ESCAPE CHECK rather than as the mechanism. The command therefore needs nothing privileged.\n\nThe design decision these mutations guard is that the command takes NO hostname. An earlier draft took `--forward host=target`, and the operator typing the hostname is the operator able to mistype it — a mistyped cloud host is not intercepted, it is resolved, so the run leaves for a third party carrying whatever credentials it holds, and the failure is silent because a request that reached the real cloud looks exactly like one that worked. Instead the mounted doors come from the emulator (GET /_feint/s3, validated by fetchS3) and each door's cloud authority comes from the pack in this binary, which is also the split s3.Profile requires: an instance fact may come from a report, a wire dialect may not. The set of possible targets is therefore the set of doors that exist, and there is no spelling of this command that sends traffic off loopback.\n\nMutation 5 carries the correction of a comment this file got wrong first: it claimed the file check held the signal relay. It does not, because the deferred removal runs regardless. Writing that down was necessary — a test believed for the wrong reason is the defect this repository measures most often." +} From 876c58c20940f9a00b1aa30bb7cf957d44d27586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 18:37:34 +0100 Subject: [PATCH 18/28] feat(cli): Terraform reaches the local door through feint run, with no privilege (#849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This closes #849's own "done means", which curl could not: a real Terraform client with a compiled-in Object Storage endpoint, driven through the command on the HOST network, with no sudo, no namespace and no change to the machine. TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork is the witness, and it is the mirror of TestScalewayTerraformStorageInIsolatedNetwork rather than a copy of it. That one REFUSES to run outside a fresh namespace and needs `sudo unshare`; this one refuses to run INSIDE one, because a pass obtained in a namespace would prove the namespace's containment and not the command's. The two are only worth having together. Measured on the published provider 2.82.0: apply complete, 139,264 identical bytes read back from the local byte store by sha256, an empty second plan, a complete teardown, inventory back to empty. The digest is the proof rather than the exit code — a request that went anywhere else cannot produce it, and the real cloud would refuse these deliberately public fake credentials long before storing anything. Three defects the measurement found in the command itself. 1. The virtual-host form was not intercepted. A bucket is addressed as ., which is what Terraform uses, so the door was reachable by my own curl proof — path-style — and by nothing a real client does. The allow-list now carries a single-level wildcard per door. It widens only what is INTERCEPTED: LocalTransport still refuses a leading label that is not a valid bucket name, so interception stays wider than service and a nonsense bucket gets a local refusal rather than a trip to the cloud. 2. AWS_CA_BUNDLE broke the client it was meant to help. I had set it on the reasoning that an S3 client would want it; the apply then died on "unable to add custom RootCAs HTTPClient, has no WithTransportOptions, *http.Client", because the provider hands its SDK a prebuilt client. Removing it made the same apply complete. It is now stripped from an inherited environment too, which is the one thing this command edits of the operator's own config, and the reason is that no value of it can help: the station's bundle cannot contain an authority minted seconds ago. SSL_CERT_FILE, which crypto/x509 reads, covers every Go client. 3. My own test raced its subject. awaitReceipt polls stdout while runChild's goroutine appends to it; -race caught it, and the buffer is now guarded. `feint doctor` gains the leftover detection #849 asks for, and it decides rather than warns. Each run records its owning PID, so a live owner reads as a run in flight and a dead one as a leftover. Without that, the check could only report "a directory exists", which is true of every healthy run — the kind of diagnostic doctor.go's own header says people stop reading. A leftover is a warn, not a fail: it holds a public certificate in an owner-only directory, so it leaks nothing and blocks nothing, unlike checkLeftoverDHCP whose leftover kills the next run. Two harness lessons worth keeping, both from believing a green. The witness passed in 0.94s, which is implausible for init+apply+plan+destroy, so I timed each step instead of accepting it. The timings turned out fine (46ms and 263ms: the provider is linked, not unpacked, and the emulator is local), but reading them surfaced a real defect — the witness was green once and red the second time, because TF_PLUGIN_CACHE_DIR pointed at the same directory as -plugin-dir, so the first run unpacked into the mirror and every later run hit "cannot install existing provider directory to itself". A witness that only works on a machine that has never run it proves nothing. The timings stay in the test for the next reader. And my first two explanations for 2 leftover directories were both WRONG, which measuring said and guessing would not have. Neither SIGPIPE nor AWS_CA_BUNDLE caused them: my script had no before-count. With one, the Terraform cycle's three runs leave an delta of exactly 0, and the two directories turned out to be residue from the falsify mutation that forgets cleanup — by construction. /falsify: nine mutations, all compiling, all biting. Mutation 6 is the one #849 names in as many words, a run that forgets its cleanup, and its first draft stayed GREEN: `if err := os.RemoveAll(dir); err != nil && false` still CALLS RemoveAll. It removes a path inside the directory instead. Mutation 8 holds the virtual-host interception, and it is why that test had to cross the proxy: an earlier draft asserted the receipt text and the transport's routing, and neither touches the allow-list. mise run prepush: green, including -race. docs:check green. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.fr.md | 26 ++ CHANGELOG.md | 25 ++ docs/object-storage-routing.md | 50 +++ internal/cli/doctor.go | 85 ++++ internal/cli/run.go | 89 ++++- .../cli/run_terraform_conformance_test.go | 219 +++++++++++ internal/cli/run_test.go | 371 ++++++++++++++++++ .../run-lands-a-client-without-privilege.json | 30 +- 8 files changed, 884 insertions(+), 11 deletions(-) create mode 100644 internal/cli/run_terraform_conformance_test.go diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index 4df15d5e..de8d01b0 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -33,6 +33,32 @@ change ni l'un ni l'autre a sa place dans `git log`. des identifiants qui ne sont pas les faux publics du dépôt sont refusés puisque l'émulateur ne vérifie aucune signature. Surface CLI 28. + Prouvé avec le client, pas avec curl : le provider Scaleway publié 2.82.0 + termine son apply à travers `feint run` sur le **réseau de l'hôte**, sans sudo + et sans namespace, et **139 264 octets identiques** sont relus depuis + l'octet-store local par sha256, suivis d'un second plan vide et d'une + destruction complète. `TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork` + est ce témoin, et il refuse de tourner dans un namespace : un succès obtenu + là-dedans prouverait le confinement du namespace, pas celui de la commande. + + Deux limites que la mesure a déplacées. La forme virtual-host + `.` est interceptée en plus de la forme nue, car une sonde + en path-style passe alors que toute requête de bucket d'un vrai client serait + refusée ; l'élargissement ne porte que sur ce qui est intercepté, le transport + refusant toujours un premier label qui n'est pas un nom de bucket valide. Et + **`AWS_CA_BUNDLE` est retiré au lieu d'être posé**, y compris d'un + environnement hérité : le provider échoue sur + `unable to add custom RootCAs HTTPClient, has no WithTransportOptions` + lorsqu'elle est présente, et `SSL_CERT_FILE` couvre déjà tout client Go. + +- **`feint doctor` signale les autorités d'interception restées en place** + (#849), et distingue un reste d'un tir en cours grâce au PID que chaque tir + enregistre. Un tir en cours possède légitimement un tel répertoire, donc un + contrôle qui se bornerait à signaler leur existence avertirait à chaque usage + correct. Un reste est un avertissement et non un échec : il contient un + certificat public dans un répertoire lisible par son seul propriétaire, donc + il ne fuit rien et ne bloque rien. + - **Le socle de livraison Object Storage** (#850, #858) : création normale, lecture et suppression de bucket Exo, usage natif issu des octets publiés et erreurs de découverte enregistrées. L'inventaire natif est diff --git a/CHANGELOG.md b/CHANGELOG.md index 491c285f..e23a456c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,31 @@ what this project is judged on: **a response shape a client can observe**, and alone, and credentials that are not this repository's public fakes are refused because the emulator checks no signature. CLI surface 28. + Proved with the client, not with curl: the published Scaleway provider 2.82.0 + completes apply through `feint run` on the **host network**, with no sudo and + no namespace, and **139,264 identical bytes** are read back from the local + byte store by sha256, followed by an empty second plan and a complete + teardown. `TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork` is + that witness, and it refuses to run inside a namespace, because a pass + obtained in one would prove the namespace's containment rather than the + command's. + + Two limits the measurement moved. The virtual-host form `.` + is intercepted as well as the bare one, since a path-style probe passes while + every bucket request a real client makes would be refused; the widening only + affects what is intercepted, as the transport still refuses a leading label + that is not a valid bucket name. And **`AWS_CA_BUNDLE` is stripped rather than + set**, including from an inherited environment: the provider fails its apply + with `unable to add custom RootCAs HTTPClient, has no WithTransportOptions` + when it is present, and `SSL_CERT_FILE` already covers every Go client. + +- **`feint doctor` reports leftover interception authorities** (#849), and tells + a leftover from a run in flight by the owning PID each run records. A run in + progress legitimately owns such a directory, so a check that only reported + their existence would warn on every correct use. A leftover is a warning, not + a failure: it holds a public certificate in an owner-only directory, so it + leaks nothing and blocks nothing. + - **Object Storage release groundwork** (#850, #858): ordinary Exo bucket creation/show/removal and native usage now use recorded default discovery errors and the actual published byte inventory. The native inventory is diff --git a/docs/object-storage-routing.md b/docs/object-storage-routing.md index aba45a8d..0c9bdcc8 100644 --- a/docs/object-storage-routing.md +++ b/docs/object-storage-routing.md @@ -473,6 +473,56 @@ sudo -n unshare --net -- sh -c \ rm "$testbin" ``` +## The mirror witness: the same trip with no privilege + +`TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork` is the other half, +and the two are only worth having together. It **refuses to run inside a +namespace**, because a pass obtained in one would prove nothing about the +command: the namespace itself would be doing the containment. It answers the +question #849 actually asks, which is whether an operator can get there on the +host network, with no privilege and no change to the machine. + +It drives `feint run`, which takes no hostname. The mounted doors come from the +emulator's own report and each door's cloud authority from the pack in the +binary, so the only possible targets are doors that exist. Run it against a +provider mirror, never a plugin cache at the same path: + +```bash +mkdir -p /tmp/tf-mirror /tmp/tf-seed +printf 'terraform {\n required_providers {\n scaleway = {\n source = "scaleway/scaleway"\n version = "= 2.82.0"\n }\n }\n}\n' > /tmp/tf-seed/main.tf +(cd /tmp/tf-seed && terraform providers mirror -platform=linux_amd64 /tmp/tf-mirror) +FEINT_TF_PLUGIN_DIR=/tmp/tf-mirror go test -tags=conformance ./internal/cli \ + -run '^TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork$' -count=1 -v +``` + +**Not `TF_PLUGIN_CACHE_DIR` pointing at that same directory.** The first run +then unpacks into the mirror and every later run fails with `cannot install +existing provider directory to itself`. This witness was green once and red the +second time for exactly that reason, which is a witness that proves nothing on a +machine that has already run it. + +Measured on the published provider 2.82.0, host network, no sudo: apply +complete, **139,264 identical bytes** read back from the local byte store by +sha256, an empty second plan, a complete teardown, and the bucket inventory back +to empty. The byte digest is the proof rather than the apply's exit code: a +request that went anywhere else cannot produce it, and the real cloud would +refuse these deliberately public fake credentials long before storing anything. + +Two findings the command carries from that measurement: + +- **`AWS_CA_BUNDLE` is stripped, not set.** An earlier draft set it, reasoning + that an S3 client would want it; the provider then failed its apply with + `unable to add custom RootCAs HTTPClient, has no WithTransportOptions, + *http.Client`, because it hands its SDK a prebuilt client. `SSL_CERT_FILE`, + which `crypto/x509` reads, covers every Go client. The variable is also + removed from an inherited environment, since the station's own bundle cannot + contain a freshly minted authority. +- **The virtual-host form must be intercepted too.** A path-style probe passes + while every `.` request, the form Terraform uses, is + refused and never tunnelled. The allow-list therefore carries a single-level + wildcard per door, and it only widens what is intercepted: `LocalTransport` + still refuses a leading label that is not a valid bucket name. + The witness checks an unreachable external address, apply, an empty second plan, **122,880 identical bytes**, byte-store close/reopen, another empty plan, destroy and explicit owned GC. This is the storage-shaped bucket/object graph, diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 1eff87fe..6671ee94 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -3,6 +3,7 @@ package cli import ( "bufio" "context" + "errors" "fmt" "io" "net" @@ -12,6 +13,7 @@ import ( "regexp" "strconv" "strings" + "syscall" "time" "github.com/stephrobert/feint/internal/core/machine" @@ -59,6 +61,7 @@ func doctor(args []string, stdout, stderr io.Writer) int { } checks = append(checks, checkRuntime(ctx, *vm)...) checks = append(checks, checkLeftoverDHCP()) + checks = append(checks, checkLeftoverRunAuthorities()) checks = append(checks, checkClients()...) checks = append(checks, checkEnvHazards()...) checks = append(checks, checkStackHazards()...) @@ -592,3 +595,85 @@ func firstLine(s string) string { } return strings.TrimSpace(s) } + +// checkLeftoverRunAuthorities reports the interception authorities `feint run` +// left on disk, which is the leftover detection #849 asks this command for. +// +// The verdict distinguishes a leftover from a run in flight, and that is the +// whole design. A healthy `feint run` owns such a directory for as long as its +// child lives, so reporting "a directory exists" would warn on every correct +// use and decide nothing — the kind of diagnostic this file's own header says +// people stop running. So each directory records its owning PID, and this reads +// it back: a live owner is reported as a run in progress, a dead one as a +// leftover to remove. +// +// A leftover is a warn and not a fail, deliberately: the only thing it holds is +// a public certificate inside an owner-only directory, so it leaks nothing and +// blocks nothing. What it costs is disk and puzzlement, which is a warning's +// job. The contrast with checkLeftoverDHCP is the point of both: that one fails +// because the next run DIES on the address block it holds. +// +// TestDoctorTellsALeftoverAuthorityFromARunInFlight fails without this. +func checkLeftoverRunAuthorities() check { + entries, err := os.ReadDir(os.TempDir()) + if err != nil { + return check{title: "could not look for leftover interception authorities", + state: verdictWarn, detail: err.Error()} + } + var stale, live []string + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), runDirPrefix) { + continue + } + path := filepath.Join(os.TempDir(), entry.Name()) + if runOwnerAlive(path) { + live = append(live, entry.Name()) + continue + } + stale = append(stale, entry.Name()) + } + switch { + case len(stale) == 0 && len(live) == 0: + return check{title: "no interception authority of `feint run` outlives its command", + state: verdictOK, + detail: "checked: " + runDirPrefix + "* directories under " + os.TempDir() + "; a run of another user is not visible here, and none is readable by this one"} + case len(stale) == 0: + return check{title: fmt.Sprintf("%d `feint run` command(s) in flight own an interception authority", len(live)), + state: verdictOK, + detail: strings.Join(live, "; ") + " — each belongs to a live process and goes when its command returns"} + default: + detail := strings.Join(stale, "; ") + if len(live) != 0 { + detail += fmt.Sprintf(" (and %d more owned by a live run, left alone)", len(live)) + } + return check{title: fmt.Sprintf("%d interception authority directory(ies) outlived their `feint run`", len(stale)), + state: verdictWarn, + detail: detail, + fix: "each holds a public certificate in an owner-only directory, so it leaks nothing; remove them with `rm -rf " + filepath.Join(os.TempDir(), runDirPrefix) + "*` once no run is in flight"} + } +} + +// runOwnerAlive answers whether the process that created this directory is +// still running. An unreadable or malformed owner file counts as DEAD, which is +// the useful direction: the alternative would hide a leftover written by an +// older version behind a missing file, and a leftover reported wrongly as live +// is a leftover nobody removes. +func runOwnerAlive(dir string) bool { + raw, err := os.ReadFile(filepath.Join(dir, runOwnerFile)) //nolint:gosec // a path this command composed from TempDir and its own prefix + if err != nil { + return false + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || pid <= 0 { + return false + } + process, err := os.FindProcess(pid) + if err != nil { + return false + } + // Signal 0 asks the kernel the question without delivering anything, and is + // the portable way to do it. A process this user may not signal answers + // EPERM rather than ESRCH, and that is still alive. + err = process.Signal(syscall.Signal(0)) + return err == nil || errors.Is(err, os.ErrPermission) +} diff --git a/internal/cli/run.go b/internal/cli/run.go index 19f43a2d..a26c462a 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -149,6 +149,15 @@ func runProfiles(endpoint string) ([]s3.Profile, error) { return profiles, nil } +// runDirPrefix and runOwnerFile are shared with `feint doctor`, which is the +// only reason they are constants: a diagnostic that hunts for leftovers by +// re-spelling the producer's naming scheme is a diagnostic that goes blind the +// day the scheme changes, and nothing fails to say so. +const ( + runDirPrefix = "feint-run-" + runOwnerFile = "owner.pid" +) + // credentialVariables are the names whose values this command refuses to pass // on unless they are known fakes. Names, not patterns: a pattern like *_KEY // would catch SSH_AUTH_SOCK's neighbours and refuse runs for no reason, and a @@ -199,6 +208,31 @@ func realCredentials(environ []string) []string { return offenders } +// withoutCABundle drops AWS_CA_BUNDLE from an inherited environment. +// +// Removing rather than emptying, because an empty value is not an absent one: +// a client that reads the variable would be asked to load a bundle from "", +// which is a different failure with the same cause. +// +// This is the one place the command edits what the operator already had, and +// the reason is that inheriting it cannot help under any value. Pointing at the +// station's own bundle makes a client reject this run's authority, since that +// bundle does not contain it; pointing anywhere else is no better. Measured on +// the published Scaleway provider 2.82.0, the failure is not even a TLS error +// but "unable to add custom RootCAs HTTPClient, has no WithTransportOptions, +// *http.Client", which sends the reader looking at the wrong layer entirely. +// SSL_CERT_FILE, which crypto/x509 reads, covers every Go client without it. +func withoutCABundle(environ []string) []string { + kept := make([]string, 0, len(environ)) + for _, pair := range environ { + if name, _, ok := strings.Cut(pair, "="); ok && name == "AWS_CA_BUNDLE" { + continue + } + kept = append(kept, pair) + } + return kept +} + // runChild mounts the forward, mints the CA, runs the command and removes both. func runChild(endpoint, addr string, profiles []s3.Profile, command []string, unsafe bool, stdout, stderr io.Writer, @@ -210,9 +244,24 @@ func runChild(endpoint, addr string, profiles []s3.Profile, command []string, un if err != nil { return 0, err } - hosts := make([]string, 0, len(profiles)) + // Two authorities per door, and the second one is not a convenience: S3 + // clients address a bucket as ., which is the form + // Terraform uses, so a door declared only by its bare authority is a door + // no real client reaches. A path-style check does NOT catch this, and did + // not — the first proof of this command used curl against the bare host and + // passed while every bucket request would have been refused. + // + // The single-level wildcard is the forward's own feature (forward.go:69) and + // it is as wide as a certificate's: one label, never two. What keeps it + // honest is that the wildcard only decides what is INTERCEPTED; where each + // request then goes is LocalTransport's decision, and it re-derives the + // bucket from the authority and refuses a name that is not a valid bucket + // (local_transport.go, "invalid virtual-host bucket"). So widening the + // allow-list here cannot widen what is served. + // TestRunInterceptsBucketsAddressedAsSubdomains holds it. + hosts := make([]string, 0, 2*len(profiles)) for _, profile := range profiles { - hosts = append(hosts, profile.CloudHost) + hosts = append(hosts, profile.CloudHost, "*."+profile.CloudHost) } authority, err := proxy.MintAuthority() @@ -234,7 +283,7 @@ func runChild(endpoint, addr string, profiles []s3.Profile, command []string, un // reads back, which is the defect this repository measures most often. // TestRunKeepsItsAuthorityOwnerOnly drives it through the child, which is // where the permissions actually matter. - dir, err := os.MkdirTemp("", "feint-run-") + dir, err := os.MkdirTemp("", runDirPrefix) if err != nil { return 0, fmt.Errorf("create the run's private directory: %w", err) } @@ -247,6 +296,15 @@ func runChild(endpoint, addr string, profiles []s3.Profile, command []string, un if err := authority.WriteCA(caPath); err != nil { return 0, fmt.Errorf("write the run's authority: %w", err) } + // The owning PID, so `feint doctor` can tell a LEFTOVER from a run in + // flight. Without it the diagnostic #849 asks for could only ever say "a + // directory exists", which is true of every healthy run and therefore + // decides nothing — a warning nobody can act on is the kind of diagnostic + // people stop reading, and doctor.go says so in its own header. + if err := os.WriteFile(filepath.Join(dir, runOwnerFile), + []byte(fmt.Sprintln(os.Getpid())), 0o600); err != nil { + return 0, fmt.Errorf("record the run's owner: %w", err) + } info, err := os.Stat(dir) if err != nil { return 0, fmt.Errorf("inspect %s: %w", dir, err) @@ -286,8 +344,8 @@ func runChild(endpoint, addr string, profiles []s3.Profile, command []string, un // somebody who interrupts the run. fmt.Fprintf(stdout, "feint run: proxy on %s, authority at %s\n", proxyURL, caPath) for _, profile := range profiles { - fmt.Fprintf(stdout, " https://%s -> %s%s (TLS terminated with the run's own authority)\n", - profile.CloudHost, endpoint, s3.DoorPath(profile.Provider, profile.Region)) + fmt.Fprintf(stdout, " https://%s and https://.%s -> %s%s (TLS terminated with the run's own authority)\n", + profile.CloudHost, profile.CloudHost, endpoint, s3.DoorPath(profile.Provider, profile.Region)) } fmt.Fprintln(stdout, " every other host is refused, never tunnelled") if unsafe { @@ -296,17 +354,28 @@ func runChild(endpoint, addr string, profiles []s3.Profile, command []string, un child := exec.Command(command[0], command[1:]...) //nolint:gosec // the operator's own command, explicit argv, no shell child.Stdin, child.Stdout, child.Stderr = os.Stdin, stdout, stderr - child.Env = append(os.Environ(), + child.Env = append(withoutCABundle(os.Environ()), "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL, "http_proxy="+proxyURL, "https_proxy="+proxyURL, // The emulator itself must not go through the proxy, or a request to it // would be intercepted by the thing forwarding to it. "NO_PROXY=127.0.0.1,localhost", "no_proxy=127.0.0.1,localhost", + // SSL_CERT_FILE and nothing else, and the "nothing else" is measured. + // + // It is read by crypto/x509 itself, so every Go client honours it + // without knowing this command exists — which covers the three official + // CLIs and the Terraform providers alike. That is what the existing + // witness measured, and it sets exactly these four variables. + // + // An earlier draft also set AWS_CA_BUNDLE, on the reasoning that an S3 + // client would want it. The Scaleway provider 2.82.0 then failed its + // apply with "unable to add custom RootCAs HTTPClient, has no + // WithTransportOptions, *http.Client": it hands the SDK a prebuilt + // *http.Client, and a CA bundle the SDK is asked to install on it has + // nowhere to go. The variable bought nothing — a Go client already + // trusts SSL_CERT_FILE — and cost the one client this feature exists + // for. TestRunGivesTheChildOnlyTheFourMeasuredVariables holds the list. "SSL_CERT_FILE="+caPath, - // Measured on the Terraform witness: the provider honours HTTPS_PROXY - // and SSL_CERT_FILE, which is why this command needs no network - // namespace and no privilege at all. - "AWS_CA_BUNDLE="+caPath, ) // A signal reaches the child and the cleanup still runs. Without this, diff --git a/internal/cli/run_terraform_conformance_test.go b/internal/cli/run_terraform_conformance_test.go new file mode 100644 index 00000000..7a0c7812 --- /dev/null +++ b/internal/cli/run_terraform_conformance_test.go @@ -0,0 +1,219 @@ +//go:build conformance + +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/conformancetest" + "github.com/stephrobert/feint/internal/s3" +) + +// This is the mirror of TestScalewayTerraformStorageInIsolatedNetwork, and the +// two are only worth having together. +// +// That witness REFUSES to run outside a fresh network namespace: it checks that +// only loopback is up, dials an external address to prove it cannot escape, and +// needs `sudo unshare`. It answers "the provider's traffic can be made to reach +// a local door, and nothing leaked while we proved it". +// +// This one answers the question #849 actually asks, which is the opposite one: +// can an operator get there on the HOST network, with no privilege, no +// namespace and no change to the machine? So it refuses to run INSIDE a +// namespace — a pass obtained in one would prove nothing about the command, +// since the namespace itself would be doing the containment. +// +// The proof is not "apply succeeded". It is that the bytes are in the LOCAL +// byte store at the expected sha256: a request that went anywhere else cannot +// produce that, and the real cloud would refuse these deliberately public fake +// credentials long before it stored anything. +func TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork(t *testing.T) { + interfaces, err := net.Interfaces() + if err != nil { + t.Fatal(err) + } + if len(interfaces) <= 1 { + t.Fatal("this witness must run on the HOST network; in a namespace the containment would be the namespace's, not the command's") + } + pluginDir := os.Getenv("FEINT_TF_PLUGIN_DIR") + if !filepath.IsAbs(pluginDir) { + t.Fatal("FEINT_TF_PLUGIN_DIR must name the predownloaded, pinned provider directory") + } + terraform, err := exec.LookPath("terraform") + if err != nil { + t.Fatal("this witness requires Terraform:", err) + } + + store, objects := doorServer(t, door("scaleway", "fr-par")) + defer func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }() + profiles, err := runProfiles(objects.URL) + if err != nil { + t.Fatalf("the emulator's doors could not be discovered: %v", err) + } + + home := t.TempDir() + if err := os.Chmod(home, 0o700); err != nil { + t.Fatal(err) + } + payload := bytes.Repeat([]byte("feint run terraform witness bytes\n"), 4096) + digest := sha256.Sum256(payload) + want := hex.EncodeToString(digest[:]) + if err := os.WriteFile(filepath.Join(home, "payload"), payload, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, "terraform.rc"), nil, 0o600); err != nil { + t.Fatal(err) + } + const bucket, key = "feint-run-witness", "payload.qcow2" + config := fmt.Sprintf(` +terraform { + required_providers { + scaleway = { + source = "scaleway/scaleway" + version = "= 2.82.0" + } + } +} +provider "scaleway" {} +resource "scaleway_object_bucket" "storage" { + name = %q + region = "fr-par" +} +resource "scaleway_object" "payload" { + bucket = scaleway_object_bucket.storage.name + key = %q + file = "${path.module}/payload" + hash = filemd5("${path.module}/payload") + content_type = "application/octet-stream" +} +`, bucket, key) + if err := os.WriteFile(filepath.Join(home, "main.tf"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + + // The command refuses credentials that are not known fakes, so these are + // the repository's public ones and the refusal is part of what is proved. + fake := conformancetest.FakeCredentials(t) + for name, value := range fake { + t.Setenv(name, value) + } + t.Setenv("SCW_API_URL", objects.URL) + t.Setenv("TF_IN_AUTOMATION", "1") + t.Setenv("TF_INPUT", "0") + t.Setenv("TF_DATA_DIR", filepath.Join(home, ".terraform")) + t.Setenv("TF_CLI_CONFIG_FILE", filepath.Join(home, "terraform.rc")) + // Deliberately NOT TF_PLUGIN_CACHE_DIR: pointing the cache at the same + // directory as -plugin-dir makes the first run unpack into the mirror and + // every later run fail with "cannot install existing provider directory to + // itself". The witness was green once and red the second time for exactly + // that reason, which is a witness that proves nothing on a machine that has + // already run it. + + // init runs OUTSIDE the command: resolving a provider is not what is being + // measured, and asking the run to intercept the registry would be a second + // claim this witness does not make. + initCtx, cancelInit := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancelInit() + initialise := exec.CommandContext(initCtx, terraform, "init", "-no-color", + "-plugin-dir="+pluginDir) + initialise.Dir = home + startedInit := time.Now() + if out, err := initialise.CombinedOutput(); err != nil { + t.Fatalf("terraform init: %v\n%s", err, out) + } + t.Logf("terraform init in %s", time.Since(startedInit).Round(time.Millisecond)) + + // through runs the client through the command under test, from the module + // directory, and returns the exit code the child reported. + through := func(args ...string) (int, string) { + t.Helper() + previous, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(home); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Chdir(previous); err != nil { + t.Error(err) + } + }() + var stdout, stderr bytes.Buffer + started := time.Now() + code, err := runChild(objects.URL, "127.0.0.1:0", profiles, + append([]string{terraform}, args...), false, &stdout, &stderr) + // Timed and logged, because a witness that passes implausibly fast is a + // witness nobody should believe: 0.94s for init+apply+plan+destroy was + // the first result here, and reading the durations is what settles it. + t.Logf("feint run %v: exit %d in %s", args, code, time.Since(started).Round(time.Millisecond)) + if err != nil { + t.Fatalf("feint run %v: %v\n%s", args, err, stderr.String()) + } + return code, stdout.String() + stderr.String() + } + + code, out := through("apply", "-auto-approve", "-no-color") + if code != 0 || !strings.Contains(out, "Apply complete!") { + t.Fatalf("apply did not complete through feint run: code %d\n%s", code, out) + } + // The receipt is part of the contract: #849 asks that every rewrite be + // inspectable, and this is where an operator reads it. + if !strings.Contains(out, ".s3.fr-par.scw.cloud") { + t.Fatalf("the receipt does not name the virtual-host rewrite:\n%s", out) + } + + // THE proof: the bytes are in the local store, read back through the door. + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + objects.URL+s3.DoorPath("scaleway", "fr-par")+bucket+"/"+key, nil) + if err != nil { + t.Fatal(err) + } + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatalf("read the object back from the local door: %v", err) + } + stored, err := io.ReadAll(io.LimitReader(response.Body, int64(len(payload))+1)) + if err := response.Body.Close(); err != nil { + t.Error(err) + } + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("the local door answered %d for the object Terraform wrote: %s", response.StatusCode, stored) + } + got := sha256.Sum256(stored) + if hex.EncodeToString(got[:]) != want { + t.Fatalf("the local store holds %d bytes digesting %s; Terraform wrote %d bytes digesting %s", + len(stored), hex.EncodeToString(got[:]), len(payload), want) + } + + // An empty second plan, which is what proves create-then-read round-trips + // rather than merely answering. + if code, out := through("plan", "-no-color", "-detailed-exitcode"); code != 0 { + t.Fatalf("the second plan reported changes (exit %d):\n%s", code, out) + } + + if code, out := through("destroy", "-auto-approve", "-no-color"); code != 0 || + !strings.Contains(out, "Destroy complete!") { + t.Fatalf("destroy did not complete through feint run: code %d\n%s", code, out) + } +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 1713c0db..486edfd6 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2,14 +2,25 @@ package cli import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" + "fmt" + "io" + "log/slog" "net/http" "net/http/httptest" + "net/url" "os" + "os/exec" + "path/filepath" "strings" + "sync" "testing" + "time" "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" "github.com/stephrobert/feint/internal/providers/scaleway" "github.com/stephrobert/feint/internal/s3" ) @@ -217,3 +228,363 @@ func TestRunKeepsItsAuthorityOwnerOnly(t *testing.T) { t.Fatalf("the child could not name its own authority: %q", stdout.String()) } } + +// deadPID returns a process identifier that has certainly exited and been +// reaped, which is what a leftover directory's owner file holds. +func deadPID(t *testing.T) int { + t.Helper() + cmd := exec.Command("true") + if err := cmd.Run(); err != nil { + t.Fatalf("could not produce an exited process: %v", err) + } + return cmd.Process.Pid +} + +func writeRunDir(t *testing.T, root, name string, pid int) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + if pid == 0 { + return + } + if err := os.WriteFile(filepath.Join(dir, runOwnerFile), + []byte(fmt.Sprintln(pid)), 0o600); err != nil { + t.Fatalf("write the owner file: %v", err) + } +} + +// TestDoctorTellsALeftoverAuthorityFromARunInFlight is cited by +// checkLeftoverRunAuthorities, and the distinction is the whole reason the +// check exists: a healthy `feint run` owns one of these directories for as long +// as its child lives, so a check that merely reported their existence would +// warn on every correct use. +func TestDoctorTellsALeftoverAuthorityFromARunInFlight(t *testing.T) { + root := t.TempDir() + t.Setenv("TMPDIR", root) + if os.TempDir() != root { + t.Skipf("this platform does not take TMPDIR, so the scan cannot be pointed at a fixture") + } + + if c := checkLeftoverRunAuthorities(); c.state != verdictOK || + !strings.Contains(c.title, "no interception authority") { + t.Fatalf("an empty directory did not read as clean: %+v", c) + } + + // A run in flight: this test's own process owns it, and it must NOT be + // reported as something to remove. + writeRunDir(t, root, runDirPrefix+"live", os.Getpid()) + live := checkLeftoverRunAuthorities() + if live.state != verdictOK || !strings.Contains(live.title, "in flight") { + t.Fatalf("a live run was reported as a leftover: %+v", live) + } + if live.fix != "" { + t.Fatalf("a live run was given something to fix: %q", live.fix) + } + + // A leftover: an owner that has exited. + writeRunDir(t, root, runDirPrefix+"dead", deadPID(t)) + stale := checkLeftoverRunAuthorities() + if stale.state != verdictWarn { + t.Fatalf("a leftover was not reported as a warning: %+v", stale) + } + if !strings.Contains(stale.detail, runDirPrefix+"dead") { + t.Fatalf("the leftover is not named: %+v", stale) + } + if strings.Contains(stale.detail, runDirPrefix+"live") && + !strings.Contains(stale.detail, "left alone") { + t.Fatalf("the live run was listed among the leftovers: %+v", stale) + } + + // An owner file an older version never wrote counts as dead, which is the + // direction that gets a leftover removed rather than hidden. + writeRunDir(t, root, runDirPrefix+"ownerless", 0) + if c := checkLeftoverRunAuthorities(); !strings.Contains(c.detail, runDirPrefix+"ownerless") { + t.Fatalf("a directory with no owner file was not reported: %+v", c) + } +} + +// TestDoctorSeesARunInFlightAsItsOwner joins the two halves, and nothing else +// does: TestDoctorTellsALeftoverAuthorityFromARunInFlight writes its own +// fixtures, so it would still pass if `feint run` stopped recording an owner +// altogether. This drives a real run and asks the diagnostic about it while the +// child is alive, which is the only arrangement that proves the producer and +// the check agree on both the prefix and the owner file. +func TestDoctorSeesARunInFlightAsItsOwner(t *testing.T) { + root := t.TempDir() + t.Setenv("TMPDIR", root) + if os.TempDir() != root { + t.Skipf("this platform does not take TMPDIR, so the run cannot be pointed at a fixture") + } + server := reportServer(t, door("scaleway", "fr-par")) + profiles, err := runProfiles(server.URL) + if err != nil { + t.Fatalf("discovery failed: %v", err) + } + + release := filepath.Join(root, "release") + done := make(chan error, 1) + var stdout, stderr syncBuffer + go func() { + _, err := runChild(server.URL, "127.0.0.1:0", profiles, + []string{"sh", "-c", `while [ ! -f "$0" ]; do sleep 0.02; done`, release}, + false, &stdout, &stderr) + done <- err + }() + defer func() { + if err := os.WriteFile(release, nil, 0o600); err != nil { + t.Errorf("release the child: %v", err) + } + if err := <-done; err != nil { + t.Errorf("the run failed: %v (%q)", err, stderr.String()) + } + }() + + // Wait for the run to own its directory, then ask the diagnostic. + deadline := time.Now().Add(10 * time.Second) + for { + c := checkLeftoverRunAuthorities() + if strings.Contains(c.title, "in flight") { + if c.state != verdictOK { + t.Fatalf("a run in flight was not reported as healthy: %+v", c) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("the diagnostic never saw the run in flight; last verdict: %+v", c) + } + time.Sleep(20 * time.Millisecond) + } +} + +// TestRunInterceptsBucketsAddressedAsSubdomains is cited by runChild, and it +// exists because the first proof of this command missed the case entirely: curl +// against the bare authority is path-style, so it passed while every +// virtual-host bucket request — the form Terraform uses — would have been +// refused and never tunnelled. +// +// It goes THROUGH the proxy on purpose. A first draft asserted the receipt text +// and the transport's routing, and neither touches the forward's allow-list: +// the mutation that drops the wildcard would have left it green. So this drives +// a real HTTPS request across the CONNECT, with the run's own authority, which +// is the only arrangement where the allow-list is the thing under test. +func TestRunInterceptsBucketsAddressedAsSubdomains(t *testing.T) { + door := door("scaleway", "fr-par") + store, objects := doorServer(t, door) + defer func() { + if err := store.Close(); err != nil { + t.Errorf("close the byte store: %v", err) + } + }() + profiles, err := runProfiles(objects.URL) + if err != nil { + t.Fatalf("discovery failed: %v", err) + } + + release := filepath.Join(t.TempDir(), "release") + done := make(chan error, 1) + var stdout, stderr syncBuffer + go func() { + _, err := runChild(objects.URL, "127.0.0.1:0", profiles, + []string{"sh", "-c", `while [ ! -f "$0" ]; do sleep 0.02; done`, release}, + false, &stdout, &stderr) + done <- err + }() + defer func() { + if err := os.WriteFile(release, nil, 0o600); err != nil { + t.Errorf("release the child: %v", err) + } + if err := <-done; err != nil { + t.Errorf("the run failed: %v (%q)", err, stderr.String()) + } + }() + + proxyURL, ca := awaitReceipt(t, &stdout) + pem, err := os.ReadFile(ca) //nolint:gosec // the path this run printed + if err != nil { + t.Fatalf("read the run's authority: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("the run's authority is not a usable certificate pool") + } + client := &http.Client{Timeout: 15 * time.Second, Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }} + defer client.CloseIdleConnections() + + // The subject: a bucket addressed as a subdomain reaches the door and the + // door answers about that bucket. 404 NoSuchBucket is a door answer; a + // refused CONNECT is a transport error and never a status. + for _, probe := range []struct { + url string + status int + }{ + {url: "https://s3.fr-par.scw.cloud/", status: http.StatusOK}, + {url: "https://feint-terraform-storage.s3.fr-par.scw.cloud/", status: http.StatusNotFound}, + } { + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, probe.url, nil) + if err != nil { + t.Fatalf("build %s: %v", probe.url, err) + } + response, err := client.Do(request) + if err != nil { + t.Fatalf("%s never reached the door: %v", probe.url, err) + } + body, readErr := io.ReadAll(io.LimitReader(response.Body, 4<<10)) + if err := response.Body.Close(); err != nil { + t.Errorf("close %s: %v", probe.url, err) + } + if readErr != nil { + t.Fatalf("read %s: %v", probe.url, readErr) + } + if response.StatusCode != probe.status { + t.Fatalf("%s answered %d, want %d: %s", probe.url, response.StatusCode, probe.status, body) + } + } + + // An authority of another region stays refused: the widening is one label + // under a declared door, never a second door. + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + "https://s3.nl-ams.scw.cloud/", nil) + if err != nil { + t.Fatal(err) + } + if response, err := client.Do(request); err == nil { + _ = response.Body.Close() + t.Fatal("an undeclared region was served; it must be refused, never tunnelled") + } +} + +// awaitReceipt reads the proxy address and authority path out of the receipt, +// which is also the only place they are published — the run owns both. +func awaitReceipt(t *testing.T, stdout *syncBuffer) (*url.URL, string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + for _, line := range strings.Split(stdout.String(), "\n") { + rest, ok := strings.CutPrefix(line, "feint run: proxy on ") + if !ok { + continue + } + address, authority, split := strings.Cut(rest, ", authority at ") + if !split { + continue + } + parsed, err := url.Parse(strings.TrimSpace(address)) + if err != nil { + t.Fatalf("the receipt names an unusable proxy %q: %v", address, err) + } + return parsed, strings.TrimSpace(authority) + } + if time.Now().After(deadline) { + t.Fatalf("the run printed no receipt: %q", stdout.String()) + } + time.Sleep(20 * time.Millisecond) + } +} + +// doorServer mounts a real S3 door over a private byte store, so a request that +// crosses the proxy is answered by the thing `feint run` exists to reach. +func doorServer(t *testing.T, d s3.Door) (*objectstore.Store, *httptest.Server) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatalf("restrict the byte store: %v", err) + } + store, err := objectstore.OpenWithLimit(t.Context(), filepath.Join(root, "bytes"), 16<<20) + if err != nil { + t.Fatalf("open the byte store: %v", err) + } + packs, err := packsFor(emulator.DefaultEnv()) + if err != nil { + t.Fatalf("build the packs: %v", err) + } + profiles, err := storageProfiles(packs, d.Provider+"/"+d.Region) + if err != nil { + t.Fatalf("profile for %s/%s: %v", d.Provider, d.Region, err) + } + handler, err := s3.Mount(http.NotFoundHandler(), store, profiles, 1<<20, + slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatalf("mount the door: %v", err) + } + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return store, server +} + +// TestRunGivesTheChildOnlyTheFourMeasuredVariables is cited by runChild, and it +// is a list rather than a shape check because the list is the measurement. +// +// The variable it is really about is the one that is ABSENT. An earlier draft +// also set AWS_CA_BUNDLE, reasoning that an S3 client would want it. The +// published Scaleway provider 2.82.0 then failed its apply with "unable to add +// custom RootCAs HTTPClient, has no WithTransportOptions, *http.Client": it +// hands the SDK a prebuilt *http.Client, so a CA bundle the SDK is asked to +// install has nowhere to go. Removing it made the same apply complete, with +// 139,264 identical bytes landing in the local byte store. A Go client already +// trusts SSL_CERT_FILE through crypto/x509, so the variable bought nothing and +// broke the one client this command exists for. +func TestRunGivesTheChildOnlyTheFourMeasuredVariables(t *testing.T) { + server := reportServer(t, door("scaleway", "fr-par")) + profiles, err := runProfiles(server.URL) + if err != nil { + t.Fatalf("discovery failed: %v", err) + } + // Set in the PARENT, which is the case that matters: an operator whose + // shell carries it would otherwise hand it to the child, and its bundle + // does not contain this run's authority under any value. + t.Setenv("AWS_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") + var stdout, stderr bytes.Buffer + // The child reports the environment it was actually given. + code, err := runChild(server.URL, "127.0.0.1:0", profiles, + []string{"sh", "-c", "env"}, false, &stdout, &stderr) + if err != nil || code != 0 { + t.Fatalf("run failed: code %d, %v, %q", code, err, stderr.String()) + } + given := make(map[string]string) + for _, line := range strings.Split(stdout.String(), "\n") { + if name, value, ok := strings.Cut(line, "="); ok { + given[name] = value + } + } + for _, name := range []string{"HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"} { + if !strings.HasPrefix(given[name], "http://127.0.0.1:") { + t.Fatalf("%s is %q, not a loopback proxy", name, given[name]) + } + } + for _, name := range []string{"NO_PROXY", "no_proxy"} { + if !strings.Contains(given[name], "127.0.0.1") { + t.Fatalf("%s does not exempt the emulator: %q", name, given[name]) + } + } + if !strings.HasSuffix(given["SSL_CERT_FILE"], "ca.pem") { + t.Fatalf("SSL_CERT_FILE is %q", given["SSL_CERT_FILE"]) + } + if value, set := given["AWS_CA_BUNDLE"]; set { + t.Fatalf("AWS_CA_BUNDLE is set to %q; the Scaleway provider 2.82.0 fails its apply when it is", value) + } +} + +// syncBuffer is a writer the test can read WHILE the run is still writing to +// it. A plain bytes.Buffer is not that, and -race says so: awaitReceipt polls +// stdout for the receipt line while runChild's goroutine is appending to it. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.String() +} diff --git a/tools/falsify/specs/run-lands-a-client-without-privilege.json b/tools/falsify/specs/run-lands-a-client-without-privilege.json index 762b46f3..ab286ab3 100644 --- a/tools/falsify/specs/run-lands-a-client-without-privilege.json +++ b/tools/falsify/specs/run-lands-a-client-without-privilege.json @@ -35,7 +35,35 @@ "find": "\t\t\t\t_ = p.Signal(sig)", "replace": "\t\t\t\tif false {\n\t\t\t\t\t_ = p.Signal(sig)\n\t\t\t\t}", "test": "TestRunRemovesItsAuthorityAfterASignal" + }, + { + "label": "the run removes a path inside its directory instead of the directory, so the authority outlives the command: the falsification #849 names in as many words. Written this way rather than by neutralising the condition, because `if err := os.RemoveAll(dir); err != nil && false` still CALLS RemoveAll — the first draft of this mutation did that and stayed green, which is the harness catching the author", + "file": "internal/cli/run.go", + "find": "\t\tif err := os.RemoveAll(dir); err != nil {", + "replace": "\t\tif err := os.RemoveAll(filepath.Join(dir, \"nothing-here\")); err != nil {", + "test": "TestRunRemovesItsAuthorityAfterASignal" + }, + { + "label": "the run stops recording its owning PID, so `feint doctor` can no longer tell a leftover from a run in flight and every healthy run reads as a leak. Note which test bites: the doctor test that writes its OWN fixtures would stay green, because it never asks feint run for anything", + "file": "internal/cli/run.go", + "find": "\tif err := os.WriteFile(filepath.Join(dir, runOwnerFile),\n\t\t[]byte(fmt.Sprintln(os.Getpid())), 0o600); err != nil {", + "replace": "\tif err := os.WriteFile(filepath.Join(dir, runOwnerFile+\".unused\"),\n\t\t[]byte(fmt.Sprintln(os.Getpid())), 0o600); err != nil {", + "test": "TestDoctorSeesARunInFlightAsItsOwner" + }, + { + "label": "the virtual-host form stops being intercepted, so every bucket request a real client makes is refused and never tunnelled while a path-style probe keeps passing. This is the defect that shipped briefly and the reason the test had to cross the proxy: an earlier draft asserted the receipt text and the transport's routing, and both would have stayed green", + "file": "internal/cli/run.go", + "find": "\t\thosts = append(hosts, profile.CloudHost, \"*.\"+profile.CloudHost)", + "replace": "\t\thosts = append(hosts, profile.CloudHost, \"*.unreachable.\"+profile.CloudHost)", + "test": "TestRunInterceptsBucketsAddressedAsSubdomains" + }, + { + "label": "AWS_CA_BUNDLE is handed to the child again, which is the measured defect this command shipped once: the published Scaleway provider 2.82.0 then fails its apply with 'unable to add custom RootCAs HTTPClient, has no WithTransportOptions, *http.Client', and the reader is sent looking at TLS rather than at an inherited variable", + "file": "internal/cli/run.go", + "find": "\t\tif name, _, ok := strings.Cut(pair, \"=\"); ok && name == \"AWS_CA_BUNDLE\" {", + "replace": "\t\tif name, _, ok := strings.Cut(pair, \"=\"); ok && name == \"AWS_CA_BUNDLE\" && false {", + "test": "TestRunGivesTheChildOnlyTheFourMeasuredVariables" } ], - "note": "The subject is `feint run` (#849): landing a client whose cloud endpoint is compiled in on the local emulator, with no sudo, no network namespace and no system mutation. Measured, not assumed: the published Scaleway Terraform provider honours HTTP_PROXY, HTTPS_PROXY and SSL_CERT_FILE, which is why the witness in object_storage_terraform_conformance_test.go uses its namespace as an ESCAPE CHECK rather than as the mechanism. The command therefore needs nothing privileged.\n\nThe design decision these mutations guard is that the command takes NO hostname. An earlier draft took `--forward host=target`, and the operator typing the hostname is the operator able to mistype it — a mistyped cloud host is not intercepted, it is resolved, so the run leaves for a third party carrying whatever credentials it holds, and the failure is silent because a request that reached the real cloud looks exactly like one that worked. Instead the mounted doors come from the emulator (GET /_feint/s3, validated by fetchS3) and each door's cloud authority comes from the pack in this binary, which is also the split s3.Profile requires: an instance fact may come from a report, a wire dialect may not. The set of possible targets is therefore the set of doors that exist, and there is no spelling of this command that sends traffic off loopback.\n\nMutation 5 carries the correction of a comment this file got wrong first: it claimed the file check held the signal relay. It does not, because the deferred removal runs regardless. Writing that down was necessary — a test believed for the wrong reason is the defect this repository measures most often." + "note": "The subject is `feint run` (#849): landing a client whose cloud endpoint is compiled in on the local emulator, with no sudo, no network namespace and no system mutation. Measured, not assumed: the published Scaleway Terraform provider honours HTTP_PROXY, HTTPS_PROXY and SSL_CERT_FILE, which is why the witness in object_storage_terraform_conformance_test.go uses its namespace as an ESCAPE CHECK rather than as the mechanism. The command therefore needs nothing privileged.\n\nThe design decision these mutations guard is that the command takes NO hostname. An earlier draft took `--forward host=target`, and the operator typing the hostname is the operator able to mistype it — a mistyped cloud host is not intercepted, it is resolved, so the run leaves for a third party carrying whatever credentials it holds, and the failure is silent because a request that reached the real cloud looks exactly like one that worked. Instead the mounted doors come from the emulator (GET /_feint/s3, validated by fetchS3) and each door's cloud authority comes from the pack in this binary, which is also the split s3.Profile requires: an instance fact may come from a report, a wire dialect may not. The set of possible targets is therefore the set of doors that exist, and there is no spelling of this command that sends traffic off loopback.\n\nMutation 5 carries the correction of a comment this file got wrong first: it claimed the file check held the signal relay. It does not, because the deferred removal runs regardless. Writing that down was necessary — a test believed for the wrong reason is the defect this repository measures most often.\n\nMutations 6 and 7 close #849's own two remaining asks. The first is the falsification that issue names in as many words: a run that forgets its cleanup. The second is why TestDoctorSeesARunInFlightAsItsOwner had to exist at all — TestDoctorTellsALeftoverAuthorityFromARunInFlight writes its own directories, so it would stay green if `feint run` stopped recording an owner altogether, and a diagnostic agreeing with a fixture rather than with its producer is a diagnostic that goes blind in silence.\n\nMutation 8 is the one this spec most needed and least expected. The command's first proof used curl against the bare authority, which is PATH-STYLE, so it passed while the virtual-host form every S3 client actually uses — ., Terraform included — was refused at the allow-list and never tunnelled. The measurement that caught it was reading the existing Terraform witness, which declares the bucket authority explicitly beside the bare one. Widening the allow-list cannot widen what is SERVED: the wildcard decides only what is intercepted, and LocalTransport still refuses a leading label that is not a valid bucket name, so interception is deliberately wider than service and a nonsense bucket gets a local refusal instead of a trip to the cloud.\n\nMutation 9 holds a variable's ABSENCE, which is the only reason the environment is asserted as a list. An earlier draft set AWS_CA_BUNDLE on the reasoning that an S3 client would want it, and that reasoning was never measured: the published Scaleway provider hands its SDK a prebuilt *http.Client, so a CA bundle the SDK is asked to install on it has nowhere to go, and the apply died on a message about transports. Removing it made the same apply complete with 139,264 identical bytes in the local byte store. The command also strips the variable from an INHERITED environment, which is the one thing it edits of the operator's own configuration, because no value of it can help: the station's bundle does not contain this run's freshly minted authority." } From bd2f9a147354809af07b58ed28ad8d7c848f2788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 19:10:46 +0100 Subject: [PATCH 19/28] docs: the hardcoded Terraform refusal fell, and its premise was the wrong one (#849) Three places said Scaleway Terraform Object Storage endpoints cannot be reached, and all three were false as of the previous commit. None was held by an instrument, which is #592's defect exactly: a sentence nobody owns cannot be contradicted, and here there were three of them. - docs/confidence.md:48 said "no". It now says opt-in, names the witness, and names what the witness does NOT cover. - internal/cli/env.go called the path uncertified. It now points at the command instead: variables cannot redirect a client that builds its endpoint in code, but that is a limit of `env`, not of the emulator. - docs/limits.md listed "hardcoded Terraform redirection" among the unsupported, and explained at length why it would stay there. That explanation is the interesting part, and it is kept rather than deleted. It held that redirecting the provider needs DNS interception plus a TLS certificate, priced the certificate half as cheap and the DNS half as neither cheap nor safe, and concluded on that split. The split was right. The REQUIREMENT was not: an HTTP forward proxy needs no DNS, because the client puts the hostname in its own CONNECT line, so the name arrives as data instead of as a lookup to be subverted. The expensive half of the blocker did not have to be paid because it did not exist. A refusal resting on a measured cost is still only as good as the question it measured. What is deliberately NOT claimed. capability.go carries no row for this, and that is not an omission: its matrix admits only control-plane rows proven by the conformance workflow, while this is a data-plane capability established by a witness that needs a provider mirror and therefore does not run on a pull request. Inventing a proof kind for one row would add a resolver nobody exercises, which is why the previous one was deleted with its mechanism in #644. The honest shape is what the table now says: a measurement you reproduce, not a gate. Putting the witness in CI is what would earn the row, and it is the next step rather than this commit's. OpenTofu is untested on this path and the table says so. mise run prepush: green, including the claim gate, which had nothing to refuse because the scaleway/terraform pair is already supported. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/confidence.md | 2 +- docs/limits.md | 39 ++++++++++++++++++++++++++++++++++++--- internal/cli/env.go | 8 +++++++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/docs/confidence.md b/docs/confidence.md index 850428f1..f0c6dd8e 100644 --- a/docs/confidence.md +++ b/docs/confidence.md @@ -45,7 +45,7 @@ prints the reason under the pack that owns it. | Waiting behaviour: a resource that stays `provisioning` for a while | **partly**, with `serve --consistency eventual` | a Scaleway server walks the states fr-par walks, one per read (#637); an Outscale volume is `creating` then `available` and a snapshot `in-queue` then `completed`, and a snapshot taken during `creating` meets the measured `409 InvalidVolumeState` (#124); immediate is still the default — [limits.md](limits.md#lifecycle-transitions-are-immediate-unless-you-ask-for-otherwise) | | Retry and backoff against 429s, 5xx or a control-plane outage | **yes**, on the operations you arm | `PUT /_feint/faults` names an operation and a status, off by default and deterministic; what an injected refusal does NOT count as is stated in [conformance.md](conformance.md#an-injected-refusal-is-not-evidence) | | S3 bytes through declared explicit doors | **opt-in**, AWS CLI 2.32.17, scw 2.62.0, exo 1.95.1 upload/list/download/delete | `mise run conformance:s3:mounted` drives private Scaleway buckets, Exo multipart/range downloads (6,340,608 bytes), AWS restart/resume/completion, and Exo dangling-upload abort/bucket removal; actual bytes, dates and inventories are checked. Normal Exoscale bucket creation is not certified — [object-storage-routing.md](object-storage-routing.md#measured-persistent-multipart-uploads) | -| Hardcoded Scaleway Terraform Object Storage endpoints | **no** | [limits.md](limits.md#object-storage-simple-explicit-doors-not-full-client-workflows) | +| Hardcoded Scaleway Terraform Object Storage endpoints | **opt-in**, Terraform with the published provider 2.82.0 through `feint run` | `TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork`, on the host network with no sudo and no namespace: apply complete, **139,264 identical bytes** read back from the local byte store by sha256, an empty second plan, a complete teardown. Not run on a pull request — it needs a provider mirror, so it is a measurement you reproduce rather than a gate, and OpenTofu is not covered — [object-storage-routing.md](object-storage-routing.md#the-mirror-witness-the-same-trip-with-no-privilege) | | A Terraform run against Exoscale | **yes**, from provider v0.71.0 | the published provider drives this pack since #644; an older one, which splits between this emulator and a paying account, is refused by user agent naming the floor — [limits.md](limits.md#what-replaced-the-refusal-a-floor) | | Control-plane multi-zone or multi-region behaviour on Exoscale | **no** | one zone per process, selectable with `FEINT_EXOSCALE_ZONE`, and the reason is the client — [limits.md](limits.md#exoscale-has-one-zone-per-process-and-the-reason-is-the-client) | | Traffic actually flowing through a gateway or a NAT | **no** | records move, packets do not — [limits.md](limits.md#outscales-gateways-and-nat-move-records-not-packets) | diff --git a/docs/limits.md b/docs/limits.md index 9ad61b6c..5705745c 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -37,11 +37,44 @@ This does not implement bucket virtual hosts, DNS/TLS interception or a complete Exoscale lifecycle; use a dedicated shell because its S3 environment replaces proxy settings. Complete Exoscale bucket workflows, ListObjects v1, owner requests, writable tags/public ACLs, -version histories, multipart copy, generic ETags, checkpoint/reset and hardcoded Terraform -redirection remain unsupported. +version histories, multipart copy, generic ETags and checkpoint/reset remain +unsupported. Hardcoded Terraform redirection no longer belongs on that list: +`feint run` reaches it, measured, and the section below says how and what it +still does not cover. No MinIO service or dependency is used by this implementation. -### Why Scaleway Terraform is still not redirected +### How Scaleway Terraform is reached, and why the DNS half was never needed + +**The premise of the refusal below fell on 2026-11-09, and it is kept here +because the reasoning is the useful part.** It held that redirecting the +provider needs *DNS interception plus a TLS certificate*, priced the certificate +half as cheap and the DNS half as neither cheap nor safe, and concluded on that +split. The split was right. The requirement was not. + +An HTTP forward proxy needs no DNS at all: the client puts the hostname in its +own CONNECT line, so the name arrives as data rather than as a lookup to be +subverted. `feint run` terminates that CONNECT with an authority minted for the +one run, re-originates the request to the local door, and refuses every host it +was not told about. No resolver, no `/etc/hosts`, no system trust store, no +privilege. The expensive half of the blocker did not have to be paid because it +did not exist. + +Measured on the published provider 2.82.0, host network, no sudo: +`TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork` completes apply, +reads **139,264 identical bytes** back from the local byte store by sha256, gets +an empty second plan and a complete teardown. + +What it still does not cover: the witness needs a provider mirror, so it is not +run on a pull request and the capability is a measurement you reproduce rather +than a gate — `capability.go` therefore carries no row for it, since its matrix +admits only control-plane rows proven by the conformance workflow. OpenTofu is +untested on this path. And `feint env --s3` still cannot help, because variables +cannot reach a client that builds its endpoint in code; that is a limit of +`env`, not of the emulator. + +The original reasoning follows. + +### Why Scaleway Terraform was called not redirectable Scaleway Object Storage is S3-compatible, so emulating it is not the hard part (MinIO does it already). The obstacle is **how clients reach it**. diff --git a/internal/cli/env.go b/internal/cli/env.go index 7fc12c62..32c34ece 100644 --- a/internal/cli/env.go +++ b/internal/cli/env.go @@ -129,7 +129,13 @@ func envCommand(args []string, stdout, stderr io.Writer) int { } } if !*unset { - fmt.Fprintln(stderr, "note: S3 supports simple bytes and private non-versioned Scaleway buckets; Exoscale ancillary workflows and hardcoded Terraform endpoints are not certified.") + // The hardcoded-endpoint half of this note went away when `feint + // run` was measured (#849): these variables cannot redirect a + // client that builds its endpoint in code, but that is now a + // limit of `env`, not of the emulator, so the note points at the + // command instead of calling the path uncertified. + fmt.Fprintln(stderr, "note: S3 supports simple bytes and private non-versioned Scaleway buckets; Exoscale ancillary workflows are not certified.") + fmt.Fprintln(stderr, "note: a client that builds its S3 endpoint in code ignores these variables; `feint run -- ` lands it on the local door instead.") if env.Vars["HTTP_PROXY"] != "" { fmt.Fprintln(stderr, "note: S3 host doors replace this shell's proxy settings with the local non-forwarding emulator. Use a dedicated shell; env --s3 --unset removes them, it does not restore previous values.") } From 3ac32681d6ae73286ac92349cbb49d5a4b189aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 20:13:44 +0100 Subject: [PATCH 20/28] ci: the hardcoded-endpoint witness runs on every pull request, and the claim is tied to it (#849, #850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit left a capability asserted in prose and established by a measurement somebody had to take. That is the shape #592 was filed for, and #850's criterion 59 names it: the Object Storage warning must become "a generated/bounded capability statement, not hand-edited marketing prose". A job of its own, not a leg of the matrix, because it needs something no other leg does: a local mirror of the pinned provider. It installs Terraform at the workflow-level pin with the same upstream checksums as the two other jobs that install it, mirrors the provider version read OUT OF the witness file rather than from a second variable, and runs the witness. It then asks `feint doctor` whether the job leaked an interception authority of its own — the diagnostic shipped for that purpose, used on itself. A MIRROR, and deliberately not a plugin cache at the same path. The witness was green once and red the second time on a developer's machine for exactly that reason, and a fresh runner would never have shown it, which is why the comment is in the workflow. What this does NOT do, and the note in the spec is the record of it: the row is still not GENERATED, because capabilityMatrix cannot express it. That matrix admits one axis — a client against a pack — and this is a data-plane capability for a pair already marked supported on the control plane. Widening it for one row would need a new mode and a new proof kind, and capability.go records why that is wrong: #644 deleted a proof kind precisely because no row rested on it, leaving a resolver nobody exercised. So the row is BOUNDED, which is the half of the criterion that prevents the defect. TestTheHardcodedEndpointRowIsOwnedByAnInstrument ties the claim to the instrument in three directions: the row may claim the path only while conformance.yml runs the witness, the witness must be DEFINED — a `go test -run` on a name that matches nothing exits 0, and that false green happened twice during this feature's own development — and the row must name what it leaves out, since "opt-in, Terraform" reads as "both engines" to anybody choosing between Terraform and OpenTofu. /falsify: one mutation, and the spec's note says why only one. Renaming the witness in the workflow makes the control bite. The "names what it leaves out" half is asserted but NOT falsified: the harness requires every name in `find` to survive into `replace`, and in a Markdown table there is no condition to neutralise, only words. That refusal is correct, so the note records which half of this control carries proof and which carries only a test. actionlint and zizmor clean. mise run prepush green. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/conformance.yml | 101 ++++++++++++++++++ internal/cli/confidence_test.go | 69 ++++++++++++ .../hardcoded-endpoint-row-is-owned.json | 13 +++ 3 files changed, 183 insertions(+) create mode 100644 tools/falsify/specs/hardcoded-endpoint-row-is-owned.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f3b51b55..45b300b7 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -555,3 +555,104 @@ jobs: run: | curl -s http://127.0.0.1:4599/_feint/health || true curl -s http://127.0.0.1:4599/_feint/routes || true + + # The hardcoded-endpoint path, driven by a real Terraform on the runner's own + # network. A job of its own rather than a leg of the matrix above, for the + # reason the page job gives: it needs something no other leg does, here a + # local mirror of the pinned provider. + # + # It exists so that the capability stops being a measurement somebody + # reproduces and becomes a gate. #850's criterion is a "generated/bounded + # capability statement, not hand-edited marketing prose", and a sentence can + # only be generated from an instrument that runs — which is what this is. + # capability.go's matrix admits a row only when the conformance workflow + # drives the pair, so without this job the row cannot honestly exist. + # + # The witness refuses to run inside a network namespace, deliberately: a pass + # obtained in one would prove the namespace's containment rather than the + # command's. A GitHub runner has eth0 beside lo, so it qualifies, and that is + # the whole point of running it here — this is an ordinary machine. + hardcoded-endpoint: + name: hardcoded endpoint + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - name: Harden runner + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + with: + egress-policy: audit + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26.10' + cache: false + + # The workflow-level pin, same checksums as the other two jobs that + # install this client: one client, one version, wherever it is proved. + - name: Install Terraform + run: | + set -euo pipefail + workdir="$(mktemp -d)" + base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" + asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" + (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) + sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin + terraform version + + # A MIRROR, not a plugin cache, and not at the same path as either. The + # witness was green once and red the second time on a developer's machine + # because TF_PLUGIN_CACHE_DIR pointed at the -plugin-dir directory: the + # first run unpacks into the mirror and every later run fails with + # "cannot install existing provider directory to itself". A runner is + # fresh each time and would never have shown it, which is exactly why it + # is written down here. + # + # The provider version is the one the witness pins in its own + # configuration; it is deliberately not a workflow variable, because the + # measurement belongs to the test that makes the claim. + - name: Mirror the pinned Scaleway provider + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/tf-mirror" "${RUNNER_TEMP}/tf-seed" + version="$(grep -oE 'version = "= [0-9]+\.[0-9]+\.[0-9]+"' \ + internal/cli/run_terraform_conformance_test.go | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" + test -n "${version}" + printf 'terraform {\n required_providers {\n scaleway = {\n source = "scaleway/scaleway"\n version = "= %s"\n }\n }\n}\n' \ + "${version}" > "${RUNNER_TEMP}/tf-seed/main.tf" + cd "${RUNNER_TEMP}/tf-seed" + terraform providers mirror -platform=linux_amd64 "${RUNNER_TEMP}/tf-mirror" + + # The witness itself. It starts its own door and byte store, drives + # `feint run`, and proves the bytes by digest rather than by exit code: + # a request that went anywhere else cannot put them in the local store. + - name: Terraform reaches the local door through feint run + env: + FEINT_TF_PLUGIN_DIR: ${{ runner.temp }}/tf-mirror + run: | + go test -tags=conformance ./internal/cli \ + -run '^TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork$' \ + -count=1 -v -timeout 15m + + # A leftover authority is this job's own leak if there is one, and the + # diagnostic that finds it is the one shipped for the purpose. + - name: No interception authority outlived the run + if: always() + run: | + set -uo pipefail + # doctor's own exit code is not the verdict here: it fails on any + # FAIL check, and this step asks one question only. So the report is + # captured, printed, and then searched. + go run ./cmd/feint doctor --vm off > doctor.txt 2>&1 || true + grep -A2 'interception authority' doctor.txt + if grep -q 'outlived their' doctor.txt; then + echo "this job left an interception authority behind" >&2 + exit 1 + fi diff --git a/internal/cli/confidence_test.go b/internal/cli/confidence_test.go index cd33a727..c9641b43 100644 --- a/internal/cli/confidence_test.go +++ b/internal/cli/confidence_test.go @@ -118,3 +118,72 @@ func slugOf(heading string) string { } return b.String() } + +// TestTheHardcodedEndpointRowIsOwnedByAnInstrument is #850's criterion 59 as a +// control rather than as a promise. +// +// That criterion asks for "a generated/bounded capability statement, not +// hand-edited marketing prose". The row cannot be GENERATED from +// capabilityMatrix: that matrix admits one axis, a client against a pack, and +// this is a different one — a data-plane capability for a pair already marked +// supported on the control plane. Widening it for one row would mean a new mode +// and a new proof kind, and capability.go records why that is the wrong move +// (#644 deleted a proof kind precisely because no row rested on it, leaving a +// resolver nobody exercised). +// +// So the row is BOUNDED instead, which is the other half of the criterion and +// the part that actually prevents the defect. The claim and the instrument are +// tied: the row may say "opt-in" only while the conformance workflow runs the +// witness that establishes it, and the witness must exist. Delete the job, or +// rename the test, and this fails — which is what "a sentence nobody owns +// cannot be contradicted" was asking for. +func TestTheHardcodedEndpointRowIsOwnedByAnInstrument(t *testing.T) { + const witness = "TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork" + root := repoRoot(t) + + page, err := os.ReadFile(filepath.Join(root, "docs", "confidence.md")) + if err != nil { + t.Fatal(err) + } + var row string + for _, line := range strings.Split(string(page), "\n") { + if strings.Contains(line, "Hardcoded Scaleway Terraform Object Storage endpoints") { + row = line + break + } + } + if row == "" { + t.Fatal("docs/confidence.md no longer carries the hardcoded-endpoint row; if the subject is gone, this control goes with it") + } + if strings.Contains(row, "**no**") { + // Nothing more to check: a refusal needs no instrument, and the row + // below is about what a CLAIM costs. + return + } + + if !strings.Contains(row, witness) { + t.Fatalf("the row claims a capability without naming the witness that establishes it:\n%s", row) + } + workflow, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "conformance.yml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(workflow), witness) { + t.Fatalf("docs/confidence.md claims the hardcoded-endpoint path while conformance.yml no longer runs %s: "+ + "the claim would rest on a measurement nobody takes", witness) + } + source, err := os.ReadFile(filepath.Join(root, "internal", "cli", "run_terraform_conformance_test.go")) + if err != nil { + t.Fatalf("the witness file the row and the workflow both name is unreadable: %v", err) + } + if !strings.Contains(string(source), "func "+witness+"(") { + t.Fatalf("%s is named by the row and by the workflow and defined nowhere: `go test -run` on a name that "+ + "matches nothing exits 0, so this would be a green job proving nothing", witness) + } + // The row must say what it does NOT cover. The witness is not the whole + // story — OpenTofu is untested on this path — and a capability row that + // only lists what works is the brochure this page exists to avoid. + if !strings.Contains(strings.ToLower(row), "opentofu") { + t.Fatalf("the row claims the path without naming what it leaves out:\n%s", row) + } +} diff --git a/tools/falsify/specs/hardcoded-endpoint-row-is-owned.json b/tools/falsify/specs/hardcoded-endpoint-row-is-owned.json new file mode 100644 index 00000000..1873ad0f --- /dev/null +++ b/tools/falsify/specs/hardcoded-endpoint-row-is-owned.json @@ -0,0 +1,13 @@ +{ + "package": "./internal/cli/", + "mutations": [ + { + "label": "the workflow stops running the witness, so docs/confidence.md keeps claiming the hardcoded-endpoint path while nothing on a pull request establishes it: the claim becomes a measurement somebody once took, which is the shape #592 was filed for", + "file": ".github/workflows/conformance.yml", + "find": " -run '^TestScalewayTerraformStorageThroughFeintRunOnTheHostNetwork$' \\", + "replace": " -run '^TestSomethingElseEntirely$' \\", + "test": "TestTheHardcodedEndpointRowIsOwnedByAnInstrument" + } + ], + "note": "The subject is #850's criterion 59: the old Object Storage warning must become \"a generated/bounded capability statement, not hand-edited marketing prose\".\n\nThe row cannot be GENERATED from capabilityMatrix, and that is a measured limit rather than laziness: the matrix admits exactly one axis, a client against a pack, and this is a different one — a data-plane capability for a pair (scaleway/terraform) already marked supported on the control plane. Widening it for a single row would need a new mode and a new proof kind, and capability.go records why that is the wrong move: #644 deleted a proof kind precisely because no row rested on it, leaving a resolver nobody exercised.\n\nSo the row is BOUNDED instead, which is the half of the criterion that prevents the defect. The claim and the instrument are tied in both directions: the row may claim the path only while conformance.yml runs the witness, the witness must be defined (a `go test -run` on a name matching nothing exits 0 — that false green happened twice in this feature's own development), and the row must name what it does NOT cover.\n\nSecond-order note on the mutation target: the mutation edits a REPLACEMENT of the witness name rather than deleting it, because deleting the job wholesale would also remove the mirror step and leave the workflow asserting nothing — a mutation that breaks the harness proves nothing about the guard.\n\nWhy there is only one mutation. The control also requires the row to name what it does NOT cover (OpenTofu is untested on this path, and 'opt-in, Terraform' reads as 'both engines' to anybody choosing between the two engines). That half is NOT falsifiable through this harness, and the refusal is correct rather than an obstacle: the harness requires every name in `find` to survive into `replace`, because a deleted term usually orphans an identifier and fails the whole package, which reads exactly like the guard being proven. On Go source the way round is to neutralise a condition; in a Markdown table there is no condition, only the words. So that half of the control is asserted and not falsified, and this note is the record of which half carries proof and which carries only a test." +} From 4b677cabb9e3cb452504918f39c0d1f80b3d999b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 20:25:54 +0100 Subject: [PATCH 21/28] fix(conformance): three starters arm the byte store, and a marked refusal is not a fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four CI jobs were red on this branch — probe, image, fields and exo-cli — and nothing local could see it. The cause is one line and it is PR #605 repeated to the letter: `mise.toml` was taught `--objectstore` and the other two conformance starters were not. The trap is the one CLAUDE.md states in as many words. `mise run conformance` drives every suite against ONE emulator, which was the one holding the flag, so the local pass was green BY CONSTRUCTION for any assertion about what that emulator serves. `conformance.yml` splits the clients into a matrix with an emulator per leg. The comment on the workflow's own start step already told this story about `--projects`, five red jobs and all, and the flag below it was missing anyway. So the fix is not three edits, it is the property: TestEveryConformanceEmulatorArmsTheSameObjectStore asserts that every starter arms the byte store, built beside the test that already does it for `--projects`. Its own detector was widened before it was trusted — Go's flag package takes `--objectstore=x` as well as `--objectstore x`, and a reader that knew one spelling would report a starter as unarmed the day somebody wrote the other. That came from asking what would make the detector wrong, which is cheaper than watching it fail. The `image` job stays deliberately unarmed, for the reason projectStarters already records: it runs the shipped container with no override, because "a proof that quietly passes different flags proves a different image". That leg needed the other half. `exoscale/v2.list-sos-buckets-usage` answers 503 without a byte store while its schema declares a 200 body, so the probe — which builds its request from the operation's own schema — reported a route answering the wrong shape. The 503 now carries the out-of-band marker #477 introduced for exactly this distinction: a status cannot say "this is the emulator refusing rather than a cloud answering". A MARKED 5xx is a skip. The narrowness is the whole of the safety, and both halves are falsified. Drop the marker requirement and the probe stops reporting server errors at all, including the deliberate faults of faults.go, which carry no marker. Widen 5xx to 4xx and it stops reading error dialects, since every pack marks its own unrouted answers. Either buys a green by giving up what the instrument measures. And no coverage was bought. Without the flag the exoscale probe goes from 2 failed to 0, with 26 skips becoming 28; WITH it the route is driven — 100 probed against 98. That second number is why this is a skip conditioned on a declaration rather than a route taken out of the plan, and it is the rule this file already follows elsewhere: branch on a declaration, never on a name you expect. Two notes on how this was found, since neither was deduction. The red was only visible because `conformance.yml` runs on `push` for main alone, so the branch had never been measured; a workflow_dispatch on the branch is what produced it. And the `hardcoded endpoint` job added in the previous commit passed, so the four failures are the Object Storage work that preceded it, not `feint run`. /falsify: two mutations, both compiling, both biting. mise run prepush green, including TestEveryCitedTestExists, which refused the first version of these citations for naming a test in another package without naming the package — "or a homonym anywhere satisfies this citation". actionlint, zizmor and shellcheck clean. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/conformance.yml | 21 ++++- internal/probe/probe.go | 36 +++++++++ internal/probe/probe_test.go | 76 +++++++++++++++++++ internal/providers/exoscale/object_storage.go | 14 ++++ tools/conformance/leg.sh | 18 ++++- .../projects_declared_everywhere_test.go | 74 ++++++++++++++++++ .../a-marked-refusal-is-not-a-fault.json | 20 +++++ 7 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 tools/falsify/specs/a-marked-refusal-is-not-a-fault.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 45b300b7..b44de64a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -236,8 +236,20 @@ jobs: # version of #572 taught the two of them and left this one, and five # jobs went red at 44 s with "the declared project did not list under # its own name: []". Held by TestEveryConformanceEmulatorDeclaresTheSameProjects. + # + # --objectstore, for the probe rather than for any client suite: + # exoscale/v2.list-sos-buckets-usage reads the byte inventory and + # answers 503 without one, while its schema declares a 200 body, so + # the probe counts it as a route answering the wrong shape. Missing + # here while mise.toml had it is PR #605 to the letter -- two of + # three starters taught a flag -- and it cost four red jobs. + # TestEveryConformanceEmulatorArmsTheSameObjectStore holds it now. + objects="$(mktemp -d)" + chmod 0700 "$objects" + echo "FEINT_OBJECTS=${objects}" >> "$GITHUB_ENV" /tmp/feint start --addr 127.0.0.1:4599 --contracts contracts \ - --projects default,platform-prod --timeout 60s + --projects default,platform-prod --timeout 60s \ + --objectstore "${objects}/objects" # The doorway step of the Outscale Terraform suite evals the real # output of `feint env` rather than restating it, so the script needs # the binary this job just built — the repo-root default it falls @@ -550,6 +562,13 @@ jobs: if: always() run: /tmp/feint stop --addr 127.0.0.1:4599 || true + # The byte store is this job's own side effect on the runner, and a runner + # is reused. Removing it is the same discipline `feint run` follows for + # its authority: whatever a run creates, the run takes away. + - name: Remove the byte store + if: always() + run: rm -rf "${FEINT_OBJECTS:-/nonexistent}" + - name: Dump the emulator state on failure if: failure() run: | diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 55f7baf6..88c0b69f 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -264,8 +264,14 @@ func (r *Runner) call(ctx context.Context, client *http.Client, step Step, pool status, decoded, err := r.exchange(ctx, client, step, path, "", body) result.Status = status var refused refusal + var declined unavailable switch { case err == nil: + case errors.As(err, &declined): + // Skipped, not failed, and the reason is carried rather than invented: + // the message names the pack that declared it. + result.Skipped = declined.Error() + return result case errors.As(err, &refused): result.Refused = true result.RefusalUnread = refused.unread @@ -373,6 +379,17 @@ type refusal struct { func (refusal) Error() string { return "refused" } +// unavailable is an operation the emulator declared it is not serving in this +// configuration, told apart from a cloud answer by the marker rather than by +// the status. It is a SKIP and not a refusal: a refusal is read against the +// contract's declared error shape, and there is no error shape to read here — +// the operation would have answered 200 had the service been armed. +type unavailable struct{ pack string } + +func (u unavailable) Error() string { + return "the " + u.pack + " pack declares this operation unavailable in this configuration" +} + // exchange makes one HTTP call and decodes what came back. A 4xx returns a // refusal carrying the error-shape verdict; any other status returns the // decoded body, nil when there is none. @@ -403,6 +420,25 @@ func (r *Runner) exchange(ctx context.Context, client *http.Client, step Step, p if err != nil { return res.StatusCode, nil, fmt.Errorf("read body: %w", err) } + // A 5xx the emulator MARKS as its own refusal is not a route answering the + // wrong shape, and the distinction is #477's: the marker exists because a + // status cannot carry "this is the emulator refusing" on its own. + // + // Narrow on purpose, and the narrowness is the point. Only 5xx, so every + // 4xx keeps going through readRefusal exactly as before and no existing + // count moves. And only MARKED answers, so the deliberate 5xx a fault + // injects — which carries no marker — is still measured. Without both + // conditions this would be a blanket excuse for server errors, which is the + // opposite of what a probe is for. + // + // What this serves is the one leg no flag can reach: the shipped container, + // run with no override. Everywhere else the starters arm the service, and + // tools/conformance's TestEveryConformanceEmulatorArmsTheSameObjectStore keeps them in step — + // this is not a substitute for arming it. + // TestAMarkedServiceRefusalIsSkippedAndAnUnmarkedOneIsNot holds both halves. + if res.StatusCode >= 500 && res.Header.Get(emulator.NotEmulatedHeader) != "" { + return res.StatusCode, nil, unavailable{pack: res.Header.Get(emulator.NotEmulatedHeader)} + } if res.StatusCode >= 400 && res.StatusCode < 500 { return res.StatusCode, nil, r.readRefusal(raw) } diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index 58a85307..41916176 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -358,3 +358,79 @@ func get(t *testing.T, ts *httptest.Server, path string) (int, string) { n, _ := res.Body.Read(buf) return res.StatusCode, string(buf[:n]) } + +// TestAMarkedServiceRefusalIsSkippedAndAnUnmarkedOneIsNot holds both halves of +// the narrowing in exchange, and the second half is the one that keeps the +// probe worth running. +// +// The subject, measured: `exoscale/v2.list-sos-buckets-usage` answers 503 +// without --objectstore while its schema declares a 200 body, so the probe +// reported a route answering the wrong shape. Four CI jobs went red on it, with +// nothing in the answer saying it was a refusal rather than a fault. The +// emulator now marks it with the out-of-band header #477 introduced for exactly +// that distinction, and a MARKED 5xx becomes a skip. +// +// The danger of that change is obvious and is why the second case exists: if +// any 5xx were skipped, the probe would stop reporting server errors — the one +// thing it is for. So an UNMARKED 5xx must still fail, which also keeps the +// deliberate faults of faults.go measured, since they carry no marker. +func TestAMarkedServiceRefusalIsSkippedAndAnUnmarkedOneIsNot(t *testing.T) { + route := contract.MountedRoute{Method: "GET", Path: "/thing", Operation: "stub/v1/API.GetThing"} + + for _, c := range []struct { + name string + handler http.HandlerFunc + skipped bool + }{ + {name: "marked", skipped: true, handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(emulator.NotEmulatedHeader, "stub") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message": "disabled; start with --objectstore"}`)) + }}, + {name: "unmarked", skipped: false, handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"message": "disabled; start with --objectstore"}`)) + }}, + } { + t.Run(c.name, func(t *testing.T) { + report := runStub(t, true, c.handler, route) + res := report.Results[0] + switch { + case c.skipped: + if res.Skipped == "" { + t.Fatalf("a marked service refusal must be skipped, not judged: %+v", res) + } + if len(report.Failures()) != 0 { + t.Fatalf("a marked service refusal must not fail the probe: %s", report) + } + if !strings.Contains(res.Skipped, "stub") { + t.Errorf("the skip does not name the pack that declared it: %q", res.Skipped) + } + default: + if res.Skipped != "" { + t.Fatalf("an UNMARKED 5xx was skipped: the probe would stop reporting server "+ + "errors, and the faults of faults.go carry no marker: %+v", res) + } + if len(report.Failures()) == 0 { + t.Fatalf("an unmarked 5xx must fail the probe: %s", report) + } + } + }) + } + + // And a marked FOUR-hundred is untouched: the narrowing is 5xx-only, so + // every refusal the probe already read against the declared error shape + // still is one. + marked4xx := func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(emulator.NotEmulatedHeader, "stub") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message": "no such thing"}`)) + } + res := runStub(t, true, http.HandlerFunc(marked4xx), route).Results[0] + if res.Skipped != "" || !res.Refused { + t.Fatalf("a marked 4xx must stay a read refusal, not become a skip: %+v", res) + } +} diff --git a/internal/providers/exoscale/object_storage.go b/internal/providers/exoscale/object_storage.go index f7653d29..21eb425e 100644 --- a/internal/providers/exoscale/object_storage.go +++ b/internal/providers/exoscale/object_storage.go @@ -32,6 +32,20 @@ type sosBucketUsage struct { func (p *Pack) listSOSBucketsUsage(w http.ResponseWriter, r *http.Request) { if p.env.Objects == nil { + // The out-of-band marker #477 introduced, for its own reason: this is + // the emulator refusing an operation, not a cloud answering, and a + // status alone cannot carry that fact. A reader that only sees 503 + // cannot tell "the operator did not arm --objectstore" from "the + // service is broken", and the probe proved it: it builds its request + // from the operation's schema, that schema declares a 200 body, so it + // reported a route answering the wrong shape on every run without the + // flag — four CI jobs, with nothing saying the answer was a refusal. + // + // This does not excuse the flag being missing. The three conformance + // starters arm it, and tools/conformance's TestEveryConformanceEmulatorArmsTheSameObjectStore + // keeps them in step. The marker is for the one leg no flag reaches, + // the shipped container run with no override. + w.Header().Set(emulator.NotEmulatedHeader, p.Name()) writeError(w, http.StatusServiceUnavailable, "Object Storage is disabled; start with --objectstore") return } diff --git a/tools/conformance/leg.sh b/tools/conformance/leg.sh index 26f79479..c02477aa 100755 --- a/tools/conformance/leg.sh +++ b/tools/conformance/leg.sh @@ -107,9 +107,23 @@ tools/conformance/guard.sh leftovers "$vm" # it filters by `name=default` and is unchanged by the second name; what the # second one buys is the case #572 is about, which no unit test can reach: `scw` # and the Terraform provider resolving a project whose name is not `default`. +# --objectstore, for the probe rather than for any client suite here. +# `exoscale/v2.list-sos-buckets-usage` reads the byte inventory and answers 503 +# "Object Storage is disabled" without one, so the probe -- which builds its +# request from the operation's own schema, and that schema declares a 200 body -- +# counts it as a route answering the wrong shape. +# +# This line was missing while mise.toml had it, and that is PR #605 repeated to +# the letter: two of three starters taught a flag, the third left out, a local +# pass green by construction and four CI jobs red (probe, fields, exo-cli and +# image). TestEveryConformanceEmulatorArmsTheSameObjectStore now holds it. +objects="$(mktemp -d)" +chmod 0700 "$objects" +trap 'rm -rf "$objects"' EXIT ./feint start --addr "$addr" --vm "$vm" --contracts contracts \ - --projects default,platform-prod --timeout 60s -trap './feint stop --addr '"$addr"' >/dev/null 2>&1 || true' EXIT + --projects default,platform-prod --timeout 60s \ + --objectstore "$objects/objects" +trap './feint stop --addr '"$addr"' >/dev/null 2>&1 || true; rm -rf "$objects"' EXIT case "$leg" in scw-cli) diff --git a/tools/conformance/projects_declared_everywhere_test.go b/tools/conformance/projects_declared_everywhere_test.go index 39ef14c3..6d840aa7 100644 --- a/tools/conformance/projects_declared_everywhere_test.go +++ b/tools/conformance/projects_declared_everywhere_test.go @@ -107,3 +107,77 @@ func TestEveryConformanceEmulatorDeclaresTheSameProjects(t *testing.T) { "resolve (tools/conformance/scaleway/scw-cli.sh, terraform/main.tf)", first) } } + +// Both spellings, because Go's flag package takes either and a reader that +// only knows one would report a starter as unarmed the day somebody writes the +// other. Found by asking what would make this detector wrong rather than by +// seeing it fail, which is the cheaper of the two ways. +var armsObjectStore = regexp.MustCompile(`--objectstore[=\s]\S+`) + +// TestEveryConformanceEmulatorArmsTheSameObjectStore is the same property as +// the test above, for the flag that repeated its defect. +// +// Measured on this branch: `mise.toml` was taught `--objectstore` and the other +// two starters were not. That is PR #605 to the letter, including the reason a +// local pass could not see it — `mise run conformance` drives every suite +// against ONE emulator, which was the one that had the flag. Four CI jobs went +// red: probe and image on +// +// FAIL exoscale/v2.list-sos-buckets-usage: answered 503 +// +// and fields and exo-cli on the client suite that calls the same operation. +// +// The generalisation worth keeping is that a per-flag test is the wrong shape: +// the test above was written for `--projects` and could not notice a second +// flag drifting. So this one asserts the property for the flag that bit, and +// the next one to be added should extend the TABLE below rather than copy the +// function a third time. +// +// The `image` job stays deliberately absent for the reason projectStarters +// gives: it runs the shipped container with no override, so a flag cannot reach +// it. That leg is served by the probe branching on the emulator's own +// declaration rather than on an expectation — the rule this file already states +// as "branch on a declaration, never on a name you expect". +func TestEveryConformanceEmulatorArmsTheSameObjectStore(t *testing.T) { + for _, flag := range []struct { + name string + pattern *regexp.Regexp + why string + }{ + {name: "--objectstore", pattern: armsObjectStore, + why: "exoscale/v2.list-sos-buckets-usage answers 503 without it while its schema " + + "declares a 200 body, so the probe reports a route answering the wrong shape"}, + } { + found := 0 + for _, file := range projectStarters { + source, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + // Comments skipped, for the reason the test above records: the + // prose explaining a flag contains the flag. + var declared bool + for _, line := range strings.Split(string(source), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + if flag.pattern.MatchString(line) { + declared = true + break + } + } + if !declared { + t.Errorf("%s starts a conformance emulator and does not arm %s: %s", + file, flag.name, flag.why) + continue + } + found++ + } + // The reader proves it can find before it judges, exactly as above: a + // pattern matching nothing would pass this file however wrong it got. + if found == 0 { + t.Fatalf("%s was found in none of the starters, so this test measured nothing", flag.name) + } + } +} diff --git a/tools/falsify/specs/a-marked-refusal-is-not-a-fault.json b/tools/falsify/specs/a-marked-refusal-is-not-a-fault.json new file mode 100644 index 00000000..88366285 --- /dev/null +++ b/tools/falsify/specs/a-marked-refusal-is-not-a-fault.json @@ -0,0 +1,20 @@ +{ + "package": "./internal/probe/", + "mutations": [ + { + "label": "the marker stops being required, so EVERY 5xx is skipped and the probe quietly stops reporting server errors — the one thing it exists for. Also silences the deliberate faults of faults.go, which carry no marker: a probe that skips them would report a clean run over an emulator failing on purpose", + "file": "internal/probe/probe.go", + "find": "\tif res.StatusCode >= 500 && res.Header.Get(emulator.NotEmulatedHeader) != \"\" {", + "replace": "\tif res.StatusCode >= 500 && (res.Header.Get(emulator.NotEmulatedHeader) != \"\" || true) {", + "test": "TestAMarkedServiceRefusalIsSkippedAndAnUnmarkedOneIsNot" + }, + { + "label": "the narrowing widens from 5xx to 4xx, so a marked refusal the probe used to read against the contract's declared error shape becomes a skip. Every pack marks its own unrouted answers, so this would stop checking error dialects wholesale — the opposite of what #477's marker was for", + "file": "internal/probe/probe.go", + "find": "\tif res.StatusCode >= 500 && res.Header.Get(emulator.NotEmulatedHeader) != \"\" {", + "replace": "\tif res.StatusCode >= 400 && res.Header.Get(emulator.NotEmulatedHeader) != \"\" {", + "test": "TestAMarkedServiceRefusalIsSkippedAndAnUnmarkedOneIsNot" + } + ], + "note": "The subject is a measured false failure and the narrow fix for it. `exoscale/v2.list-sos-buckets-usage` reads the byte inventory and answers 503 without --objectstore, while its schema declares a 200 body, so the probe — which builds its request from the operation's own schema — reported a route answering the wrong shape. Four CI jobs went red on it (probe, image, fields, exo-cli) and nothing in the answer said the 503 was a refusal rather than a fault.\n\nThe fix reuses the out-of-band marker #477 introduced for exactly that distinction: a status cannot carry \"this is the emulator refusing rather than a cloud answering\". A MARKED 5xx becomes a skip.\n\nBoth mutations guard the NARROWNESS, because the narrowness is the whole of the safety. Mutation 1 drops the marker requirement and the probe stops reporting server errors at all. Mutation 2 widens 5xx to 4xx and the probe stops reading error dialects, since every pack marks its own unrouted answers. Either one buys a green by giving up what the instrument measures, which is the trade this repository exists to refuse.\n\nMeasured either way: without --objectstore the exoscale probe goes from 2 failed to 0, with 26 skipped becoming 28; WITH it, the route is driven — 100 probed against 98 — so no coverage was bought. That second number is the one that matters, and it is why this is a skip conditioned on a declaration rather than a route removed from the plan.\n\nAnd the marker is not an excuse for the flag being absent. Three conformance starters arm --objectstore and TestEveryConformanceEmulatorArmsTheSameObjectStore keeps them in step; this serves only the one leg no flag reaches, the shipped container run with no override. That parity test is asserted and not falsified here, for the reason recorded in hardcoded-endpoint-row-is-owned.json: the harness requires every name in `find` to survive into `replace`, and removing a flag from a shell script or a workflow cannot satisfy that." +} From df1874aa81a197f7b754d5fb45cf6eeb7af08564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 21:24:56 +0100 Subject: [PATCH 22/28] feat(stacks): an Exoscale stack reaches local object storage, and no object beside it (#850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #850 asks that one Exoscale SOS-dependent stack reach LOCAL object storage instead of the real endpoint. It now does: examples/stacks/exoscale creates a bucket against the local SOS door, and tools/conformance/stacks.sh asserts it is in that door's inventory after the apply — a bucket listed there cannot have been created anywhere else. Nothing is intercepted for it. SOS honours `sos_endpoint`, so the redirection is configuration; `feint run` is for clients that build their endpoint in code, and this is not one of them. ## What the measurements decided, rather than taste **It goes through the AWS provider.** The Exoscale provider has no bucket resource at all: on 0.72.0 its only storage resource is `exoscale_sos_bucket_policy`, which does not create a bucket, and whose three verbs this emulator answers 501 — no policy shape is recorded and this project does not invent formats. The AWS provider pointed at the door is also what Exoscale users actually write. **There is no `aws_s3_object`.** It applies and its bytes land in the local store, sha256 identical. But every later plan wants to update it in place, because it compares the ETag a read returns against the one it computed with `filemd5`, and this emulator returns none — verified on PUT, HEAD and GET alike. internal/s3/multipart.go states that position: an ETag is an opaque entity tag, not an MD5. Serving one on simple PUT means persisting it, which is a v5 byte format older binaries could not read. The choice was between that migration and an example carrying a permanent diff, and a permanent diff in an example teaches a reader that drift is normal here. **The byte store is armed from the declaration, not for everyone.** feint.yaml gains `storage.directory` and `storage.doors`, and `up` renders them into the flags the CLI already has — one direction only, as startArgs' own comment requires. Absent by default for `runtime.mode`'s reason: writing bytes is a side effect on the operator's disk, so it is asked for. **The store is NOT inside the repository**, and that is the object store's own guard rather than a preference: it refuses a parent writable by the group without the sticky bit. A checkout is commonly 775, so a path in the tree would work for one operator's umask and fail for another's, which is the worst possible property for an example. ## One decision I made and the tests reversed I first added `exoscale/ch-dk-2` to defaultS3Doors, reasoning that a single default beats a parity to maintain. Three tests of the env surface failed at once: two Exoscale regions make `feint env exoscale --s3` ambiguous, so it answers "select one with --s3-region" to somebody who asked for nothing unusual. A convenience for this repository's own stack, paid for by every reader. The default is back to two doors and the three conformance starters arm the wider list, with the door list added to TestEveryConformanceEmulatorArmsTheSameObjectStore so the three cannot drift. ## What the repository's own gates caught - `docs:check` refused a committed `.terraform.lock.hcl` under a stack: the generator reads constraints it reports as unresolved. No stack carries one; this one does not either. - The testplan's claim gate caught a sentence saying feint.yaml carries no declarative storage configuration, citing the ABSENCE of `--objectstore` from up.go as its evidence. Both flags are there now, so the sentence went rather than the gate. - `docs/environment.md` regenerated, since the schema grew two fields. /falsify: three mutations on the arming guard, all biting, including the silent direction — storage armed for every declaration would write bytes for a reader who never mentioned one. Two guards are asserted and NOT falsified, and the spec note says which and why: the harness requires every name in `find` to survive into `replace`, which neither a YAML endpoint nor a shell door list can satisfy. The inventory assertion was proved to bite by hand: with its variable emptied, the leg fails naming the door it looked in. mise run prepush green. actionlint, zizmor and shellcheck clean. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/conformance.yml | 3 +- docs/clients.md | 1 + docs/environment.md | 2 + docs/limits.md | 40 +++++ examples/stacks/exoscale/feint.yaml | 25 +++ examples/stacks/exoscale/main.tf | 80 +++++++++ internal/cli/object_storage.go | 13 ++ internal/cli/storage_declaration_test.go | 155 ++++++++++++++++++ internal/cli/up.go | 9 + internal/environment/environment.go | 45 +++++ internal/environment/environment_test.go | 4 + mise.toml | 3 +- tools/conformance/leg.sh | 3 +- .../projects_declared_everywhere_test.go | 41 +++++ tools/conformance/stacks.sh | 26 +++ .../a-stack-reaches-local-object-storage.json | 27 +++ tools/testplan/rules.go | 22 ++- 17 files changed, 487 insertions(+), 12 deletions(-) create mode 100644 internal/cli/storage_declaration_test.go create mode 100644 tools/falsify/specs/a-stack-reaches-local-object-storage.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index b44de64a..af2cd2f8 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -249,7 +249,8 @@ jobs: echo "FEINT_OBJECTS=${objects}" >> "$GITHUB_ENV" /tmp/feint start --addr 127.0.0.1:4599 --contracts contracts \ --projects default,platform-prod --timeout 60s \ - --objectstore "${objects}/objects" + --objectstore "${objects}/objects" \ + --s3-doors scaleway/fr-par,exoscale/ch-dk-2,exoscale/ch-gva-2 # The doorway step of the Outscale Terraform suite evals the real # output of `feint env` rather than restating it, so the script needs # the binary this job just built — the repo-root default it falls diff --git a/docs/clients.md b/docs/clients.md index 8bc7c929..f2bab906 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -78,6 +78,7 @@ Each row is one `required_providers` entry, read where it is written. | `tools/conformance/outscale/terraform-doorway` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | | `tools/conformance/scaleway/terraform` | `scaleway/scaleway` | `2.83.0` | exact: the version that answered | yes | | `examples/stacks/exoscale` | `exoscale/exoscale` | `0.72.0` | exact: the version that answered | yes | +| `examples/stacks/exoscale` | `hashicorp/aws` | `6.72.0` | exact: the version that answered | yes | | `examples/stacks/outscale` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | | `examples/stacks/outscale/modules/net` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | | `examples/stacks/scaleway` | `scaleway/scaleway` | `2.83.0` | exact: the version that answered | yes | diff --git a/docs/environment.md b/docs/environment.md index d09e25b1..dd9226b8 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -233,6 +233,8 @@ gate. The sentence a field carries lives on the field. | `emulator.env` | a block of FEINT_* variables | — | `feint up` | The environment the emulator's own process starts with — FEINT_BOOT_IMAGES and the other FEINT_* knobs, which are read server-side and so cannot be exported after the start. FEINT_* only: a declaration that could set any variable of a process it spawns would be a different kind of file. | | `runtime.mode` | off, incus, incus-vm, incus-ovn, auto | `off` | `feint up` | What backs a powered-on server: the `--vm` values and nothing else. Absent means `off`, because starting machines is a side effect this project asks for rather than assumes. `up` checks the host can deliver the named mode before it starts anything, and refuses rather than downgrading. | | `runtime.images` | a list of family/version | — | `feint up` | The machine images this environment needs present. `up` checks them and refuses naming what is missing and the `feint images` command that builds it; it never builds them itself, because a build takes minutes. Ignored when `runtime.mode` is `off`, where nothing boots. | +| `storage.directory` | a path | — | `feint up` | Where an object-storage stack's bytes are kept. Absent means no object storage, for the reason `runtime.mode` is absent by default: writing bytes is a side effect on the operator's disk, so it is asked for rather than assumed. Resolved against this file, the way `emulator.state` is. | +| `storage.doors` | a list of provider/region | — | `feint up` | The S3 doors served, as `--s3-doors` takes them. Empty with a `storage.directory` set leaves the CLI's own default in place. A door named here is what an `iac.vars` endpoint may point at, and TestEveryDeclaredStorageEndpointNamesADeclaredDoor refuses a stack whose variable names a door the file never asked for. | | `snapshot.load` | a snapshot name | — | `feint up` | A snapshot loaded once the emulator answers, so this environment starts from a known state rather than from nothing: `feint snapshot load `. The name, never a path — snapshots live where `feint snapshot list` says they live. | | `iac.engine` | terraform or opentofu | — | `feint up`, `feint down` | The engine `up` runs and `down` destroys with. Absent means this environment declares no infrastructure, and `up` brings the control plane up and stops there. | | `iac.directory` | a directory | `.` | `feint up`, `feint down` | Where the engine runs, relative to the declaration's own directory. The engine runs there in place — a copy of `*.tf` would leave a local module behind, which is one of the four things this file exists to make impossible. | diff --git a/docs/limits.md b/docs/limits.md index 5705745c..26595e76 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -33,6 +33,46 @@ default), including peak reservation; abort is not GC. Opaque SHA-256 part tags are not generic object ETags. Completion Location consumers are not certified; same-key pagination markers require a live session. Exo's bucket remover does not paginate uploads, so removal beyond 1000 sessions is not certified. +### A SOS bucket in Terraform, and why no object beside it + +`examples/stacks/exoscale` creates a bucket against the local SOS door, which +is #850's criterion that a SOS-dependent stack reach local storage. Nothing is +intercepted for it: SOS honours `sos_endpoint`, so the redirection is +configuration, and `feint run` is for clients that build their endpoint in code. + +It goes through the **AWS provider**, because the Exoscale provider has no +bucket resource. Measured on 0.72.0: its only storage resource is +`exoscale_sos_bucket_policy`, which does not create a bucket, and whose three +verbs this emulator answers 501 — no policy shape is recorded, and this project +does not invent formats. The AWS provider pointed at the door is also what +Exoscale users actually write. + +**There is no `aws_s3_object` in that stack, and the reason is measured.** The +object applies and its bytes land in the local store, sha256 identical. But +every later plan then wants to update it in place: + +```text +~ resource "aws_s3_object" "index" { + + etag = "ff010b1a42027160f88b26c8834cbb3e" +``` + +`aws_s3_object` compares the ETag a read returns against the one it computed +with `filemd5`, and this emulator returns none — on PUT, on HEAD or on GET, +verified on all three. That is the position `internal/s3/multipart.go` states: +the SDK defines an ETag as an opaque entity tag rather than an MD5, and the +multipart tags here identify verified SHA-256 content. Serving an MD5 on simple +PUT would mean persisting it, which is a v5 byte format older binaries could +not read, so the choice was between that migration and an example carrying a +permanent diff. A permanent diff in an example is worse: it teaches a reader +that drift is normal here. + +So the bucket is driven and the object is not, and `tools/conformance/stacks.sh` +asserts the bucket is in the LOCAL door's inventory after the apply — a bucket +listed there cannot have been created anywhere else. That assertion exists +because the stack's bucket is `count = 0` without `TF_VAR_sos_endpoint`: the +leg would otherwise pass while proving nothing, and it was proved to bite with +that variable emptied. + This does not implement bucket virtual hosts, DNS/TLS interception or a complete Exoscale lifecycle; use a dedicated shell because its S3 environment replaces proxy settings. diff --git a/examples/stacks/exoscale/feint.yaml b/examples/stacks/exoscale/feint.yaml index a3bddf50..5f376773 100644 --- a/examples/stacks/exoscale/feint.yaml +++ b/examples/stacks/exoscale/feint.yaml @@ -32,9 +32,34 @@ runtime: images: - ubuntu/24.04 +# Object storage, asked for rather than assumed: writing bytes is a side effect +# on the operator's disk, which is why `storage.directory` is absent by default +# exactly as `runtime.mode` is. +# +# The door and the variable below name the same prefix twice by necessity — one +# arms the emulator, the other points the engine at it — and +# TestEveryDeclaredStorageEndpointNamesADeclaredDoor refuses the two drifting +# apart. The ADDRESS is written once, in `emulator.addr`, and substituted. +# +# NOT a path inside this repository, and that is the object store's own guard +# rather than a preference: it refuses a parent directory writable by the group +# without the sticky bit (internal/core/objectstore/files.go, "writable storage +# parent requires sticky protection"). A checkout is commonly 775, so a path in +# here would work for one operator's umask and fail for another's — which is +# the worst possible property for an example. /tmp is 1777 everywhere. +storage: + directory: /tmp/feint-exoscale-sos + doors: + - exoscale/ch-dk-2 + iac: engine: terraform directory: . + vars: + # SOS honours this endpoint, so no interception is needed here: the AWS + # provider that drives the bucket takes it as configuration. The zone must + # match `var.zone` of main.tf and the door above. + sos_endpoint: ${feint.endpoint}/_feint/s3/exoscale/ch-dk-2 ready: - http:/v2/instance diff --git a/examples/stacks/exoscale/main.tf b/examples/stacks/exoscale/main.tf index 36c8fb37..f291342c 100644 --- a/examples/stacks/exoscale/main.tf +++ b/examples/stacks/exoscale/main.tf @@ -51,6 +51,20 @@ terraform { # the Scaleway fixture is pinned exactly too (#257). version = "0.72.0" } + # SOS is S3, and the Exoscale provider has almost no grip on it: its only + # storage resource is `exoscale_sos_bucket_policy`, which does not create a + # bucket and whose three verbs this emulator answers 501 (no recorded + # policy shape, and this project does not invent formats). So the bucket + # below is driven the way Exoscale users actually drive it in Terraform, + # through the AWS provider pointed at `sos_endpoint`. + # + # Pinned exactly, for the reason the Exoscale pin above records at length: + # a floating constraint resolved to two different versions under the two + # engines on 2026-09-14, both reporting success (#778). + aws = { + source = "hashicorp/aws" + version = "6.72.0" + } } } @@ -64,6 +78,15 @@ variable "pool_size" { default = 2 } +# The SOS door, handed in by `feint up` from feint.yaml, where the emulator's +# address is written once and substituted with ${feint.endpoint}. A stack that +# spelled the address itself would carry it twice. +variable "sos_endpoint" { + type = string + description = "the local Object Storage door; empty leaves the bucket out" + default = "" +} + # No endpoint attribute: the provider has none. It reads EXOSCALE_API_ENDPOINT, # which `feint env exoscale` prints — and which only the patched build honours # for both of its clients. @@ -367,3 +390,60 @@ output "nlb_address" { # TEST-NET-1, and routed nowhere on purpose: see docs/limits.md. value = exoscale_nlb.front.ip_address } + +# --------------------------------------------------------------------------- +# Object storage, through the provider Exoscale users actually use for it +# --------------------------------------------------------------------------- +# +# This is the SOS-dependent half of #850's criteria: a stack reaching LOCAL +# object storage instead of the real endpoint. It goes through the AWS provider +# because the Exoscale provider has no bucket resource at all — measured on +# 0.72.0, whose only storage resource is `exoscale_sos_bucket_policy`. +# +# Unlike the control plane above, nothing is intercepted here: SOS honours +# `sos_endpoint`, so the redirection is configuration. `feint run` is for the +# clients that build their endpoint in code, and this is not one of them. +# +# WHY NO `aws_s3_object` HERE, and it is a measured refusal rather than an +# oversight. The object applies and its bytes land in the local store — sha256 +# identical, verified — but every later plan wants to update it in place, +# because `aws_s3_object` compares the ETag it computed with `filemd5` against +# the one a read returns, and this emulator returns none. That is deliberate: +# internal/s3/multipart.go states the position, the SDK defines an ETag as an +# opaque entity tag rather than an MD5, and the multipart tags here identify +# verified SHA-256 content. Serving an MD5 on simple PUT would mean persisting +# it, which is a v5 byte format older binaries could not read. docs/limits.md +# carries the measurement, and a permanent diff in an example is worse than an +# example that stops where the proof stops. + +provider "aws" { + region = var.zone + access_key = "EXOxxxxxxxxxxxxxxxxxxxx" + secret_key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + # Every one of these keeps the provider from leaving for AWS itself: without + # them it validates credentials, resolves an account id and reads the + # instance metadata service, none of which exists here and none of which this + # emulator serves. + skip_credentials_validation = true + skip_requesting_account_id = true + skip_region_validation = true + skip_metadata_api_check = true + + # The door is a path prefix, not a host, so virtual-host addressing would ask + # for .127.0.0.1 and resolve nowhere. + s3_use_path_style = true + + endpoints { + s3 = var.sos_endpoint + } +} + +resource "aws_s3_bucket" "assets" { + count = var.sos_endpoint == "" ? 0 : 1 + bucket = "platform-assets" +} + +output "assets_bucket" { + value = one(aws_s3_bucket.assets[*].bucket) +} diff --git a/internal/cli/object_storage.go b/internal/cli/object_storage.go index e5799fa6..3aa578ee 100644 --- a/internal/cli/object_storage.go +++ b/internal/cli/object_storage.go @@ -20,6 +20,19 @@ import ( "github.com/stephrobert/feint/internal/s3" ) +// defaultS3Doors is what a reader gets with --objectstore and no door list. +// +// TWO doors, one per provider that has a recorded dialect, and the count is the +// constraint rather than the content: a second Exoscale region here makes +// `feint env exoscale --s3` ambiguous, so it answers "multiple S3 regions are +// enabled; select one with --s3-region" to somebody who asked for nothing +// unusual. Measured by adding exoscale/ch-dk-2 and watching three tests of the +// env surface fail at once. +// +// The conformance starters arm their own wider list, because they drive a stack +// in another zone; see tools/conformance's +// TestEveryConformanceEmulatorArmsTheSameObjectStore for what keeps the three +// of them in step. const defaultS3Doors = "scaleway/fr-par,exoscale/ch-gva-2" type objectStorageFlags struct { diff --git a/internal/cli/storage_declaration_test.go b/internal/cli/storage_declaration_test.go new file mode 100644 index 00000000..9b6ee3e7 --- /dev/null +++ b/internal/cli/storage_declaration_test.go @@ -0,0 +1,155 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/environment" + "github.com/stephrobert/feint/internal/s3" +) + +// TestEveryDeclaredStorageEndpointNamesADeclaredDoor is cited by the +// `storage.doors` field of the environment schema, and it holds the join +// between two things one file says separately. +// +// A stack that drives object storage says it twice by necessity: `storage.doors` +// arms the prefix the emulator serves, and an `iac.vars` value points the engine +// at it. Nothing makes the two agree — the second is a string with a path in it +// — so a door renamed in one place and not the other produces a 404 from a +// prefix nobody mounted, which reads as a broken emulator rather than as a +// declaration out of step. That is the defect shape `--projects` already paid +// for across three starters. +// +// The reverse is checked too, and it is the one that bites silently: doors +// declared with no `storage.directory` are dropped by startArgs without a word, +// so the file would ask for a prefix and never get it. +func TestEveryDeclaredStorageEndpointNamesADeclaredDoor(t *testing.T) { + root := repoRoot(t) + declarations := 0 + for _, source := range []string{"examples/stacks", "examples/quickstart"} { + base := filepath.Join(root, source) + dirs, err := stackDirs(base) + if err != nil { + // A root this repository does not have is not a failure: the + // subject of this test is whatever stacks exist. + continue + } + for _, dir := range dirs { + path := filepath.Join(base, dir, "feint.yaml") + if _, err := os.Stat(path); err != nil { + continue + } + decl, err := environment.Load(path) + if err != nil { + t.Errorf("%s: %v", path, err) + continue + } + declarations++ + checkStorageDeclaration(t, path, decl) + } + } + // The reader proves it can find before it judges: a glob that matched + // nothing would pass this file however wrong the stacks became. + if declarations == 0 { + t.Fatal("no feint.yaml was read, so this test measured nothing") + } +} + +func checkStorageDeclaration(t *testing.T, path string, decl *environment.File) { + t.Helper() + declared := make(map[string]bool, len(decl.Storage.Doors)) + for _, door := range decl.Storage.Doors { + declared[door] = true + } + if len(decl.Storage.Doors) > 0 && decl.Storage.Directory == "" { + t.Errorf("%s declares storage.doors %v and no storage.directory: `up` drops the doors "+ + "without a word, so the file asks for a prefix it will never get", + path, decl.Storage.Doors) + } + for name, value := range decl.IaC.Vars { + index := strings.Index(value, s3.ReportPath+"/") + if index < 0 { + continue + } + rest := strings.TrimPrefix(value[index:], s3.ReportPath+"/") + parts := strings.SplitN(strings.TrimSuffix(rest, "/"), "/", 3) + if len(parts) < 2 { + t.Errorf("%s: iac.vars.%s points inside %s and names no provider/region door: %q", + path, name, s3.ReportPath, value) + continue + } + door := parts[0] + "/" + parts[1] + if !declared[door] { + t.Errorf("%s: iac.vars.%s points at the %s door and storage.doors declares %v: "+ + "the engine would ask for a prefix the emulator never mounted, and a 404 there "+ + "reads as a broken emulator rather than as a declaration out of step", + path, name, door, decl.Storage.Doors) + } + } +} + +// TestStartArgsArmsTheByteStoreOnlyWhenDeclared holds both directions of the +// one thing `up` gained, and the silent half is the second. +// +// A declaration that asks for storage must get it, or the stack's bucket is +// `count = 0` and the leg passes while proving nothing. And a declaration that +// does NOT ask must not get it: writing bytes is a side effect on the +// operator's disk, which is why `storage.directory` is absent by default +// exactly as `runtime.mode` is. Arming it for every stack would make `feint up` +// create a byte store for a reader who never mentioned one. +func TestStartArgsArmsTheByteStoreOnlyWhenDeclared(t *testing.T) { + const declared = `version: 1 +cloud: + provider: exoscale +emulator: + addr: 127.0.0.1:4599 +storage: + directory: /tmp/feint-test-objects + doors: + - exoscale/ch-dk-2 +iac: + engine: terraform + directory: . +` + const silent = `version: 1 +cloud: + provider: exoscale +emulator: + addr: 127.0.0.1:4599 +iac: + engine: terraform + directory: . +` + for _, c := range []struct { + name string + src string + armed bool + }{ + {name: "declared", src: declared, armed: true}, + {name: "silent", src: silent, armed: false}, + } { + t.Run(c.name, func(t *testing.T) { + decl, err := environment.Parse(c.src) + if err != nil { + t.Fatalf("parse: %v", err) + } + args := strings.Join(startArgs(decl), " ") + switch { + case c.armed: + if !strings.Contains(args, "--objectstore") { + t.Fatalf("a declaration asking for storage got none: %s", args) + } + if !strings.Contains(args, "--s3-doors exoscale/ch-dk-2") { + t.Fatalf("the declared door did not reach the command line: %s", args) + } + default: + if strings.Contains(args, "--objectstore") || strings.Contains(args, "--s3-doors") { + t.Fatalf("a declaration mentioning no storage was given one anyway, which writes "+ + "bytes on a disk nobody offered: %s", args) + } + } + }) + } +} diff --git a/internal/cli/up.go b/internal/cli/up.go index d40f36c3..cd344478 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -444,6 +444,15 @@ func startArgs(decl *environment.File) []string { if len(decl.Cloud.Projects) > 0 { args = append(args, "--projects", strings.Join(decl.Cloud.Projects, ",")) } + // Object storage, and only when the file asked for it: writing bytes is a + // side effect on the operator's disk, so it follows `runtime.mode` rather + // than being armed for every stack that does not need it. + if decl.Storage.Directory != "" { + args = append(args, "--objectstore", decl.Resolve(decl.Storage.Directory)) + if len(decl.Storage.Doors) > 0 { + args = append(args, "--s3-doors", strings.Join(decl.Storage.Doors, ",")) + } + } return args } diff --git a/internal/environment/environment.go b/internal/environment/environment.go index f524d3d0..9feec910 100644 --- a/internal/environment/environment.go +++ b/internal/environment/environment.go @@ -95,6 +95,18 @@ type File struct { Load string } + Storage struct { + // Directory holds the persisted bytes. Absent means no object storage + // at all, for the reason Runtime.Mode is absent by default: writing + // bytes is a side effect on the operator's disk, so it is asked for + // rather than assumed. + Directory string + // Doors are the provider/region prefixes served, the `--s3-doors` + // values. Empty with a Directory set takes the CLI's own default, which + // is the same arrangement every other flag here follows. + Doors []string + } + IaC struct { Engine string Directory string @@ -308,6 +320,7 @@ var branches = map[string]bool{ "emulator": true, "runtime": true, "snapshot": true, + "storage": true, "iac": true, } @@ -465,6 +478,38 @@ var fields = []Field{ return nil }, }, + { + Path: "storage.directory", Takes: "a path", Default: "", + Doc: "Where an object-storage stack's bytes are kept. Absent means no object storage, for " + + "the reason `runtime.mode` is absent by default: writing bytes is a side effect on the " + + "operator's disk, so it is asked for rather than assumed. Resolved against this file, " + + "the way `emulator.state` is.", + ReadBy: []string{"up"}, + assign: func(f *File, n node) error { + return scalarInto(n, &f.Storage.Directory) + }, + }, + { + Path: "storage.doors", Takes: "a list of provider/region", Default: "", + Doc: "The S3 doors served, as `--s3-doors` takes them. Empty with a `storage.directory` set " + + "leaves the CLI's own default in place. A door named here is what an `iac.vars` endpoint " + + "may point at, and TestEveryDeclaredStorageEndpointNamesADeclaredDoor refuses a stack " + + "whose variable names a door the file never asked for.", + ReadBy: []string{"up"}, + assign: func(f *File, n node) error { + list, err := scalarList(n) + if err != nil { + return err + } + for _, name := range list { + if strings.Count(name, "/") != 1 { + return fmt.Errorf("%q is not a provider/region door", name) + } + } + f.Storage.Doors = list + return nil + }, + }, { Path: "snapshot.load", Takes: "a snapshot name", Default: "", Doc: "A snapshot loaded once the emulator answers, so this environment starts from a known " + diff --git a/internal/environment/environment_test.go b/internal/environment/environment_test.go index bf361a61..a78675fc 100644 --- a/internal/environment/environment_test.go +++ b/internal/environment/environment_test.go @@ -150,6 +150,10 @@ runtime: - ubuntu/24.04 snapshot: load: fixture +storage: + directory: objects + doors: + - exoscale/ch-dk-2 iac: engine: opentofu directory: terraform diff --git a/mise.toml b/mise.toml index 56093854..4c10514d 100644 --- a/mise.toml +++ b/mise.toml @@ -677,7 +677,8 @@ chmod 0700 "$conformance_objects" trap 'rm -rf "$conformance_objects"' EXIT ./feint start --addr $FEINT_ADDR --vm ${FEINT_VM:-off} --contracts contracts \ --projects default,platform-prod --timeout 60s \ - --objectstore "$conformance_objects/objects" + --objectstore "$conformance_objects/objects" \ + --s3-doors scaleway/fr-par,exoscale/ch-dk-2,exoscale/ch-gva-2 trap './feint stop --addr $FEINT_ADDR >/dev/null 2>&1 || true; rm -rf "$conformance_objects"' EXIT tools/conformance/scaleway/scw-cli.sh "http://$FEINT_ADDR" tools/conformance/scaleway/terraform.sh "http://$FEINT_ADDR" diff --git a/tools/conformance/leg.sh b/tools/conformance/leg.sh index c02477aa..6404f0b7 100755 --- a/tools/conformance/leg.sh +++ b/tools/conformance/leg.sh @@ -122,7 +122,8 @@ chmod 0700 "$objects" trap 'rm -rf "$objects"' EXIT ./feint start --addr "$addr" --vm "$vm" --contracts contracts \ --projects default,platform-prod --timeout 60s \ - --objectstore "$objects/objects" + --objectstore "$objects/objects" \ + --s3-doors scaleway/fr-par,exoscale/ch-dk-2,exoscale/ch-gva-2 trap './feint stop --addr '"$addr"' >/dev/null 2>&1 || true; rm -rf "$objects"' EXIT case "$leg" in diff --git a/tools/conformance/projects_declared_everywhere_test.go b/tools/conformance/projects_declared_everywhere_test.go index 6d840aa7..f45e41b8 100644 --- a/tools/conformance/projects_declared_everywhere_test.go +++ b/tools/conformance/projects_declared_everywhere_test.go @@ -114,6 +114,11 @@ func TestEveryConformanceEmulatorDeclaresTheSameProjects(t *testing.T) { // seeing it fail, which is the cheaper of the two ways. var armsObjectStore = regexp.MustCompile(`--objectstore[=\s]\S+`) +// declaresDoors reads the door list. It must not match `--objectstore-budget-mib` +// or any other flag merely starting with the same letters, which `[=\s]` is what +// prevents — the same care armsObjectStore needs for `--objectstore-budget-mib`. +var declaresDoors = regexp.MustCompile(`--s3-doors[=\s]\S+`) + // TestEveryConformanceEmulatorArmsTheSameObjectStore is the same property as // the test above, for the flag that repeated its defect. // @@ -147,6 +152,10 @@ func TestEveryConformanceEmulatorArmsTheSameObjectStore(t *testing.T) { {name: "--objectstore", pattern: armsObjectStore, why: "exoscale/v2.list-sos-buckets-usage answers 503 without it while its schema " + "declares a 200 body, so the probe reports a route answering the wrong shape"}, + {name: "--s3-doors", pattern: declaresDoors, + why: "examples/stacks/exoscale creates its bucket in ch-dk-2, which the CLI's own " + + "default does not serve — and widening that default instead made `feint env " + + "exoscale --s3` ambiguous for every reader, which is worse"}, } { found := 0 for _, file := range projectStarters { @@ -179,5 +188,37 @@ func TestEveryConformanceEmulatorArmsTheSameObjectStore(t *testing.T) { if found == 0 { t.Fatalf("%s was found in none of the starters, so this test measured nothing", flag.name) } + // And for a flag carrying a LIST, presence is not enough: three + // starters each naming a different set of doors is the shape that + // passes on the leg that matches and fails on the others, which reads + // as a flake rather than as a declaration nobody kept — the argument + // the projects test above makes in the same words. + if flag.name == "--s3-doors" { + var first, firstFile string + for _, file := range projectStarters { + source, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + for _, line := range strings.Split(string(source), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + m := flag.pattern.FindString(line) + if m == "" { + continue + } + value := strings.TrimLeft(strings.TrimPrefix(m, "--s3-doors"), "= \t") + if first == "" { + first, firstFile = value, file + } else if value != first { + t.Errorf("%s declares %s %q and %s declares %q: one emulator per leg, two door lists", + file, flag.name, value, firstFile, first) + } + break + } + } + } } } diff --git a/tools/conformance/stacks.sh b/tools/conformance/stacks.sh index 22041e33..bda22f44 100755 --- a/tools/conformance/stacks.sh +++ b/tools/conformance/stacks.sh @@ -107,6 +107,13 @@ run_stack() { # name export EXOSCALE_API_ENDPOINT="$ENDPOINT/v2" export EXOSCALE_ZONE="${EXOSCALE_ZONE:-ch-dk-2}" export TF_VAR_zone="$EXOSCALE_ZONE" + # The SOS door, so the stack's bucket is created against LOCAL object + # storage rather than left out. The bucket is `count = 0` without this + # variable, which is how the stack stays appliable for a reader whose + # emulator serves no door — and also how this leg would pass while + # proving nothing, if the variable were forgotten here. The assertion + # after the apply is what makes that impossible to forget silently. + export TF_VAR_sos_endpoint="$ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE" ;; *) VARS=(-var "endpoint=$ENDPOINT") @@ -135,6 +142,25 @@ run_stack() { # name || fail "$name: apply failed" ok "applied" + # The bucket of the Exoscale stack is `count = 0` when TF_VAR_sos_endpoint is + # empty, which is how the stack stays appliable for a reader whose emulator + # serves no door. It is also how this leg would pass while proving nothing, + # so the inventory is asked rather than assumed: a bucket that is in the + # LOCAL door's listing cannot have been created anywhere else. + # + # Proved to bite: with the export above emptied, this leg fails naming the + # door it looked in. + if [ "$name" = exoscale ]; then + local inventory + inventory="$(curl -fsS --max-time 20 "$ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE/" || true)" + case "$inventory" in + *"platform-assets"*) + ok "the SOS bucket is in the local door's inventory" ;; + *) + fail "$name: platform-assets is absent from the local door at $ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE/ — either TF_VAR_sos_endpoint was not exported, so the bucket was left out, or it was created somewhere this emulator cannot see" ;; + esac + fi + # The assertion that separates a test from a demonstration. Both defects this # file's header names were caught here rather than by the apply. echo "- $name: the second plan changes no resource" diff --git a/tools/falsify/specs/a-stack-reaches-local-object-storage.json b/tools/falsify/specs/a-stack-reaches-local-object-storage.json new file mode 100644 index 00000000..2884ed12 --- /dev/null +++ b/tools/falsify/specs/a-stack-reaches-local-object-storage.json @@ -0,0 +1,27 @@ +{ + "package": "./internal/cli/", + "mutations": [ + { + "label": "a declaration that asks for object storage no longer gets it, so the stack's bucket falls to `count = 0` and the conformance leg passes while proving nothing about local storage — which is exactly the failure mode the inventory assertion in stacks.sh exists to make impossible", + "file": "internal/cli/up.go", + "find": "\tif decl.Storage.Directory != \"\" {", + "replace": "\tif decl.Storage.Directory != \"\" && false {", + "test": "TestStartArgsArmsTheByteStoreOnlyWhenDeclared" + }, + { + "label": "object storage is armed for EVERY declaration, so `feint up` writes bytes on the disk of a reader who never mentioned one. The silent half of the same guard, and the one a careless fix would break: `runtime.mode` is absent by default for this very reason", + "file": "internal/cli/up.go", + "find": "\tif decl.Storage.Directory != \"\" {", + "replace": "\tif decl.Storage.Directory != \"\" || true {", + "test": "TestStartArgsArmsTheByteStoreOnlyWhenDeclared" + }, + { + "label": "the declared doors stop reaching the command line, so the emulator serves the CLI's default list and the stack's ch-dk-2 bucket lands on a prefix nobody mounted. A 404 there reads as a broken emulator rather than as a door that was asked for and dropped", + "file": "internal/cli/up.go", + "find": "\t\tif len(decl.Storage.Doors) > 0 {", + "replace": "\t\tif len(decl.Storage.Doors) > 0 && false {", + "test": "TestStartArgsArmsTheByteStoreOnlyWhenDeclared" + } + ], + "note": "The subject is #850's criterion that an Exoscale SOS-dependent stack reach LOCAL object storage, and the pieces `up` needed for it: a `storage.*` block in feint.yaml, armed only when asked.\n\nWhy the second mutation matters as much as the first. Writing bytes is a side effect on the operator's disk, which is why `storage.directory` is absent by default exactly as `runtime.mode` is — a fix that armed storage unconditionally would make `feint up` create a byte store for a reader who never mentioned one. The guard has two directions and a mutation for each.\n\nMeasured on the way here, and both findings are in docs/limits.md rather than only in a commit message. The Exoscale provider has NO bucket resource (0.72.0: its only storage resource is `exoscale_sos_bucket_policy`, which does not create a bucket and whose three verbs this emulator answers 501), so the bucket goes through the AWS provider pointed at `sos_endpoint` — which is what Exoscale users write. And there is no `aws_s3_object` beside it: the object applies and its bytes land locally, sha256 identical, but every later plan wants to update it in place because it compares the ETag a read returns against `filemd5` and this emulator returns none, on PUT, HEAD and GET alike. Serving an MD5 would mean a v5 byte format older binaries could not read, and a permanent diff in an example teaches a reader that drift is normal here.\n\nTwo guards here are asserted and NOT falsified, for the reason recorded in hardcoded-endpoint-row-is-owned.json: the harness requires every name in `find` to survive into `replace`, and neither a door list in a shell script nor a YAML endpoint can satisfy that. They are TestEveryDeclaredStorageEndpointNamesADeclaredDoor, which joins `storage.doors` to the `iac.vars` endpoint, and the door-list parity across the three conformance starters. The inventory assertion in stacks.sh WAS proved to bite by hand: with its variable emptied, the leg fails naming the door it looked in." +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 49c3e2da..57be5348 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -271,15 +271,19 @@ var rules = []rule{ Unproven: "the proxy is not a network namespace and does not constrain arbitrary clients which ignore it", }, { - Path: "internal/cli/object_storage.go", - Why: "S3 activation, discovery and actual-process persistence", - Runs: []string{"conformance:s3:mounted"}, - Unproven: "complete provider CLI lifecycles, declarative feint.yaml storage configuration and hardcoded Terraform endpoints", - Cites: []claim{{ - About: "declarative feint.yaml storage configuration", - In: "internal/cli/up.go", - Absent: []string{"--objectstore", "--s3-doors"}, - }}, + Path: "internal/cli/object_storage.go", + Why: "S3 activation, discovery and actual-process persistence", + // The second leg since the declaration grew `storage.*`: `up` now arms + // the byte store from feint.yaml, and tools/conformance/stacks.sh is + // what drives a stack through it. + Runs: []string{"conformance:s3:mounted", "conformance:leg -- terraform"}, + Unproven: "complete provider CLI lifecycles and hardcoded Terraform endpoints", + // The claim that used to live here said feint.yaml carried no + // declarative storage configuration, and cited the ABSENCE of + // `--objectstore` and `--s3-doors` from up.go as its evidence. Both are + // there now, and this file's own gate caught the stale sentence rather + // than letting it age — which is the mechanism working, so the sentence + // went instead of the gate. }, { Path: "internal/cli/object_storage_conformance_test.go", From b8dfc6d4ee97434b1e5fba84032a2bafa541829e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Mon, 9 Nov 2026 21:44:51 +0100 Subject: [PATCH 23/28] fix(conformance): the SOS door is asked for, not assumed, so the image leg survives it (#850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit turned the `image` job red, and the cause is the population trap CLAUDE.md states in as many words: a control eprouve on the richest run and broken by the poorest. stacks.sh exported TF_VAR_sos_endpoint unconditionally, so the stack asked for a bucket on every emulator. The `image` job runs the SHIPPED container with no override — deliberately, because "a proof that quietly passes different flags proves a different image" — so it serves no door at all, and the apply failed there while ten other jobs passed. I had measured only the run that had the flag, which is the run I happened to have in hand. It now asks the emulator which doors it serves (GET /_feint/s3) and decides from the answer. That is this repository's own rule, already written in projectStarters for this exact job: branch on a declaration, never on a name you expect. The decision is recorded in SOS_DOOR rather than inferred from the variable, and that is not tidiness. An assertion keyed off its own input cannot report the bug where that input went missing — #459's shape, a suite skipping itself. Three executions measured by hand, because one would have proved nothing: - no door at all: a note, the bucket is left out, the stack still applies, verdict 0 — this is the image job; - three doors: the inventory assertion holds, verdict 0 — this is the matrix; - a served door with the export emptied: the leg FAILS, naming the door it looked in. This is the one that proves the guard cannot skip itself. A note on the harness rather than the subject, since it cost two wrong readings. A `head -3` on the output of stacks.sh truncated before the exoscale stack was even reached, so three `ok:` lines from the SCALEWAY stack read as a guard that had passed. Before that, a sed whose pattern did not match left the subject unchanged and the test reported on nothing. Both were caught by asking why a result looked impossible rather than by the gates. mise run prepush green. shellcheck clean. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/conformance/stacks.sh | 38 +++++++++++++++---- .../a-stack-reaches-local-object-storage.json | 2 +- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/tools/conformance/stacks.sh b/tools/conformance/stacks.sh index bda22f44..d1d2cbc7 100755 --- a/tools/conformance/stacks.sh +++ b/tools/conformance/stacks.sh @@ -90,6 +90,9 @@ trap cleanup_stack EXIT run_stack() { # name local name="$1" local src="$ROOT/examples/stacks/$name" + # Reset per stack: a decision taken for one stack must not be inherited by + # the next, which is how a guard starts reporting about the wrong subject. + SOS_DOOR=0 [ -d "$src" ] || fail "no stack at $src" case "$name" in @@ -107,13 +110,32 @@ run_stack() { # name export EXOSCALE_API_ENDPOINT="$ENDPOINT/v2" export EXOSCALE_ZONE="${EXOSCALE_ZONE:-ch-dk-2}" export TF_VAR_zone="$EXOSCALE_ZONE" - # The SOS door, so the stack's bucket is created against LOCAL object - # storage rather than left out. The bucket is `count = 0` without this - # variable, which is how the stack stays appliable for a reader whose - # emulator serves no door — and also how this leg would pass while - # proving nothing, if the variable were forgotten here. The assertion - # after the apply is what makes that impossible to forget silently. - export TF_VAR_sos_endpoint="$ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE" + # The SOS door, ASKED FOR rather than assumed, and that is this file's + # own rule: branch on a declaration, never on a name you expect. + # + # The emulator of the `image` job runs the shipped container with no + # override — deliberately, since "a proof that quietly passes different + # flags proves a different image" — so it serves no door at all. Setting + # the variable unconditionally made that job's apply FAIL, measured, and + # it is the population trap CLAUDE.md names: a control eprouve on the + # richest run and broken by the poorest. + # + # Without the variable the stack's bucket is `count = 0`, so the stack + # stays appliable against an emulator that serves no bytes. + # + # SOS_DOOR records the decision, and the assertion after the apply keys + # off THAT rather than off the variable. The difference is #459's: a + # guard that skips itself when its own input is missing cannot report the + # bug where the input went missing. With this, a served door and a + # forgotten export fail the leg instead of quietly passing it. + SOS_DOOR=0 + if curl -fsS --max-time 20 "$ENDPOINT/_feint/s3" 2>/dev/null \ + | grep -q "\"path\":\"/_feint/s3/exoscale/$EXOSCALE_ZONE/\""; then + SOS_DOOR=1 + export TF_VAR_sos_endpoint="$ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE" + else + echo " note: this emulator serves no exoscale/$EXOSCALE_ZONE door, so the stack's bucket is left out" + fi ;; *) VARS=(-var "endpoint=$ENDPOINT") @@ -150,7 +172,7 @@ run_stack() { # name # # Proved to bite: with the export above emptied, this leg fails naming the # door it looked in. - if [ "$name" = exoscale ]; then + if [ "$name" = exoscale ] && [ "${SOS_DOOR:-0}" = 1 ]; then local inventory inventory="$(curl -fsS --max-time 20 "$ENDPOINT/_feint/s3/exoscale/$EXOSCALE_ZONE/" || true)" case "$inventory" in diff --git a/tools/falsify/specs/a-stack-reaches-local-object-storage.json b/tools/falsify/specs/a-stack-reaches-local-object-storage.json index 2884ed12..5b0968a9 100644 --- a/tools/falsify/specs/a-stack-reaches-local-object-storage.json +++ b/tools/falsify/specs/a-stack-reaches-local-object-storage.json @@ -23,5 +23,5 @@ "test": "TestStartArgsArmsTheByteStoreOnlyWhenDeclared" } ], - "note": "The subject is #850's criterion that an Exoscale SOS-dependent stack reach LOCAL object storage, and the pieces `up` needed for it: a `storage.*` block in feint.yaml, armed only when asked.\n\nWhy the second mutation matters as much as the first. Writing bytes is a side effect on the operator's disk, which is why `storage.directory` is absent by default exactly as `runtime.mode` is — a fix that armed storage unconditionally would make `feint up` create a byte store for a reader who never mentioned one. The guard has two directions and a mutation for each.\n\nMeasured on the way here, and both findings are in docs/limits.md rather than only in a commit message. The Exoscale provider has NO bucket resource (0.72.0: its only storage resource is `exoscale_sos_bucket_policy`, which does not create a bucket and whose three verbs this emulator answers 501), so the bucket goes through the AWS provider pointed at `sos_endpoint` — which is what Exoscale users write. And there is no `aws_s3_object` beside it: the object applies and its bytes land locally, sha256 identical, but every later plan wants to update it in place because it compares the ETag a read returns against `filemd5` and this emulator returns none, on PUT, HEAD and GET alike. Serving an MD5 would mean a v5 byte format older binaries could not read, and a permanent diff in an example teaches a reader that drift is normal here.\n\nTwo guards here are asserted and NOT falsified, for the reason recorded in hardcoded-endpoint-row-is-owned.json: the harness requires every name in `find` to survive into `replace`, and neither a door list in a shell script nor a YAML endpoint can satisfy that. They are TestEveryDeclaredStorageEndpointNamesADeclaredDoor, which joins `storage.doors` to the `iac.vars` endpoint, and the door-list parity across the three conformance starters. The inventory assertion in stacks.sh WAS proved to bite by hand: with its variable emptied, the leg fails naming the door it looked in." + "note": "The subject is #850's criterion that an Exoscale SOS-dependent stack reach LOCAL object storage, and the pieces `up` needed for it: a `storage.*` block in feint.yaml, armed only when asked.\n\nWhy the second mutation matters as much as the first. Writing bytes is a side effect on the operator's disk, which is why `storage.directory` is absent by default exactly as `runtime.mode` is — a fix that armed storage unconditionally would make `feint up` create a byte store for a reader who never mentioned one. The guard has two directions and a mutation for each.\n\nMeasured on the way here, and both findings are in docs/limits.md rather than only in a commit message. The Exoscale provider has NO bucket resource (0.72.0: its only storage resource is `exoscale_sos_bucket_policy`, which does not create a bucket and whose three verbs this emulator answers 501), so the bucket goes through the AWS provider pointed at `sos_endpoint` — which is what Exoscale users write. And there is no `aws_s3_object` beside it: the object applies and its bytes land locally, sha256 identical, but every later plan wants to update it in place because it compares the ETag a read returns against `filemd5` and this emulator returns none, on PUT, HEAD and GET alike. Serving an MD5 would mean a v5 byte format older binaries could not read, and a permanent diff in an example teaches a reader that drift is normal here.\n\nTwo guards here are asserted and NOT falsified, for the reason recorded in hardcoded-endpoint-row-is-owned.json: the harness requires every name in `find` to survive into `replace`, and neither a door list in a shell script nor a YAML endpoint can satisfy that. They are TestEveryDeclaredStorageEndpointNamesADeclaredDoor, which joins `storage.doors` to the `iac.vars` endpoint, and the door-list parity across the three conformance starters. The inventory assertion in stacks.sh WAS proved to bite by hand: with its variable emptied, the leg fails naming the door it looked in.\n\nWhat the CI run then taught, and it was a population error of exactly the kind CLAUDE.md names. The first version exported TF_VAR_sos_endpoint unconditionally, so the `image` job — which runs the shipped container with NO override and therefore serves no door — failed its apply: the stack asked for a bucket on a prefix nobody mounted. Eprouve on the richest run, broken by the poorest. stacks.sh now asks the emulator which doors it serves (GET /_feint/s3) and decides from that, which is this repository's own rule: branch on a declaration, never on a name you expect.\n\nAnd the decision is recorded in SOS_DOOR rather than inferred from the variable, because an assertion keyed off its own input cannot report the bug where that input went missing (#459, a suite skipping itself). Three executions were measured by hand: no door at all (a note, verdict 0), three doors (the assertion holds, verdict 0), and a served door with the export emptied (the leg FAILS, naming the door it looked in). The third is the one that proves the guard cannot skip itself." } From 0691306e195b53edb9b0ae1690f1f0526ab11aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Tue, 10 Nov 2026 01:05:28 +0100 Subject: [PATCH 24/28] test(cli): the authority test reads its own CA instead of stat'ing it, so macOS can run it (#849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TestRunKeepsItsAuthorityOwnerOnly` failed on both macOS runners, and on nothing the test exists to measure: it ran `stat -c '%a %n'`, which is GNU, and the BSD stat on macOS answered `illegal option -- c` with its usage. The run directory was 0700 the whole time. Rewritten POSIX-only, and stronger than before rather than merely portable: the child now `cat`s its authority to /dev/null, so exit 0 proves the file is READABLE by the client process — the property proxy.WriteCA's deliberate 0644 exists for, and which the old form only inferred from a mode string. The directory's owner-only mode is read from `ls -ld`, which every Unix spells `drwx------`. The question still comes from inside the child, through the variable the child was actually given. A test that stat'd a path the test itself computed would be checking its own arithmetic, and that is why it is written this way. Measured, not assumed: `grep -rn 'stat -c'` finds no other use in the repository, so this was the only GNU-only invocation. The nine mutations of tools/falsify/specs/run-lands-a-client-without-privilege.json all still bite, which is what proves the rewrite did not soften the assertion. Green on Linux locally; the macOS half is proved by the two jobs that failed, not by me. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- internal/cli/run_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 486edfd6..8d70f88c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -211,7 +211,12 @@ func TestRunKeepsItsAuthorityOwnerOnly(t *testing.T) { } var stdout, stderr bytes.Buffer code, err := runChild(server.URL, "127.0.0.1:0", profiles, - []string{"sh", "-c", `stat -c '%a %n' "$SSL_CERT_FILE" "$(dirname "$SSL_CERT_FILE")"`}, + // POSIX only, and read rather than stat'd: `stat -c` is GNU, and the + // BSD stat on macOS rejected it, which failed this test on a property + // it was not measuring. `cat` to /dev/null proves the child can READ + // its authority (exit 0 below), and `ls -ld` prints the directory's + // mode the one way every Unix spells it. + []string{"sh", "-c", `cat "$SSL_CERT_FILE" > /dev/null || exit 3; printf 'authority=%s\n' "$SSL_CERT_FILE"; ls -ld "$(dirname "$SSL_CERT_FILE")"`}, false, &stdout, &stderr) if err != nil || code != 0 { t.Fatalf("the child could not read its own authority: code %d, %v, %q", code, err, stderr.String()) @@ -221,8 +226,8 @@ func TestRunKeepsItsAuthorityOwnerOnly(t *testing.T) { // still be READABLE by the client process. proxy.WriteCA writes 0644 for // that second reason, deliberately; the privacy comes from the directory, // and the authority's private key is never written at all. - if !strings.Contains(stdout.String(), "700 ") { - t.Fatalf("the child saw a directory mode other than 700: %q", stdout.String()) + if !strings.Contains(stdout.String(), "drwx------") { + t.Fatalf("the child saw a directory mode other than owner-only: %q", stdout.String()) } if !strings.Contains(stdout.String(), "ca.pem") { t.Fatalf("the child could not name its own authority: %q", stdout.String()) From e52e1c7a70aa30dedee831ffaf5d402358856076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Wed, 11 Nov 2026 09:00:23 +0100 Subject: [PATCH 25/28] fix(cli): feint run accepts the fakes feint env publishes, derived rather than copied (#849, #262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two strangers' stacks from examples/stacks/surveyed.md were replayed against this branch, chosen because the register names them as the two the Object Storage data plane could change. Both moved, and the replay found one defect in feint that no test had. ## The defect The sequence both commands document — eval `feint env exoscale`, then `feint run -- terraform apply` — was refused by its own binary: feint run: refusing to run with credentials that are not known fakes: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, EXOSCALE_API_KEY, EXOSCALE_API_SECRET `knownFakes` was a second, hand-written copy of values the packs already publish through `Pack.Env`, and the two had drifted: the pack publishes a 23-character EXOSCALE_API_KEY, the list held a 25-character one, and the 40-character secret was in neither. The operator who followed the two commands' own notes was told their credentials were real. The fix removes the second list rather than correcting it. `acceptedFakes` reads the packs — the same source `feint env` prints — so the two cannot drift again. knownFakes stays for the placeholders no pack publishes. It fails closed: if the packs cannot be built, the accepted set shrinks, and a refusal names the variable and offers --unsafe-real-credentials. TestEveryPublishedFakeIsAcceptedByRun asserts it from the packs rather than from a copy of their values, because a copy is exactly what drifted. It carries a denominator (a loop over nothing would pass) and asserts the refusing half, so it cannot be satisfied by accepting everything. /falsify: the mutation neutralises the derivation so the accepted set falls back to knownFakes alone, and the test goes red. Neutralised rather than deleted, because the harness requires every name in `find` to survive into `replace`. ## What the two replays measured **ioandev/scaleway-flatcar-k3s** @85cb05d: 11 resources applied against every earlier run's 9. The bucket and the uploaded qcow2 land in the local scaleway/fr-par door instead of taking a 403 from the real s3.fr-par.scw.cloud, and the object reads back sha256-identical. Re-plan `0 to change`, so scaleway_object carries no permanent diff. The chain now stops one resource later, on a refusal whose stated reason is no longer true: createSnapshot refuses an import because "this emulator does not serve Object Storage", while the object is in the door. Destroy not measured, and recorded as missing rather than assumed clean. **PhilippeChepy/platform** @84791e3: both SOS backup buckets reach the local de-muc-1 door, where every earlier run took a 403 from the real host. Nothing was configured for it: the aws provider builds that endpoint in code, and `feint run` intercepted it. The control-plane half is not comparable with the August figures, because the repository's lock file pins exoscale/exoscale at 0.49.0, below the v0.71.0 floor, so every /v2/ write is refused by user agent (#644) — the guard doing its job on a stranger's stack. One gap named rather than invented: aws provider 5.4.0 reads GetBucketEncryption after a create and the door answers 501; no recording of that shape exists for SOS. Both entries are written into examples/stacks/surveyed.md in the register's own form, with their harness inputs and what was NOT measured. No request left the station in either replay: feint run refuses every host that is not a door mounted by the emulator it was pointed at. mise run prepush green. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/stacks/surveyed.md | 51 +++++++++++++++++++ internal/cli/run.go | 41 +++++++++++++-- internal/cli/run_test.go | 43 ++++++++++++++++ .../run-lands-a-client-without-privilege.json | 7 +++ 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/examples/stacks/surveyed.md b/examples/stacks/surveyed.md index 875d13cc..2b9001a5 100644 --- a/examples/stacks/surveyed.md +++ b/examples/stacks/surveyed.md @@ -373,6 +373,37 @@ answered everything the modern provider sent. instance pool, the `local_file` behind it, and the one `exoscale_nlb_service` behind that: the service is blocked by the SOS branch, not by this emulator. +- **Replayed 2026-11-11, `feat/850-object-storage-release@0691306`, through + `feint run`** — **the SOS branch reaches the local door**, which is what this + replay was for. Both backup buckets exist in the `exoscale/de-muc-1` door + (`my-own-paas-…-etcd-backups.de-muc-1` and `…-vault-backups.de-muc-1`) where + every earlier run took a 403 from the real `sos-de-muc-1.exo.io`. The aws + provider's endpoint is built in code + (`https://sos-${local.platform_backup_zone}.exo.io`), so nothing was + configured: `feint run` intercepted it. + **The control-plane half is not comparable with the August figures**, and the + reason is a guard added since: the repository's own `.terraform.lock.hcl` + pins `exoscale/exoscale` at **0.49.0**, below the v0.71.0 floor, so every + `/v2/` write is refused by user agent (#644). Seven resources applied, all + provider-agnostic (tls, random, local). That refusal did its job on a + stranger's stack: without it, half of this apply would have reached Exoscale + and billed. + **One gap found, and not invented away**: after creating a bucket, aws + provider 5.4.0 reads `GetBucketEncryption`, and the door answers 501. No + recording of that shape exists for SOS, so it is named here rather than + guessed — the same treatment as the two unrecorded OOS discovery shapes. + **Harness inputs**: `locals.tf` from the published `locals.tf.example` with + `platform_admin_networks` pinned instead of `"auto"` (the auto path calls + ipconfig.me, which `feint run` refuses because it is not a door), and the + public `exoscale_prefixes.json` seeded into the local `ch-dk-2` door, since + `data.http.exoscale_ip_addresses` reads it from + `exoscale-prefixes.sos-ch-dk-2.exo.io` — a virtual-host address that + `feint run` intercepts through its wildcard. + **A defect in feint, found here and fixed in the same branch**: the sequence + both commands document — eval `feint env exoscale`, then `feint run` — was + refused by its own binary, because `knownFakes` was a second hand-written + copy of the credentials the packs publish and had drifted from them. See + `acceptedFakes` and `TestEveryPublishedFakeIsAcceptedByRun`. ### 3. PhilippeChepy/terraform-exoscale-vault — applied after a recorded edit, fully green @@ -578,6 +609,26 @@ platform entry. `CreateBucket` still routed by the provider to the real `s3.fr-par.scw.cloud` (403), Cloudflare external, `0 to change`, 9 destroyed. +- **Replayed 2026-11-11, `feat/850-object-storage-release@0691306`, through + `feint run`** — the first replay where this stack's Object Storage branch + does not leave the station. **11 applied** against every earlier run's 9: + `scaleway_object_bucket.flatcar` and `scaleway_object.flatcar_qcow2` now land + in the local `scaleway/fr-par` door instead of taking a 403 from the real + `s3.fr-par.scw.cloud`, and the uploaded qcow2 reads back **sha256-identical** + (65 536 bytes, `faef4dcf…`) through that door. Provider 2.91.0 under + `~> 2.0`, not the 2.x of August. Re-plan **`8 to add / 0 to change`**: the + bucket and the object round-trip, so `scaleway_object` carries no permanent + diff — unlike `aws_s3_object`, whose ETag comparison is documented in + docs/object-storage-routing.md. + **The chain now stops one resource later, and on a refusal whose stated + reason is no longer true**: `scaleway_instance_snapshot` with `import {}` + answers *"importing a snapshot reads Object Storage, which this emulator does + not serve"* (`internal/providers/scaleway/snapshots.go`), while the object it + would read is in the door, named by the request. The refusal is + unconditional; it predates the data plane. + **Destroy not measured.** The emulator was stopped before the teardown ran, + so this entry carries no destroy figure — recorded as missing rather than + assumed clean. ### 3. HealsCodes/ephemeral-devbox — applied in part diff --git a/internal/cli/run.go b/internal/cli/run.go index a26c462a..9d2b7a71 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -170,14 +170,17 @@ var credentialVariables = []string{ "EXOSCALE_API_KEY", "EXOSCALE_API_SECRET", } -// knownFakes are the deliberately public placeholders this repository ships, -// and the only credential values `feint run` passes on without --unsafe. +// knownFakes are deliberately public placeholders that no pack publishes, kept +// as a hand-written list for the clients that carry their own. // // A list and not a shape test, because every shape test gets this wrong in the // expensive direction: "twenty identical characters" would admit a real key // that happens to repeat, and "looks random" would refuse the fakes. The cost // of the list is that a fake it does not know is refused, which is the harmless // direction — the operator sees exactly which variable and can use a known one. +// +// It is NOT the whole accepted set: see acceptedFakes, and the defect that +// made the difference matter. var knownFakes = []string{ "AAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", @@ -187,17 +190,49 @@ var knownFakes = []string{ "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", } +// acceptedFakes is every credential value this binary itself tells an operator +// to export, plus the hand-written knownFakes no pack publishes. +// +// Derived rather than listed, and the reason is measured. The two lists were +// written separately and drifted: `feint env exoscale` published a 23-character +// EXOSCALE_API_KEY and a 40-character secret, while knownFakes held a +// 25-character key and neither secret. So the sequence both commands document — +// eval `feint env`, then `feint run` — was refused by its own binary, naming +// four variables the operator had never chosen. Found by replaying +// PhilippeChepy/platform from examples/stacks/surveyed.md, not by a test, which +// is why the fix removes the second list rather than correcting it. +// +// TestEveryPublishedFakeIsAcceptedByRun fails without this function. +func acceptedFakes() []string { + accepted := slices.Clone(knownFakes) + packs, err := packsFor(emulator.DefaultEnv()) + if err != nil { + // Fail closed: fewer accepted values means more refusals, and a + // refusal names the variable and offers --unsafe-real-credentials. + return accepted + } + for _, pack := range packs { + for name, value := range pack.Env(defaultEndpoint()).Vars { + if value != "" && slices.Contains(credentialVariables, name) { + accepted = append(accepted, value) + } + } + } + return accepted +} + // realCredentials names the credential variables carrying something other than // a known fake. Empty is the safe answer: no credentials set at all is a run // the emulator serves happily, since it checks no signature. func realCredentials(environ []string) []string { + accepted := acceptedFakes() var offenders []string for _, pair := range environ { name, value, ok := strings.Cut(pair, "=") if !ok || value == "" || !slices.Contains(credentialVariables, name) { continue } - if !slices.Contains(knownFakes, value) { + if !slices.Contains(accepted, value) { // The NAME only. Printing the value would put a real credential in // a terminal, a CI log and a bug report, which is the accident this // refusal exists to prevent. diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 8d70f88c..b4afe37a 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "sync" "testing" @@ -593,3 +594,45 @@ func (s *syncBuffer) String() string { defer s.mu.Unlock() return s.buf.String() } + +// TestEveryPublishedFakeIsAcceptedByRun is cited by acceptedFakes, and it is +// the test the hand-written list did not have. +// +// The defect it holds was found by replaying a stranger's stack +// (PhilippeChepy/platform, examples/stacks/surveyed.md): `feint env exoscale` +// publishes EXOSCALE_API_KEY and EXOSCALE_API_SECRET, `feint run` refused both, +// and the operator who followed the two commands' own notes was told their +// credentials were real. The two lists were written apart and drifted by two +// characters on one value and by a whole missing value on another. +// +// Asserted from the packs rather than from a copy of their values, because a +// copy is exactly what drifted. +func TestEveryPublishedFakeIsAcceptedByRun(t *testing.T) { + packs, err := packsFor(emulator.DefaultEnv()) + if err != nil { + t.Fatalf("packs: %v", err) + } + seen := 0 + for _, pack := range packs { + for name, value := range pack.Env(defaultEndpoint()).Vars { + if value == "" || !slices.Contains(credentialVariables, name) { + continue + } + seen++ + if offenders := realCredentials([]string{name + "=" + value}); len(offenders) != 0 { + t.Errorf("%s publishes %s, and feint run calls it real: %v", + pack.Name(), name, offenders) + } + } + } + // A denominator, because a loop that iterates nothing passes silently and + // would have passed before the fix too. + if seen < 4 { + t.Fatalf("only %d published credential(s) examined; the packs stopped publishing them", seen) + } + // The refusing half, so this test cannot be satisfied by accepting + // everything: a value no pack publishes is still refused. + if offenders := realCredentials([]string{"SCW_SECRET_KEY=a-secret-no-pack-publishes-0001"}); len(offenders) != 1 { + t.Fatalf("an unpublished credential was not refused: %v", offenders) + } +} diff --git a/tools/falsify/specs/run-lands-a-client-without-privilege.json b/tools/falsify/specs/run-lands-a-client-without-privilege.json index ab286ab3..a9a36d9f 100644 --- a/tools/falsify/specs/run-lands-a-client-without-privilege.json +++ b/tools/falsify/specs/run-lands-a-client-without-privilege.json @@ -63,6 +63,13 @@ "find": "\t\tif name, _, ok := strings.Cut(pair, \"=\"); ok && name == \"AWS_CA_BUNDLE\" {", "replace": "\t\tif name, _, ok := strings.Cut(pair, \"=\"); ok && name == \"AWS_CA_BUNDLE\" && false {", "test": "TestRunGivesTheChildOnlyTheFourMeasuredVariables" + }, + { + "label": "the derivation stops appending, so the accepted set falls back to the hand-written knownFakes alone: `feint env exoscale` publishes a 23-character EXOSCALE_API_KEY and a 40-character secret, that list held a 25-character key and neither secret, and the sequence both commands document was refused by its own binary. Neutralised rather than deleted because the harness requires every name in `find` to survive into `replace`, and the defect was a second list rather than a missing check", + "file": "internal/cli/run.go", + "find": "\t\t\tif value != \"\" && slices.Contains(credentialVariables, name) {\n\t\t\t\taccepted = append(accepted, value)", + "replace": "\t\t\tif value != \"\" && slices.Contains(credentialVariables, name) && false {\n\t\t\t\taccepted = append(accepted, value)", + "test": "TestEveryPublishedFakeIsAcceptedByRun" } ], "note": "The subject is `feint run` (#849): landing a client whose cloud endpoint is compiled in on the local emulator, with no sudo, no network namespace and no system mutation. Measured, not assumed: the published Scaleway Terraform provider honours HTTP_PROXY, HTTPS_PROXY and SSL_CERT_FILE, which is why the witness in object_storage_terraform_conformance_test.go uses its namespace as an ESCAPE CHECK rather than as the mechanism. The command therefore needs nothing privileged.\n\nThe design decision these mutations guard is that the command takes NO hostname. An earlier draft took `--forward host=target`, and the operator typing the hostname is the operator able to mistype it — a mistyped cloud host is not intercepted, it is resolved, so the run leaves for a third party carrying whatever credentials it holds, and the failure is silent because a request that reached the real cloud looks exactly like one that worked. Instead the mounted doors come from the emulator (GET /_feint/s3, validated by fetchS3) and each door's cloud authority comes from the pack in this binary, which is also the split s3.Profile requires: an instance fact may come from a report, a wire dialect may not. The set of possible targets is therefore the set of doors that exist, and there is no spelling of this command that sends traffic off loopback.\n\nMutation 5 carries the correction of a comment this file got wrong first: it claimed the file check held the signal relay. It does not, because the deferred removal runs regardless. Writing that down was necessary — a test believed for the wrong reason is the defect this repository measures most often.\n\nMutations 6 and 7 close #849's own two remaining asks. The first is the falsification that issue names in as many words: a run that forgets its cleanup. The second is why TestDoctorSeesARunInFlightAsItsOwner had to exist at all — TestDoctorTellsALeftoverAuthorityFromARunInFlight writes its own directories, so it would stay green if `feint run` stopped recording an owner altogether, and a diagnostic agreeing with a fixture rather than with its producer is a diagnostic that goes blind in silence.\n\nMutation 8 is the one this spec most needed and least expected. The command's first proof used curl against the bare authority, which is PATH-STYLE, so it passed while the virtual-host form every S3 client actually uses — ., Terraform included — was refused at the allow-list and never tunnelled. The measurement that caught it was reading the existing Terraform witness, which declares the bucket authority explicitly beside the bare one. Widening the allow-list cannot widen what is SERVED: the wildcard decides only what is intercepted, and LocalTransport still refuses a leading label that is not a valid bucket name, so interception is deliberately wider than service and a nonsense bucket gets a local refusal instead of a trip to the cloud.\n\nMutation 9 holds a variable's ABSENCE, which is the only reason the environment is asserted as a list. An earlier draft set AWS_CA_BUNDLE on the reasoning that an S3 client would want it, and that reasoning was never measured: the published Scaleway provider hands its SDK a prebuilt *http.Client, so a CA bundle the SDK is asked to install on it has nowhere to go, and the apply died on a message about transports. Removing it made the same apply complete with 139,264 identical bytes in the local byte store. The command also strips the variable from an INHERITED environment, which is the one thing it edits of the operator's own configuration, because no value of it can help: the station's bundle does not contain this run's freshly minted authority." From 56179a2c860b93bb6d154ae3fd3a34d94c25dda9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Wed, 11 Nov 2026 09:22:47 +0100 Subject: [PATCH 26/28] feat(scaleway): a snapshot import reads the object, so a surveyed stack applies whole (#850, #262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ioandev/scaleway-flatcar-k3s, from examples/stacks/surveyed.md, now applies **19 resources of 19 with a second plan reporting no changes**, against 9 at every replay since August. It is #850's first Done-means criterion — one surveyed Scaleway stack that no longer escapes to real Object Storage — held by a stranger's stack rather than by an example this repository wrote. Two defects stood between 9 and 19, and neither was visible to a unit test. ## A refusal that outlived its reason `instance/v1/API.CreateSnapshot` with `{bucket, key}` was refused unconditionally, telling the client that "importing a snapshot reads Object Storage, which this emulator does not serve" — while the object sat in the local door, named by the request being refused. The stack had just uploaded it through `scaleway_object`. Seven resources hung behind that one call, every one a plain dependency: the image on the snapshot, three servers on the image, three private NICs on the servers. So the fix is worth more than its size. It is served now, and it checks rather than assumes: - the object is there: its size becomes the snapshot's size; - the object is absent: refused, as upstream refuses it; - no byte store armed: refused, naming `--objectstore` and `--s3-doors` rather than the limits page, because the operator can act on that. The refusal's DIALECT stays unmeasured and docs/limits.md says so: the real API's shape for a missing object is not recorded here, so it borrows this route's own invalid_arguments envelope. ## An empty base volume, which only the stack could find The first version left the route's default `{id, name}` in place. Every unit test passed. The stack's second plan then answered `2 to add, 0 to change, 2 to destroy`: the provider reads an empty base as `volume_id "fr-par-1/"` and calls that `forces replacement`, taking the image cut from it along. An imported snapshot has no parent, so the field has to say null. This is the create-then-read round trip the pack skill names, and the reason the measurement had to be a real client and not an assertion. ## One reader, two products block/v1's import is wired to the same reader. #807 documented the divergence in as many words — "the real API refuses an object that does not exist, and this handler does not check the opt-in byte store" — and leaving it there while fixing the instance side is precisely the shape CLAUDE.md warns about: the same lifecycle written twice and fixed once. With a store armed, both read the object and both refuse its absence. With none, block keeps #807's intention and instance refuses, because their prior contracts differed and silently aligning a served route would change behaviour for an operator who armed nothing. That difference is asserted, not assumed. `s3.StoreNamespace` replaces the hand-composed door scope so a pack cannot drift from what a door writes — a drift that would be silent, the pack simply never finding an object that is there. Its holding test spells the scope out by hand on purpose: a witness that computed it the same way as the subject would agree with any format. ## Tests and falsification - TestASnapshotImportTakesTheSizeOfTheObjectItNames - TestASnapshotImportIsRefusedWithoutTheObject (three absent populations, plus the unarmed refusal naming its flags) - TestAnImportedSnapshotHasNoBaseVolumeAndReplansEmpty, which also asserts that an ordinary snapshot still HAS a base volume, so it cannot be satisfied by nulling the field for everybody - TestABlockImportRefusesAnObjectTheDoorDoesNotHold tools/falsify/specs/a-snapshot-import-reads-the-object.json: three mutations, all compiled, all biting. docs/limits.md gains the instance-import section, closes the #807 divergence on the armed side, and records a third measurement from the same replay: a door answers 501 on `encryption`, which AWS provider 6.72.0 never reads and 5.4.0 reads after every bucket create. Both ends measured, so the limit is written with its version rather than as a blanket. mise run prepush green. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.fr.md | 2 +- README.md | 2 +- docs/limits.md | 64 +++- examples/stacks/surveyed.md | 45 +-- internal/providers/scaleway/block.go | 25 +- .../scaleway/snapshot_import_test.go | 303 ++++++++++++++++++ internal/providers/scaleway/snapshots.go | 115 ++++++- internal/providers/scaleway/snapshots_test.go | 15 +- internal/s3/doors.go | 21 +- .../a-snapshot-import-reads-the-object.json | 27 ++ 10 files changed, 569 insertions(+), 50 deletions(-) create mode 100644 internal/providers/scaleway/snapshot_import_test.go create mode 100644 tools/falsify/specs/a-snapshot-import-reads-the-object.json diff --git a/README.fr.md b/README.fr.md index d11ee515..7738e695 100644 --- a/README.fr.md +++ b/README.fr.md @@ -31,7 +31,7 @@ > > **Prouvé** : 384 des 409 opérations montées sont pilotées par un vrai client, à chaque pull request. `scw`, `octl`, `exo`, Terraform et OpenTofu tournent contre l'émulateur en CI, et les machines démarrent réellement : connexion ssh sur le compte par défaut de chaque provider, subnets isolés, pare-feu qui filtre. La chaîne complète est décrite dans [docs/conformance.md](docs/conformance.md). > -> **Pas prouvé** : quotas, prix, capacité réelle, validation des identifiants, authentification, cohérence à terme. Les 58 sections de [docs/limits.md](docs/limits.md) disent chacune ce qu'elle coûte. Un émulateur avec un seul compte implicite et aucune grille tarifaire devrait inventer ces chiffres, et quelqu'un agirait dessus. +> **Pas prouvé** : quotas, prix, capacité réelle, validation des identifiants, authentification, cohérence à terme. Les 59 sections de [docs/limits.md](docs/limits.md) disent chacune ce qu'elle coûte. Un émulateur avec un seul compte implicite et aucune grille tarifaire devrait inventer ces chiffres, et quelqu'un agirait dessus. > > **Inconnu** : 25 opérations sont montées et n'ont jamais été pilotées par un client. Chacune dit pourquoi aucun client officiel ne l'atteint, à la route et dans [docs/routes.md](docs/routes.md). Elles sont comptées plutôt qu'escamotées, une par une, dans [coverage/evidence.json](coverage/evidence.json). > diff --git a/README.md b/README.md index 37c83055..650345bb 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ > > **Proven**: 384 of the 409 mounted operations are driven by a real client, on every pull request: each pack's own official CLI, and an infrastructure engine wherever a pack admits one, all of them running against the emulator in CI, and machines really boot: an ssh login on each provider's own default account, isolated subnets, a firewall that filters. The whole chain is described in [docs/conformance.md](docs/conformance.md). > -> **Not proven**: quotas, prices, real capacity, identifier validation, authentication, eventual consistency. The 58 sections of [docs/limits.md](docs/limits.md) each say what one costs. An emulator with a single implicit account and no price list would have to invent those figures, and somebody would act on them. +> **Not proven**: quotas, prices, real capacity, identifier validation, authentication, eventual consistency. The 59 sections of [docs/limits.md](docs/limits.md) each say what one costs. An emulator with a single implicit account and no price list would have to invent those figures, and somebody would act on them. > > **Unknown**: 25 operations are mounted and have never been driven by a client. Every one of them states why no official client reaches it, at the route and in [docs/routes.md](docs/routes.md). They are counted rather than glossed, one by one, in [coverage/evidence.json](coverage/evidence.json). > diff --git a/docs/limits.md b/docs/limits.md index 26595e76..65af05a3 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -33,6 +33,26 @@ default), including peak reservation; abort is not GC. Opaque SHA-256 part tags are not generic object ETags. Completion Location consumers are not certified; same-key pagination markers require a live session. Exo's bucket remover does not paginate uploads, so removal beyond 1000 sessions is not certified. +### Bucket discovery after a create, and the provider version that decides it + +A door answers `acl`, `versioning`, `tagging` and `cors`; it answers **501** on +`encryption`, `location` and `policy`, because no recording of those shapes +exists and this project does not invent them. + +Which of those a client meets depends on the provider version, and both ends +are measured rather than reasoned: + +| client | after `aws_s3_bucket` create | verdict | +|---|---|---| +| AWS provider **6.72.0** | reads no encryption | the bucket applies; `examples/stacks/exoscale` does this in CI | +| AWS provider **5.4.0** | reads `GetBucketEncryption` | `Error: getting S3 Bucket encryption: NotImplemented` | + +Measured 2026-11-11 by replaying PhilippeChepy/platform, whose pinned lock file +carries 5.4.0 (examples/stacks/surveyed.md). So "Terraform creates a bucket on +the local door" is true of the current provider and false of that one, and a +stack pinned below 6.x stops on a discovery call rather than on anything about +bytes. + ### A SOS bucket in Terraform, and why no object beside it `examples/stacks/exoscale` creates a bucket against the local SOS door, which @@ -153,6 +173,33 @@ one product on this one client. The escape-path section below (#280) carries the full measured list, what warns before the run, and the egress cuts that were tested rather than described. +## An instance snapshot imported from Object Storage, and what it checks (#850) + +`instance/v1/API.CreateSnapshot` takes `{bucket, key}`, and that path was +refused outright while nothing here held bytes. The refusal outlived its +reason: with a door armed it told the client that "importing a snapshot reads +Object Storage, which this emulator does not serve" while the object sat in +that door, named by the request being refused. Found on 2026-11-11 by replaying +ioandev/scaleway-flatcar-k3s, whose image pipeline uploads a qcow2 and imports +it; see examples/stacks/surveyed.md. + +It is served now, and it checks: + +- **the object is there**: its size becomes the snapshot's size, and the + snapshot carries **no base volume** — a null, not an empty one. An empty + `{id, name}` reads back as `volume_id "fr-par-1/"` and makes every later plan + answer `forces replacement` on the snapshot and on the image cut from it. + Measured: that is what the first version did, and the stack's second plan + said `2 to add, 0 to change, 2 to destroy` until it was fixed. +- **the object is absent**: refused. The dialect is this route's own + `invalid_arguments` envelope, because the real API's shape for a missing + object is unrecorded, exactly as for the block import below. +- **no byte store armed**: refused, naming `--objectstore` and `--s3-doors` + rather than the limits page, because the operator can act on it. + +With that, the whole surveyed stack applies: **19 resources of 19, and a second +plan reporting no changes**, its qcow2 sha256-identical in the local door. + ## An imported block snapshot is an intention, and the object is never read (#807) `block/v1/API.ImportSnapshotFromObjectStorage` and its `block/v1alpha1` twin are @@ -167,11 +214,18 @@ carrying the name and project the client gave — the request declares no tags, unlike the create beside it — with no parent volume and a size of zero, because this handler does not read object bytes. -The divergence, stated rather than hidden: the real API refuses an object that -does not exist, and this handler does not check the opt-in byte store. The -snapshot-import route is not wired to the S3 gateway, so an import naming an object nobody uploaded -answers success here and answers an error upstream. A plan that depends on that -refusal is not covered. +The divergence that stood here is closed on one side and stated on the other. +**With a byte store armed**, the handler now reads the object through the door +of its own region: the snapshot takes the object's size, and an import naming +an object nobody uploaded is refused, as upstream refuses it. **With no byte +store**, the intention above stands unchanged — nothing can be read, so nothing +is asserted about bytes, and a plan depending on the refusal is still not +covered in that configuration. + +What remains unmeasured is the refusal's **dialect**, not its existence: the +real API's status and body for a missing object are not recorded here, so the +refusal borrows this route's own `invalid_arguments` envelope. A client that +branches on the exact error shape is not covered. One thing neither side of #807 could answer: which status a real import reports while it is in flight. The block contract lists `creating` and `available`, no diff --git a/examples/stacks/surveyed.md b/examples/stacks/surveyed.md index 2b9001a5..e84c6c87 100644 --- a/examples/stacks/surveyed.md +++ b/examples/stacks/surveyed.md @@ -609,26 +609,31 @@ platform entry. `CreateBucket` still routed by the provider to the real `s3.fr-par.scw.cloud` (403), Cloudflare external, `0 to change`, 9 destroyed. -- **Replayed 2026-11-11, `feat/850-object-storage-release@0691306`, through - `feint run`** — the first replay where this stack's Object Storage branch - does not leave the station. **11 applied** against every earlier run's 9: - `scaleway_object_bucket.flatcar` and `scaleway_object.flatcar_qcow2` now land - in the local `scaleway/fr-par` door instead of taking a 403 from the real - `s3.fr-par.scw.cloud`, and the uploaded qcow2 reads back **sha256-identical** - (65 536 bytes, `faef4dcf…`) through that door. Provider 2.91.0 under - `~> 2.0`, not the 2.x of August. Re-plan **`8 to add / 0 to change`**: the - bucket and the object round-trip, so `scaleway_object` carries no permanent - diff — unlike `aws_s3_object`, whose ETag comparison is documented in - docs/object-storage-routing.md. - **The chain now stops one resource later, and on a refusal whose stated - reason is no longer true**: `scaleway_instance_snapshot` with `import {}` - answers *"importing a snapshot reads Object Storage, which this emulator does - not serve"* (`internal/providers/scaleway/snapshots.go`), while the object it - would read is in the door, named by the request. The refusal is - unconditional; it predates the data plane. - **Destroy not measured.** The emulator was stopped before the teardown ran, - so this entry carries no destroy figure — recorded as missing rather than - assumed clean. +- **Replayed 2026-11-11, `feat/850-object-storage-release`, through + `feint run`** — **19 resources of 19, and a second plan reporting no + changes**, against 9 at every earlier replay. The first run where this stack + goes all the way, and the first where its Object Storage branch does not + leave the station: the bucket and the qcow2 land in the local + `scaleway/fr-par` door instead of taking a 403 from the real + `s3.fr-par.scw.cloud`, and the object reads back sha256-identical + (65 536 bytes, `faef4dcf…`). Provider 2.91.0 under `~> 2.0`, not the 2.x of + August. Cloudflare stays out of the plan, its token being unset. + **It took two findings and two fixes, in this order.** The first run after + the data plane applied **11**, and stopped on + `scaleway_instance_snapshot.flatcar[0]`: a refusal whose stated reason had + died, *"importing a snapshot reads Object Storage, which this emulator does + not serve"*, answered while the object sat in the door, named by the request + being refused. Seven resources hung behind that one call, every one of them + a plain dependency — the image on the snapshot, the three servers on the + image, the three private NICs on the servers. Serving the import took it to + 19, and then the **second plan said `2 to add, 0 to change, 2 to destroy`**: + the imported snapshot carried the route's default empty base volume, which + the provider reads back as `volume_id "fr-par-1/"` and treats as + `forces replacement`, taking the image with it. An imported snapshot has no + parent, so the field had to say null. Both are held by tests and by the + mutations in `tools/falsify/specs/a-snapshot-import-reads-the-object.json`. + **Not measured: the destroy.** The emulator was stopped before a teardown + ran. Recorded as missing rather than assumed clean. ### 3. HealsCodes/ephemeral-devbox — applied in part diff --git a/internal/providers/scaleway/block.go b/internal/providers/scaleway/block.go index 1d1db239..5ebd2c6a 100644 --- a/internal/providers/scaleway/block.go +++ b/internal/providers/scaleway/block.go @@ -676,6 +676,23 @@ func (p *Pack) importBlockSnapshotFromObjectStorage(w http.ResponseWriter, r *ht return } project, _ := projectOf(textPtr(req.ProjectID)) + // Size from the object when a byte store is armed, zero when none is. + // #807 documented the divergence this closes in as many words: "the real + // API refuses an object that does not exist, and this handler does not + // check the opt-in byte store". It does now, through the same reader the + // instance import uses — one behaviour, two products, written once, + // because this repository has already paid for the same lifecycle written + // twice and fixed on one side. Unarmed, the intention stands: that is + // #807's own contract, and nothing here can read bytes nobody serves. + // TestABlockImportRefusesAnObjectTheDoorDoesNotHold fails without this. + imported := uint64(0) + if p.env.Objects != nil { + size, ok := p.importedObjectSize(r.Context(), w, regionOfZone(zone), req.Bucket, req.Key) + if !ok { + return + } + imported = size + } now := p.env.Now() res := resource.New(p.env.NewID(), kindBlockSnapshot, resource.Tenant{Provider: Name, Project: project, Zone: zone}, "available", now) res.Attrs = map[string]any{ @@ -685,10 +702,10 @@ func (p *Pack) importBlockSnapshotFromObjectStorage(w http.ResponseWriter, r *ht // field is present and empty rather than absent — a client reading a // snapshot gets the same shape whichever door made it. "tags": []string{}, - // No parent volume and no bytes: the snapshot was imported from an - // object this emulator does not hold, so it carries the inputs the - // client named and nothing it could not have observed. - "size": uint64(0), + // No parent volume. The size is the object's when a door holds it, and + // zero when no byte store is armed: either way it is a number the + // client could have observed, never one invented here. + "size": imported, "zone": zone, "public": false, "parent_volume": nil, diff --git a/internal/providers/scaleway/snapshot_import_test.go b/internal/providers/scaleway/snapshot_import_test.go new file mode 100644 index 00000000..14d5cd65 --- /dev/null +++ b/internal/providers/scaleway/snapshot_import_test.go @@ -0,0 +1,303 @@ +package scaleway_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stephrobert/feint/internal/core/emulator" + "github.com/stephrobert/feint/internal/core/objectstore" + "github.com/stephrobert/feint/internal/core/store" + "github.com/stephrobert/feint/internal/providers/scaleway" + "github.com/stephrobert/feint/internal/s3" +) + +// Importing an instance snapshot from Object Storage, now that there is some. +// +// The subject is the refusal this replaced, and WHY it had to go rather than +// be relaxed: it was unconditional, and it told the client that "importing a +// snapshot reads Object Storage, which this emulator does not serve" while the +// object sat in the local door, named by the request being refused. +// +// It was found by replaying a stranger's stack — ioandev/scaleway-flatcar-k3s, +// recorded in examples/stacks/surveyed.md — whose image pipeline uploads a +// qcow2 through scaleway_object and then imports it. The upload landed in the +// door; the import refused on the premise the upload had just falsified. Eight +// resources of that stack sit behind this one call. + +// storeBackedServer is newTestServer with a byte store armed, because the +// property under test is what the handler does when there IS one. +func storeBackedServer(t testing.TB) (*httptest.Server, *objectstore.Store) { + t.Helper() + + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + objects, err := objectstore.Open(t.Context(), filepath.Join(root, "objects")) + if err != nil { + t.Fatalf("open the byte store: %v", err) + } + t.Cleanup(func() { + if err := objects.Close(); err != nil { + t.Error(err) + } + }) + + var seq int + env := &emulator.Env{ + Store: store.New(), + Objects: objects, + Now: func() time.Time { return time.Unix(1700000000, 0).UTC() }, + NewID: func() string { + seq++ + return fmt.Sprintf("00000000-0000-4000-8000-%012d", seq) + }, + } + srv, err := emulator.NewServer(env, scaleway.New(env)) + if err != nil { + t.Fatalf("build emulator: %v", err) + } + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + return ts, objects +} + +// TestASnapshotImportTakesTheSizeOfTheObjectItNames is cited by +// importedSnapshotSize. +// +// The size is the assertion and not an afterthought: it is the ONE fact about +// the imported object a client can check, and a snapshot that answered a +// default size would be the half-success this project exists to avoid — green +// here, and a wrong disk there. +// +// The bucket is addressed through s3.StoreNamespace rather than spelled out, +// so that a door that changes its scope breaks this test instead of silently +// making the handler find nothing. +func TestASnapshotImportTakesTheSizeOfTheObjectItNames(t *testing.T) { + ts, objects := storeBackedServer(t) + + const bucket, key = "flatcar-image-import", "flatcar.qcow2" + scope := s3.StoreNamespace("scaleway", "fr-par") + "/" + bucket + if err := objects.CreateBucket(t.Context(), scope); err != nil { + t.Fatalf("create the bucket: %v", err) + } + const body = "not a real qcow2, and the size is what the import reads" + if _, err := objects.PutObject(t.Context(), scope, key, + strings.NewReader(body), objectstore.PutOptions{Size: int64(len(body))}); err != nil { + t.Fatalf("put the object: %v", err) + } + + status, got := do(t, ts, "POST", zoneURL+"/snapshots", + `{"name":"flatcar-stable","bucket":"`+bucket+`","key":"`+key+`"}`) + if status != http.StatusCreated { + t.Fatalf("import of an object that is there: expected 201, got %d (%v)", status, got) + } + snapshot, _ := got["snapshot"].(map[string]any) + if snapshot == nil { + t.Fatalf("no snapshot in the answer: %v", got) + } + if size, _ := snapshot["size"].(float64); int(size) != len(body) { + t.Errorf("the snapshot took size %v, not the object's %d", snapshot["size"], len(body)) + } + if name, _ := snapshot["name"].(string); name != "flatcar-stable" { + t.Errorf("the snapshot lost its name: %v", snapshot["name"]) + } + + // And it is readable afterwards, because an import that answers 201 and + // stores nothing is the same lie in a slower form. + _, listed := do(t, ts, "GET", zoneURL+"/snapshots", "") + if items, _ := listed["snapshots"].([]any); len(items) != 1 { + t.Errorf("the imported snapshot was not recorded: %v", listed) + } +} + +// TestASnapshotImportIsRefusedWithoutTheObject is cited by +// importedSnapshotSize, and it is the half that keeps the fix from becoming a +// rubber stamp. +// +// Two populations, because they are refused for DIFFERENT reasons and only one +// of them is about this run's configuration: +// +// - a byte store is armed and the object is not in it: the real API refuses +// an object that is not there, and so does this; +// - no byte store at all: nothing can be read, so nothing is asserted about +// bytes. The message names the flags rather than the limits page, because +// the operator can fix this one. +func TestASnapshotImportIsRefusedWithoutTheObject(t *testing.T) { + armed, objects := storeBackedServer(t) + scope := s3.StoreNamespace("scaleway", "fr-par") + "/present" + if err := objects.CreateBucket(t.Context(), scope); err != nil { + t.Fatalf("create the bucket: %v", err) + } + + for _, probe := range []struct{ name, body, expect string }{ + {"an absent key in a bucket that exists", + `{"name":"x","bucket":"present","key":"nothing.qcow2"}`, "nothing to import"}, + {"a bucket that does not exist", + `{"name":"x","bucket":"absent","key":"disk.qcow2"}`, "nothing to import"}, + {"an import naming no key at all", + `{"name":"x","bucket":"present"}`, ""}, + } { + t.Run(probe.name, func(t *testing.T) { + status, got := do(t, armed, "POST", zoneURL+"/snapshots", probe.body) + if status != http.StatusBadRequest { + t.Fatalf("expected 400, got %d (%v)", status, got) + } + if probe.expect != "" && !strings.Contains(fmt.Sprint(got), probe.expect) { + t.Errorf("the refusal does not say what is missing: %v", got) + } + }) + } + + // Nothing was recorded by any of the three. + _, listed := do(t, armed, "GET", zoneURL+"/snapshots", "") + if items, _ := listed["snapshots"].([]any); len(items) != 0 { + t.Errorf("a refused import was recorded anyway: %v", listed) + } + + // And with no byte store, the refusal names the flags that arm one rather + // than claiming the emulator cannot do this at all. + unarmed := newTestServer(t) + status, got := do(t, unarmed, "POST", zoneURL+"/snapshots", + `{"name":"x","bucket":"present","key":"disk.qcow2"}`) + if status != http.StatusBadRequest { + t.Fatalf("unarmed import: expected 400, got %d (%v)", status, got) + } + for _, want := range []string{"--objectstore", "--s3-doors", "scaleway/fr-par"} { + if !strings.Contains(fmt.Sprint(got), want) { + t.Errorf("the unarmed refusal does not name %q: %v", want, got) + } + } +} + +// TestAnImportedSnapshotHasNoBaseVolumeAndReplansEmpty is cited by +// createSnapshot, and it holds a defect the unit tests could not have found. +// +// The first version of the import left the route's default base volume in +// place, an empty {id, name}. Everything passed: the snapshot was created, its +// size was right, it read back. Then the stack replay answered +// `2 to add, 0 to change, 2 to destroy` on its second plan, because the +// provider reads an empty base as volume_id "fr-par-1/" and that forces +// replacement of the snapshot and of the image cut from it. +// +// So this asserts the shape on BOTH sides of the round trip, which is the rule +// the provider pack lives by: whatever a create returns, the following read +// returns identically. A null here is not tidiness, it is the difference +// between an example that converges and one that churns two resources on every +// plan forever. +func TestAnImportedSnapshotHasNoBaseVolumeAndReplansEmpty(t *testing.T) { + ts, objects := storeBackedServer(t) + + const bucket, key = "images", "disk.qcow2" + scope := s3.StoreNamespace("scaleway", "fr-par") + "/" + bucket + if err := objects.CreateBucket(t.Context(), scope); err != nil { + t.Fatalf("create the bucket: %v", err) + } + if _, err := objects.PutObject(t.Context(), scope, key, + strings.NewReader("bytes"), objectstore.PutOptions{Size: 5}); err != nil { + t.Fatalf("put the object: %v", err) + } + + status, created := do(t, ts, "POST", zoneURL+"/snapshots", + `{"name":"imported","bucket":"`+bucket+`","key":"`+key+`"}`) + if status != http.StatusCreated { + t.Fatalf("import: expected 201, got %d (%v)", status, created) + } + snapshot, _ := created["snapshot"].(map[string]any) + base, present := snapshot["base_volume"] + if !present { + t.Fatalf("base_volume is absent from the create; the client expects the field: %v", snapshot) + } + if base != nil { + t.Errorf("an imported snapshot answered a base volume %v; an empty one reads back "+ + "as volume_id \"fr-par-1/\" and forces replacement on every later plan", base) + } + + // The read, which is the half that actually drives the diff. + id, _ := snapshot["id"].(string) + status, read := do(t, ts, "GET", zoneURL+"/snapshots/"+id, "") + if status != http.StatusOK { + t.Fatalf("read back: expected 200, got %d (%v)", status, read) + } + again, _ := read["snapshot"].(map[string]any) + if base, present := again["base_volume"]; !present || base != nil { + t.Errorf("the read does not agree with the create about base_volume: %v", again) + } + + // And a snapshot taken of a real volume still HAS one, so this cannot be + // satisfied by nulling the field for everybody. + ofVolume := snapshotOfVolume(t, ts, "of-a-volume", instanceVolumeOf(t, ts)) + status, normal := do(t, ts, "GET", zoneURL+"/snapshots/"+ofVolume, "") + if status != http.StatusOK { + t.Fatalf("read the ordinary snapshot: expected 200, got %d (%v)", status, normal) + } + snap, _ := normal["snapshot"].(map[string]any) + if base, _ := snap["base_volume"].(map[string]any); base == nil || base["id"] == "" { + t.Errorf("an ordinary snapshot lost its base volume: %v", snap) + } +} + +// TestABlockImportRefusesAnObjectTheDoorDoesNotHold is cited by +// importBlockSnapshotFromObjectStorage and by importedObjectSize. +// +// It exists because the two imports of this pack must not disagree about what +// they do when a byte store IS armed. The instance one was fixed first, after +// a stranger's stack walked into it; writing the same reading a second time +// for block is exactly the shape this repository has already paid for, so the +// reader is shared and this test holds the block end of it. +// +// What the two still do differently is deliberate and asserted below: with no +// byte store, the block import records an intention with a size of zero, which +// is #807's own contract, while the instance one refuses. Their prior +// contracts differed, and silently aligning them would change a served route's +// behaviour for an operator who armed nothing. +func TestABlockImportRefusesAnObjectTheDoorDoesNotHold(t *testing.T) { + ts, objects := storeBackedServer(t) + + const bucket, key = "disks", "root.qcow2" + scope := s3.StoreNamespace("scaleway", "fr-par") + "/" + bucket + if err := objects.CreateBucket(t.Context(), scope); err != nil { + t.Fatalf("create the bucket: %v", err) + } + const body = "seven!!" + if _, err := objects.PutObject(t.Context(), scope, key, + strings.NewReader(body), objectstore.PutOptions{Size: int64(len(body))}); err != nil { + t.Fatalf("put the object: %v", err) + } + + // Armed and present: the size is the object's, not zero. + status, got := do(t, ts, "POST", blockURL+"/snapshots/import-from-object-storage", + `{"name":"from-object","bucket":"`+bucket+`","key":"`+key+`"}`) + if status != http.StatusCreated && status != http.StatusOK { + t.Fatalf("import of an object that is there: got %d (%v)", status, got) + } + snapshot, _ := got["snapshot"].(map[string]any) + if snapshot == nil { + snapshot = got + } + if size, _ := snapshot["size"].(float64); int(size) != len(body) { + t.Errorf("the block import took size %v, not the object's %d", snapshot["size"], len(body)) + } + + // Armed and absent: refused, because the real API refuses it and an + // intention recorded here would be a snapshot of bytes nobody has. + status, got = do(t, ts, "POST", blockURL+"/snapshots/import-from-object-storage", + `{"name":"from-nothing","bucket":"`+bucket+`","key":"absent.qcow2"}`) + if status != http.StatusBadRequest { + t.Fatalf("import of an absent object: expected 400, got %d (%v)", status, got) + } + + // Unarmed: #807's intention stands, unchanged. + unarmed := newTestServer(t) + status, got = do(t, unarmed, "POST", blockURL+"/snapshots/import-from-object-storage", + `{"name":"intention","bucket":"`+bucket+`","key":"`+key+`"}`) + if status != http.StatusCreated && status != http.StatusOK { + t.Fatalf("unarmed import: expected the #807 intention, got %d (%v)", status, got) + } +} diff --git a/internal/providers/scaleway/snapshots.go b/internal/providers/scaleway/snapshots.go index ba28fb8f..2c3abfcb 100644 --- a/internal/providers/scaleway/snapshots.go +++ b/internal/providers/scaleway/snapshots.go @@ -1,12 +1,14 @@ package scaleway import ( + "context" "net/http" "strings" "time" "github.com/stephrobert/feint/internal/core/emulator" "github.com/stephrobert/feint/internal/core/resource" + "github.com/stephrobert/feint/internal/s3" ) // Snapshots, as control-plane records. @@ -40,10 +42,9 @@ type createSnapshotRequest struct { Organization *string `json:"organization"` Project *string `json:"project"` VolumeType string `json:"volume_type"` - // Bucket and Key belong to the import-from-Object-Storage path, which stays - // declined: docs/limits.md says why Object Storage is not emulated, and a - // snapshot restored from a bucket that does not exist would answer success - // about bytes nobody has. + // Bucket and Key carry the import-from-Object-Storage path. It was declined + // while nothing here held bytes; importedSnapshotSize serves it now, and + // still refuses a snapshot restored from an object nobody has. Bucket *string `json:"bucket"` Key *string `json:"key"` } @@ -70,19 +71,27 @@ func (p *Pack) createSnapshot(w http.ResponseWriter, r *http.Request) { writeInvalidArguments(w, ArgumentError{ArgumentName: "name", Reason: "required"}) return } - if req.Bucket != nil || req.Key != nil { - writeInvalidArguments(w, ArgumentError{ - ArgumentName: "bucket", - Reason: "constraint", - HelpMessage: "importing a snapshot reads Object Storage, which this emulator does not serve; see docs/limits.md", - }) - return - } - volumeType := req.VolumeType size := uint64(rootVolumeSize) base := map[string]any{"id": "", "name": ""} - if req.VolumeID != nil && *req.VolumeID != "" { + imported := req.Bucket != nil || req.Key != nil + if imported { + bytes, ok := p.importedSnapshotSize(r.Context(), w, zone, req) + if !ok { + return + } + size = bytes + // No base volume, and NULL rather than an empty one. Measured on the + // replay: an empty {id, name} renders base_volume with an empty id, + // the Scaleway provider reads it back as volume_id \"fr-par-1/\" and + // every later plan answers `forces replacement`. An imported snapshot + // has no parent, so the field has to say so, the way block/v1\u0027s + // import already answers parent_volume: nil. + // TestAnImportedSnapshotHasNoBaseVolumeAndReplansEmpty fails without + // this. + base = nil + } + if !imported && req.VolumeID != nil && *req.VolumeID != "" { // instance/v1 alone, and the resource name in the refusal says which // product answered (#648). // @@ -392,3 +401,81 @@ func slicePtr(s *[]string) []string { } return *s } + +// importedSnapshotSize resolves the object an import names and answers its +// size in bytes. +// +// The refusal this replaces was unconditional, and its stated reason died with +// the data plane: it told the client that "importing a snapshot reads Object +// Storage, which this emulator does not serve" while the object sat in the +// local door, named by the very request being refused. Found by replaying a +// stranger's stack, ioandev/scaleway-flatcar-k3s in examples/stacks/surveyed.md, +// whose image pipeline uploads a qcow2 and then imports it: the upload landed, +// and the import refused on a premise the upload had just falsified. +// +// Three answers, and the one that matters is the middle one: +// +// - no byte store armed: refused, naming the flags that arm it. Nothing can +// be read, and a snapshot sized from nothing is the half-success this +// project exists to avoid. +// - the object is absent: refused. The real API refuses an object that is +// not there; what this emulator does NOT know is the exact shape it uses, +// so the refusal borrows this route's own invalid_arguments envelope and +// docs/limits.md says that the dialect is unmeasured. +// - the object is there: its size becomes the snapshot's size, which is the +// one fact about it a client can check. +// +// The bucket is addressed through s3.StoreNamespace rather than by composing +// the scope here, because a pack that spells the door's scope itself finds +// nothing the day the door changes it, and finds it silently. +// +// TestASnapshotImportTakesTheSizeOfTheObjectItNames and +// TestASnapshotImportIsRefusedWithoutTheObject fail without this. +func (p *Pack) importedSnapshotSize(ctx context.Context, w http.ResponseWriter, + zone string, req createSnapshotRequest) (uint64, bool) { + bucket, key := textPtr(req.Bucket), textPtr(req.Key) + if bucket == "" { + writeInvalidArguments(w, ArgumentError{ArgumentName: "bucket", Reason: "required"}) + return 0, false + } + if key == "" { + writeInvalidArguments(w, ArgumentError{ArgumentName: "key", Reason: "required"}) + return 0, false + } + region := regionOfZone(zone) + if p.env.Objects == nil { + writeInvalidArguments(w, ArgumentError{ArgumentName: "bucket", Reason: "constraint", + HelpMessage: "importing a snapshot reads Object Storage, and this run serves none: " + + "start the emulator with --objectstore, and mount this region's door with " + + "--s3-doors " + Name + "/" + region}) + return 0, false + } + return p.importedObjectSize(ctx, w, region, bucket, key) +} + +// importedObjectSize resolves an object a CONTROL-PLANE import names, and +// answers its size. ok=false means the refusal has already been written. +// +// Shared by the instance and block imports on purpose. They answer differently +// when no byte store is armed, and that difference is deliberate — the block +// import has been served as an intention since #807, the instance one was +// refused outright — but what they do when there IS one must not be written +// twice. This repository has already paid for the same lifecycle written in +// two packs and fixed in one. +// +// TestASnapshotImportIsRefusedWithoutTheObject and +// TestABlockImportRefusesAnObjectTheDoorDoesNotHold fail without this. +func (p *Pack) importedObjectSize(ctx context.Context, w http.ResponseWriter, + region, bucket, key string) (uint64, bool) { + object, err := p.env.Objects.HeadObject(ctx, s3.StoreNamespace(Name, region)+"/"+bucket, key) + if err != nil { + writeInvalidArguments(w, ArgumentError{ArgumentName: "key", Reason: "constraint", + HelpMessage: "the " + Name + "/" + region + " door holds no object " + key + + " in bucket " + bucket + ", so there is nothing to import"}) + return 0, false + } + if object.Size < 0 { + return 0, true + } + return uint64(object.Size), true +} diff --git a/internal/providers/scaleway/snapshots_test.go b/internal/providers/scaleway/snapshots_test.go index bda97aec..a7412ab8 100644 --- a/internal/providers/scaleway/snapshots_test.go +++ b/internal/providers/scaleway/snapshots_test.go @@ -134,11 +134,18 @@ func TestASnapshotOfNothingIsRefused(t *testing.T) { } } -// Importing from Object Storage stays refused, and says which argument. +// Importing from Object Storage is refused when this RUN serves none. // -// docs/limits.md carries why Object Storage is not emulated. A snapshot -// restored from a bucket nobody serves would answer success about bytes that do -// not exist, which is worse than the refusal. +// The reason used to be that the emulator served no Object Storage at all, and +// that sentence stopped being true: see importedSnapshotSize and +// TestASnapshotImportTakesTheSizeOfTheObjectItNames, which import an object +// that is actually there. What survives is the narrower refusal this test +// holds — a snapshot restored from bytes nobody has would answer success about +// a disk that does not exist, which is worse than refusing. +// +// The message it must NOT go back to is the unconditional one: with a byte +// store armed, the old text told the client the emulator does not serve Object +// Storage while the object sat in the door, named by the refused request. func TestASnapshotImportedFromABucketIsRefused(t *testing.T) { ts := newTestServer(t) diff --git a/internal/s3/doors.go b/internal/s3/doors.go index 6588bc40..cac4f847 100644 --- a/internal/s3/doors.go +++ b/internal/s3/doors.go @@ -137,7 +137,7 @@ func mount(next http.Handler, store *objectstore.Store, profiles []Profile, maxO return nil, fmt.Errorf("s3: duplicate door %s", path) } server, err := New(store, Config{Region: p.Region, MaxObjectBytes: maxObjectBytes, - MissingBucket: p.MissingBucket, Namespace: p.Provider + "/" + p.Region, Log: log, + MissingBucket: p.MissingBucket, Namespace: StoreNamespace(p.Provider, p.Region), Log: log, BucketConfiguration: p.BucketConfiguration, AbsentConfigurations: p.AbsentConfigurations, ObjectDiscovery: p.ObjectDiscovery, ListObjectsV1: p.ListObjectsV1, OwnerID: p.OwnerID, PrivateACLWrite: p.PrivateACLWrite}) @@ -280,3 +280,22 @@ func (w *doorWriter) Write(p []byte) (int, error) { } func (w *doorWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +// StoreNamespace is the scope a door's buckets live under in the shared byte +// store, so that two doors can serve a bucket of the same name without seeing +// each other's. +// +// Exported because a pack sometimes has to find the object a CONTROL-PLANE +// call names: Scaleway's CreateSnapshot takes {bucket, key} and the real cloud +// reads that object server-side. A second copy of this composition in the pack +// is exactly how the two would drift, and the drift would be invisible — the +// pack would simply never find an object that is there. +// +// TestMountedDoorsPreserveRawKeysAndIsolateScopes fails when this stops +// agreeing with what a door writes. That test spells the scope out by hand +// instead of calling this function, deliberately: a witness that computed the +// expected name the same way as the subject would agree with any format, +// including a wrong one. +func StoreNamespace(provider, region string) string { + return provider + "/" + region +} diff --git a/tools/falsify/specs/a-snapshot-import-reads-the-object.json b/tools/falsify/specs/a-snapshot-import-reads-the-object.json new file mode 100644 index 00000000..569f0323 --- /dev/null +++ b/tools/falsify/specs/a-snapshot-import-reads-the-object.json @@ -0,0 +1,27 @@ +{ + "package": "./internal/providers/scaleway/", + "note": "The subject is instance/v1 CreateSnapshot with {bucket, key}: the import path that was refused unconditionally while the emulator served no bytes, and whose refusal outlived its reason. Found by replaying ioandev/scaleway-flatcar-k3s (examples/stacks/surveyed.md, #262), whose image pipeline uploads a qcow2 through scaleway_object and then imports it: the upload landed in the local door and the import answered 'importing a snapshot reads Object Storage, which this emulator does not serve', naming the very object it had just stored. Eight of that stack's resources sit behind this one call, every one of them a plain dependency of the snapshot.\n\nWhat these mutations guard is that the fix did not become a rubber stamp. Serving the import is easy; serving it only when the object is really there, with the size the object really has, is the part that keeps a green run here from being a wrong disk there. The absence refusal borrows this route's own invalid_arguments envelope rather than inventing an upstream dialect, and docs/limits.md says that the real API's shape for a missing object is unmeasured.", + "mutations": [ + { + "label": "the imported snapshot stops taking the object's size and falls back to the route's default, so a client that asked for a 65 536-byte image is told it has a 10 GB one. This is the half-success the project exists to avoid: green here, wrong disk there, and the size is the ONE fact about the imported object a client can check", + "file": "internal/providers/scaleway/snapshots.go", + "find": "\t\tsize = bytes", + "replace": "\t\tsize = bytes * 0", + "test": "TestASnapshotImportTakesTheSizeOfTheObjectItNames" + }, + { + "label": "an object that is not in the door is imported anyway, which is the unconditional refusal's defect reflected: it used to refuse everything on a false premise, and this refuses nothing on an unchecked one. The real API refuses an object that does not exist, and a snapshot of bytes nobody has is exactly what the old refusal was right to prevent", + "file": "internal/providers/scaleway/snapshots.go", + "find": "\tobject, err := p.env.Objects.HeadObject(ctx, s3.StoreNamespace(Name, region)+\"/\"+bucket, key)\n\tif err != nil {", + "replace": "\tobject, err := p.env.Objects.HeadObject(ctx, s3.StoreNamespace(Name, region)+\"/\"+bucket, key)\n\tif err != nil && false {", + "test": "TestASnapshotImportIsRefusedWithoutTheObject" + }, + { + "label": "the unarmed refusal stops naming the flags that arm a byte store and points at the limits page instead, which is the message this whole change exists to retire: it tells an operator that the emulator cannot do this, when in fact their own run was started without --objectstore and one flag would fix it. A refusal that cannot be acted on reads as a missing feature", + "file": "internal/providers/scaleway/snapshots.go", + "find": "\t\t\tHelpMessage: \"importing a snapshot reads Object Storage, and this run serves none: \" +\n\t\t\t\t\"start the emulator with --objectstore, and mount this region's door with \" +\n\t\t\t\t\"--s3-doors \" + Name + \"/\" + region})", + "replace": "\t\t\tHelpMessage: \"importing a snapshot reads Object Storage, which this emulator \" +\n\t\t\t\t\"does not serve; see docs/limits.md\" +\n\t\t\t\t\"\" + Name + \"\" + region})", + "test": "TestASnapshotImportIsRefusedWithoutTheObject" + } + ] +} From 4e2059510717299d86811df8af78ece82ccc561d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Wed, 11 Nov 2026 09:28:01 +0100 Subject: [PATCH 27/28] fix(ci): move the Go pin to 1.26.11, and say where the question is asked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GO-2026-6859: an HTTP/1 server crash on a MaxBytesReader header race in net/http, found in go1.26.10 and fixed in go1.26.11. Nothing in this repository is the cause and nothing here can fix it; the pin is what moves, and every file that names a patch moves together — TestTheGoWorkflowPinsTheVersionMiseDoes holds that, and passes. Seventeen pins in eleven files. The historical comments in go.yml are left alone: they record why the pin moved each time, and rewriting them would erase the only account of it. Verified rather than assumed: `govulncheck ./...` on go1.26.11 answers "No vulnerabilities found" locally. Before the bump it reported GO-2026-6859 through net/http in the standard library. ## Why this was found by CI and not before the push This is the fourth time: 1.26.6, 1.26.9, 1.26.10, now 1.26.11, each a red on pull requests that touched no Go file. govulncheck runs in go.yml and nowhere else — not in `mise run prepush`, not in .pre-commit-config.yaml. That absence is correct, and the reason is CLAUDE.md's own argument for keeping conformance out of the hook, in a sharper form. prepush is offline, deterministic and a few seconds long. govulncheck is none of the three: it queries vuln.go.dev, so it fails on a network it cannot reach, and its verdict changes without a line of this repository changing. A gate that red teaches `--no-verify`, which disarms every other hook at once. So the fix is not to move it into the hook. A scan whose verdict moves on its own belongs on a SCHEDULE — the shape drift.yml already has for the upstream surface, and night-report.yml for a scan that could not conclude. Until that exists, this red lands on whoever happens to push next, which is what it did all four times. That is written in go.yml beside the pin, and filed. What this commit adds meanwhile is `mise run vulncheck`, so the question can be ASKED before pushing by whoever wants to ask it, and so the answer to "why did CI see this and I did not" is one command rather than a workflow file. Not done here, and named rather than silently skipped: go1.27.4 is released, and this repository's own rule is to work on the latest major. Moving a minor is a decision with its own measurements (golangci-lint, the toolchain a contributor installs, every pinned job), not a line to change inside a security patch. mise run prepush green. actionlint clean. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/adoption-metrics.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/conformance.yml | 8 ++++---- .github/workflows/corpus-cloud.yml | 2 +- .github/workflows/drift.yml | 2 +- .github/workflows/falsify.yml | 2 +- .github/workflows/go.yml | 28 +++++++++++++++++++++++--- .github/workflows/release.yml | 2 +- .github/workflows/runtime-proof.yml | 4 ++-- .github/workflows/tap.yml | 2 +- mise.toml | 22 +++++++++++++++++++- 11 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/workflows/adoption-metrics.yml b/.github/workflows/adoption-metrics.yml index 2eb9d339..f0678f1d 100644 --- a/.github/workflows/adoption-metrics.yml +++ b/.github/workflows/adoption-metrics.yml @@ -53,7 +53,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # The data branch, in a directory of its own. It carries no code, only diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a2e9f79d..6f50155f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this module # dependency-free, and a pre-commit hook fails if the file ever # appears. Pointing the cache at it warned on every run and never diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index af2cd2f8..d55c1b3a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -54,7 +54,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # No browser is installed here: the runner image ships Google Chrome, and @@ -93,7 +93,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # The same build the release performs (static, trimmed), because the @@ -208,7 +208,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this module # dependency-free, and a pre-commit hook fails if the file ever # appears. Pointing the cache at it warned on every run and never @@ -610,7 +610,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # The workflow-level pin, same checksums as the other two jobs that diff --git a/.github/workflows/corpus-cloud.yml b/.github/workflows/corpus-cloud.yml index 818ca1d6..f552c7c0 100644 --- a/.github/workflows/corpus-cloud.yml +++ b/.github/workflows/corpus-cloud.yml @@ -53,7 +53,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # Asserted, never skipped. A job that quietly did nothing when the credential was absent diff --git a/.github/workflows/drift.yml b/.github/workflows/drift.yml index bc502f8b..01a24737 100644 --- a/.github/workflows/drift.yml +++ b/.github/workflows/drift.yml @@ -61,7 +61,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this module # dependency-free, and a pre-commit hook fails if the file ever # appears. Pointing the cache at it warned on every run and never diff --git a/.github/workflows/falsify.yml b/.github/workflows/falsify.yml index 70fee368..953bdedd 100644 --- a/.github/workflows/falsify.yml +++ b/.github/workflows/falsify.yml @@ -74,7 +74,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this # module dependency-free. cache: false diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7ab075b6..e410d207 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -62,7 +62,29 @@ jobs: # the pin to a minor — that was tried and is recorded above as the # thing that made CI resolve a version nobody named. It is a reason to # expect this line to move often, and to keep moving it by hand. - go-version: '1.26.10' + # + # AND A FOURTH TIME, to 1.26.11 on 2026-11-11: GO-2026-6859, an + # HTTP/1 server crash on a MaxBytesReader header race in net/http, + # found in go1.26.10 and fixed in go1.26.11. Same shape as the three + # above — nothing in this repository was the cause, nothing here + # could fix it, the pin is what moves. + # + # FOUR TIMES IS A PATTERN, AND THE PATTERN IS WHERE THE QUESTION IS + # ASKED. govulncheck runs in this workflow and nowhere else: not in + # `mise run prepush`, not in .pre-commit-config.yaml. That absence is + # deliberate and CLAUDE.md states the reason — prepush is offline, + # deterministic and a few seconds long, while govulncheck queries + # vuln.go.dev and its verdict changes without a single line of this + # repository changing. A gate like that in a hook fails on the + # network rather than on the code, and teaches --no-verify. + # + # So the answer is not to move it into the hook: it is that a scan + # whose verdict moves on its own belongs on a SCHEDULE, the way + # drift.yml already scans the upstream surface every night and opens + # a pull request. Until that exists, this red lands on whoever + # happens to push next, which is what it did all four times. + # Tracked in the issue this comment's commit names. + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this module # dependency-free, and a pre-commit hook fails if the file ever # appears. Pointing the cache at it warned on every run and never @@ -200,7 +222,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false - name: Every operation that changed hands is named, or signed @@ -256,7 +278,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false - name: go build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb1499a0..e44f5112 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false # a release build must not consume a cache a PR could poison - name: Build diff --git a/.github/workflows/runtime-proof.yml b/.github/workflows/runtime-proof.yml index 47d8b39c..2ecec0a3 100644 --- a/.github/workflows/runtime-proof.yml +++ b/.github/workflows/runtime-proof.yml @@ -85,7 +85,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this # module dependency-free. cache: false @@ -620,7 +620,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' cache: false - name: Install Incus (Zabbly stable, key fingerprint pinned) diff --git a/.github/workflows/tap.yml b/.github/workflows/tap.yml index 8201d8c1..a88b46bd 100644 --- a/.github/workflows/tap.yml +++ b/.github/workflows/tap.yml @@ -61,7 +61,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.26.10' + go-version: '1.26.11' # No go.sum to key a cache on: the zero-dependency rule keeps this # module dependency-free. cache: false diff --git a/mise.toml b/mise.toml index 4c10514d..e16014fe 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,7 @@ # mise — unified toolchain + task runner for feint (versions pinned, reproducible across machines # and CI). Activate: `mise install` then `mise trust`. Run a task: `mise run `. List: `mise tasks`. [tools] -go = "1.26.10" # the feint binary +go = "1.26.11" # the feint binary # Repeated in .github/workflows/go.yml, which installs it by version; the major # is repeated again as .golangci.yml's schema version. Both are held by # TestTheWorkflowsPinTheSameToolsAsMise and @@ -467,6 +467,26 @@ run = "tools/conformance/outscale/octl.sh http://$FEINT_ADDR" description = "shellcheck every script in the repository" run = "git ls-files -z '*.sh' | xargs -0 shellcheck --severity=style" +# Deliberately NOT in prepush, and the reason is the one CLAUDE.md gives for +# keeping conformance out of the hook, in a sharper form: prepush is offline, +# deterministic and a few seconds long, and govulncheck is none of the three. +# It queries vuln.go.dev, so it fails on a network it cannot reach, and its +# verdict changes without a single line here changing — which is exactly what +# happened on 1.26.6, 1.26.9, 1.26.10 and 1.26.11, four reds on pull requests +# that touched no Go file. A hook that red means --no-verify, which disarms +# every other hook at once. +# +# It is here so the question can be ASKED before pushing, by whoever wants to +# ask it, and so that the answer to "why did CI see this and I did not" is one +# command rather than a workflow file. The scan that should catch it before +# anybody pushes belongs on a schedule, the way drift.yml works. +[tasks.vulncheck] +description = "govulncheck against the pinned toolchain (network, non-deterministic verdict)" +run = """ +go install golang.org/x/vuln/cmd/govulncheck@v1.3.0 +"$(go env GOPATH)/bin/govulncheck" ./... +""" + [tasks."compat:check"] description = "Run an older release's consumer expressions against this one, and refuse a silently-wrong verdict" # The consumer half of the versioning policy (#170). #132 froze the shapes and From 386a15869265c9d2b0121d1b48663bc1fc56aec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Wed, 11 Nov 2026 09:39:40 +0100 Subject: [PATCH 28/28] chore(release): 0.15.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Unreleased section becomes `## [0.15.0] - 2026-11-11`, which is step 1 of RELEASING.md and the thing every generated install block reads for the version it tells a reader to download. commitizen's rule gives the number: 19 `feat` and 9 `fix` since v0.14.0, no `!` and no BREAKING CHANGE anywhere, so the minor moves and 0.x holds. Two entries added for work that landed after the section was first written, and both are the kinds of change this file's own header says deserve a line whatever their size — a response shape a client can observe, and a limit that moved: - the instance snapshot import from Object Storage, with what it checks and what stays unmeasured; - the two defects a surveyed third-party stack found: `feint run` refusing the fakes `feint env` publishes, and an imported snapshot answering an empty base volume that made the Scaleway provider replace it on every plan. `mise run docs:coverage` regenerated README.md, README.fr.md and docs/install.md in the same commit, as that step requires: the heading is where their pinned download URLs get their version, and leaving them behind makes `feint docs --check` exit 2 at the hook, on the pull request, in the preflight and inside the release workflow. Six hand-written files follow the same number, which nothing generates and TestEveryExampleNamesTheReleasedVersion holds: the three examples, the compose file, the breaking-configuration issue template and the demo tape. They were once wrong in both directions at once, telling a reader to install a release that did not exist. `mise run release:surface` is green: one change this release must carry, named — the withdrawn refusal on `exoscale/exoscale/v2.list-sos-buckets-usage` — zero excused, zero neither. `tools/release/unnamed.json` is empty, so step 5 is a no-op. mise run prepush green. Assisted-by: Claude Code (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ISSUE_TEMPLATE/breaking_configuration.yml | 2 +- CHANGELOG.fr.md | 30 ++++++++++++++++++- CHANGELOG.md | 29 +++++++++++++++++- README.fr.md | 6 ++-- README.md | 12 ++++---- docs/install.md | 16 +++++----- examples/README.md | 2 +- examples/compose/compose.yaml | 2 +- examples/github-actions/terraform.yml | 4 +-- examples/gitlab-ci/.gitlab-ci.yml | 2 +- tools/demo/ci.tape | 2 +- 11 files changed, 81 insertions(+), 26 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/breaking_configuration.yml b/.github/ISSUE_TEMPLATE/breaking_configuration.yml index 5c2ebaea..c37f05f6 100644 --- a/.github/ISSUE_TEMPLATE/breaking_configuration.yml +++ b/.github/ISSUE_TEMPLATE/breaking_configuration.yml @@ -59,7 +59,7 @@ body: attributes: label: Versions description: Feint, the client, and the provider — `feint version`, `terraform version`. - placeholder: "feint 0.14.0, terraform 1.13.3, scaleway/scaleway 2.81.0" + placeholder: "feint 0.15.0, terraform 1.13.3, scaleway/scaleway 2.81.0" validations: required: true diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index de8d01b0..9559f72a 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -15,7 +15,7 @@ parce que c'est là-dessus que ce projet est jugé : **une forme de réponse qu' client peut observer**, et **une limite qui a bougé**. Une refactorisation qui ne change ni l'un ni l'autre a sa place dans `git log`. -## [Unreleased] +## [0.15.0] - 2026-11-11 ### Ajouté @@ -147,6 +147,34 @@ change ni l'un ni l'autre a sa place dans `git log`. neutre version 2 lit la version 1 et la migre à la prochaine mutation ; les anciens binaires ne peuvent pas relire cette version 2. +- **Un snapshot d'instance s'importe depuis le stockage objet** (#850, #262) : + `instance/v1/API.CreateSnapshot` accepte `{bucket, key}` et lit l'objet par la + porte de sa propre région, là où il était refusé sans condition. Le snapshot + prend la taille de l'objet et ne porte aucun volume parent ; un objet absent + est refusé, comme l'API réelle le refuse ; et sans magasin d'octets armé, le + refus nomme `--objectstore` et `--s3-doors` plutôt que la page des limites. + L'import de block/v1 passe par le même code, ce qui referme la divergence que + #807 avait écrite lui-même. **Une limite a bougé** : `docs/limits.md` dit ce + qui est servi et que le dialecte réel du refus d'un objet absent reste non + mesuré. +- **`mise run vulncheck`** : govulncheck à la demande contre la chaîne d'outils + épinglée. Pas dans `prepush`, et #865 dit pourquoi : son verdict change sans + qu'une ligne du dépôt change. + +### Corrigé + +- **`feint run` refusait les faux identifiants publiés par `feint env`** + (#849) : la séquence que les deux commandes documentent, `eval` de + `feint env` puis `feint run`, était refusée par son propre binaire, parce que + la liste acceptée était une seconde copie écrite à la main de ce que les packs + publient et qu'elle en avait dérivé de deux caractères sur une valeur, et + d'une valeur entière sur l'autre. La liste se dérive des packs désormais. +- **Un snapshot importé rendait un volume parent vide**, que le provider + Scaleway relit en `volume_id "fr-par-1/"` et traite comme + `forces replacement`, ce qui faisait churner le snapshot et l'image qui en + est tirée à chaque plan. Il rend null, et une stack tierce du registre + s'applique maintenant en entier avec un second plan vide. + ## [0.14.0] - 2026-10-26 ### Ajouté diff --git a/CHANGELOG.md b/CHANGELOG.md index e23a456c..684e6798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Two kinds of change deserve their own line whatever their size, because they are what this project is judged on: **a response shape a client can observe**, and **a limit that moved**. A refactor that changes neither belongs in `git log`. -## [Unreleased] +## [0.15.0] - 2026-11-11 ### Added @@ -142,6 +142,33 @@ what this project is judged on: **a response shape a client can observe**, and roots are dispatched without redirects. Neutral private metadata version 2 reads version 1 and upgrades on mutation; older binaries cannot read it. +- **An instance snapshot imports from Object Storage** (#850, #262): + `instance/v1/API.CreateSnapshot` takes `{bucket, key}` and reads the object + through the door of its own region, where it was refused outright before. The + snapshot takes the object's size and carries no base volume; an absent object + is refused, as upstream refuses it; and with no byte store armed the refusal + names `--objectstore` and `--s3-doors` rather than the limits page. block/v1's + import reads through the same code, which closes the divergence #807 recorded + in its own words. **A limit moved**: `docs/limits.md` says what is now served + and that the real API's dialect for a missing object is still unmeasured. +- **`mise run vulncheck`**: govulncheck on demand against the pinned toolchain. + Not in `prepush`, and #865 says why: its verdict changes without a line of + this repository changing. + +### Fixed + +- **`feint run` accepted the fakes `feint env` publishes** (#849): the sequence + both commands document — eval `feint env`, then `feint run` — was refused by + its own binary, because the accepted list was a second hand-written copy of + the credentials the packs publish and had drifted from them by two characters + on one value and by a whole missing value on another. The list is derived from + the packs now, so the two cannot disagree again. +- **An imported snapshot answered an empty base volume**, which the Scaleway + provider reads back as `volume_id "fr-par-1/"` and treats as + `forces replacement`, churning the snapshot and the image cut from it on + every plan. It answers null, and a surveyed third-party stack now applies + whole with an empty second plan. + ## [0.14.0] - 2026-10-26 ### Added diff --git a/README.fr.md b/README.fr.md index 7738e695..7c855433 100644 --- a/README.fr.md +++ b/README.fr.md @@ -104,18 +104,18 @@ terraform apply # le vrai provider Scaleway ```yaml services: feint: - image: ghcr.io/stephrobert/feint:v0.14.0 + image: ghcr.io/stephrobert/feint:v0.15.0 ports: ["4599:4599"] ``` -L'image ne sert que le plan de contrôle et ne porte aucun tag `latest` ; [docs/install.md](docs/install.md) donne la forme GitLab, la forme compose et la vérification de signature. Pour la tirer directement : `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.14.0`. +L'image ne sert que le plan de contrôle et ne porte aucun tag `latest` ; [docs/install.md](docs/install.md) donne la forme GitLab, la forme compose et la vérification de signature. Pour la tirer directement : `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.15.0`. **Dans GitHub Actions, sans conteneur**, l'action du Marketplace : ```yaml - uses: stephrobert/setup-feint@v1 with: - version: 0.14.0 + version: 0.15.0 provider: scaleway # exporte ce dont le client officiel a besoin ``` diff --git a/README.md b/README.md index 650345bb..1d4b0d7e 100644 --- a/README.md +++ b/README.md @@ -93,18 +93,18 @@ terraform apply # the real Scaleway provider ```yaml services: feint: - image: ghcr.io/stephrobert/feint:v0.14.0 + image: ghcr.io/stephrobert/feint:v0.15.0 ports: ["4599:4599"] ``` -The image is control-plane only and carries no `latest` tag; [docs/install.md](docs/install.md) has the GitLab form, the compose form and the signature verification. Pull it directly with `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.14.0`. +The image is control-plane only and carries no `latest` tag; [docs/install.md](docs/install.md) has the GitLab form, the compose form and the signature verification. Pull it directly with `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.15.0`. **In GitHub Actions without a container** — the action from the Marketplace: ```yaml - uses: stephrobert/setup-feint@v1 with: - version: 0.14.0 + version: 0.15.0 provider: scaleway # exports what the official client needs ``` @@ -233,7 +233,7 @@ the same 6.0.4, and says what to install, which is the point of having it. -Version **0.14.0**, named rather than fetched as `latest`: a mutable reference +Version **0.15.0**, named rather than fetched as `latest`: a mutable reference downloads whatever is newest, which is a binary neither of us can name afterwards and a release adopted the day it is published. This line moves when the CHANGELOG does, and `feint docs --check` fails until it has. @@ -242,7 +242,7 @@ Needs `cosign` (or `gh`, further down) and nothing else. Every file is fetched to disk and verified before anything runs it. ```bash -base=https://github.com/stephrobert/feint/releases/download/v0.14.0 +base=https://github.com/stephrobert/feint/releases/download/v0.15.0 # The published binary for this machine, or a refusal: no default, because # the wrong architecture fails at exec with a message naming neither. @@ -285,7 +285,7 @@ difference between an identity and a provenance: without it the check accepts anything this repository attested, whichever workflow did it. ```bash -gh release download v0.14.0 --repo stephrobert/feint --pattern 'feint-linux-amd64' +gh release download v0.15.0 --repo stephrobert/feint --pattern 'feint-linux-amd64' gh attestation verify feint-linux-amd64 --repo stephrobert/feint \ --signer-workflow stephrobert/feint/.github/workflows/release.yml ``` diff --git a/docs/install.md b/docs/install.md index 1e87f842..fdf8cbdd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,7 +5,7 @@ Feint itself needs nothing: one Go binary, no dependency, no daemon. -Version **0.14.0**, named rather than fetched as `latest`: a mutable reference +Version **0.15.0**, named rather than fetched as `latest`: a mutable reference downloads whatever is newest, which is a binary neither of us can name afterwards and a release adopted the day it is published. This line moves when the CHANGELOG does, and `feint docs --check` fails until it has. @@ -14,7 +14,7 @@ Needs `cosign` (or `gh`, further down) and nothing else. Every file is fetched to disk and verified before anything runs it. ```bash -base=https://github.com/stephrobert/feint/releases/download/v0.14.0 +base=https://github.com/stephrobert/feint/releases/download/v0.15.0 # The published binary for this machine, or a refusal: no default, because # the wrong architecture fails at exec with a message naming neither. @@ -57,7 +57,7 @@ difference between an identity and a provenance: without it the check accepts anything this repository attested, whichever workflow did it. ```bash -gh release download v0.14.0 --repo stephrobert/feint --pattern 'feint-linux-amd64' +gh release download v0.15.0 --repo stephrobert/feint --pattern 'feint-linux-amd64' gh attestation verify feint-linux-amd64 --repo stephrobert/feint \ --signer-workflow stephrobert/feint/.github/workflows/release.yml ``` @@ -176,7 +176,7 @@ One tag per release, the release's own, and nothing mutable: no `latest`, for the same reason the commands above name a version. ```bash -docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.14.0 +docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.15.0 curl http://127.0.0.1:4599/_feint/health ``` @@ -186,7 +186,7 @@ healthcheck answers, so the first step can talk to it immediately: ```yaml services: feint: - image: ghcr.io/stephrobert/feint:v0.14.0 + image: ghcr.io/stephrobert/feint:v0.15.0 ports: - 4599:4599 ``` @@ -195,7 +195,7 @@ In GitLab CI: ```yaml services: - - name: ghcr.io/stephrobert/feint:v0.14.0 + - name: ghcr.io/stephrobert/feint:v0.15.0 alias: feint ``` @@ -210,11 +210,11 @@ just pushed, and `tools/release/preflight.sh` runs it against the previous release before a tag exists. ```bash -cosign verify ghcr.io/stephrobert/feint:v0.14.0 \ +cosign verify ghcr.io/stephrobert/feint:v0.15.0 \ --certificate-identity-regexp '^https://github\.com/stephrobert/feint/\.github/workflows/release\.yml@refs/tags/v' \ --certificate-oidc-issuer https://token.actions.githubusercontent.com -gh attestation verify oci://ghcr.io/stephrobert/feint:v0.14.0 \ +gh attestation verify oci://ghcr.io/stephrobert/feint:v0.15.0 \ --repo stephrobert/feint \ --signer-workflow stephrobert/feint/.github/workflows/release.yml ``` diff --git a/examples/README.md b/examples/README.md index 2aba16ea..a972b4c5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,7 +47,7 @@ that wants real machines behind `--vm` on a host with Incus: ```yaml - uses: stephrobert/setup-feint@v1 with: - version: 0.14.0 + version: 0.15.0 provider: scaleway # exports what the official client needs - run: terraform apply -auto-approve ``` diff --git a/examples/compose/compose.yaml b/examples/compose/compose.yaml index d5dcf6ef..01ad4a75 100644 --- a/examples/compose/compose.yaml +++ b/examples/compose/compose.yaml @@ -14,7 +14,7 @@ services: feint: - image: ghcr.io/stephrobert/feint:v0.14.0 + image: ghcr.io/stephrobert/feint:v0.15.0 # Published on the loopback rather than on every interface. The emulator # accepts any credential and holds its state in memory — SECURITY.md is # explicit that it must not be exposed to a network you do not control. diff --git a/examples/github-actions/terraform.yml b/examples/github-actions/terraform.yml index c931e8ac..991bc48a 100644 --- a/examples/github-actions/terraform.yml +++ b/examples/github-actions/terraform.yml @@ -11,7 +11,7 @@ # # - uses: stephrobert/setup-feint@v1 # with: -# version: 0.14.0 +# version: 0.15.0 # provider: scaleway # # which installs the verified binary and exports the client environment. Use the @@ -36,7 +36,7 @@ jobs: feint: # One tag per release, and no `latest`: a mutable reference pulls # whatever is newest, which is an image nobody can name afterwards. - image: ghcr.io/stephrobert/feint:v0.14.0 + image: ghcr.io/stephrobert/feint:v0.15.0 ports: - 4599:4599 diff --git a/examples/gitlab-ci/.gitlab-ci.yml b/examples/gitlab-ci/.gitlab-ci.yml index 7a0567d9..f88a5248 100644 --- a/examples/gitlab-ci/.gitlab-ci.yml +++ b/examples/gitlab-ci/.gitlab-ci.yml @@ -15,7 +15,7 @@ # fact. Change it when you upgrade, on purpose. variables: - FEINT_IMAGE: ghcr.io/stephrobert/feint:v0.14.0 + FEINT_IMAGE: ghcr.io/stephrobert/feint:v0.15.0 # The service is reachable by its alias inside the job's network, and the # emulator's own port is the same one it uses everywhere else. FEINT_ENDPOINT: http://feint:4599 diff --git a/tools/demo/ci.tape b/tools/demo/ci.tape index f3331535..1b6125dc 100644 --- a/tools/demo/ci.tape +++ b/tools/demo/ci.tape @@ -70,7 +70,7 @@ Sleep 1200ms # worked, and no viewer could copy the line. A hard-coded version in a recording # is the kind of figure this repository refuses, so tools/demo/ci.sh fails when # this line and the CHANGELOG disagree. -Type "docker run -d --rm --name feint -p 4599:4599 ghcr.io/stephrobert/feint:v0.14.0" +Type "docker run -d --rm --name feint -p 4599:4599 ghcr.io/stephrobert/feint:v0.15.0" Enter Sleep 4s