test(firestore): enforce Node parity for Pipelines and overhaul the E2E suite

.github/workflows/e2e_pipeline.yml

@@ -97,21 +97,21 @@ jobs:
           # Same Enterprise-edition database FlutterFire's pipeline suite
           # targets; not a secret, it is hardcoded in FlutterFire's own tests.
           FIRESTORE_PIPELINE_E2E_DATABASE_ID: firestore-pipeline-test
-        # --concurrency=1: every test seeds and deletes its own documents in a
-        # shared live database, matching how scripts/firestore-coverage.sh runs
-        # the other prod-tagged suites.
+        # --concurrency=1: every file under test/e2e/pipeline/ seeds and
+        # deletes its own documents in a shared live database, matching how
+        # scripts/firestore-coverage.sh runs the other prod-tagged suites.
         #
         # Prerequisite: the target database needs the `pipeline_e2e_books`
         # vector index described in test/e2e/README.md, otherwise the
-        # findNearest test fails. It is created once, out of band.
+        # findNearest tests fail. It is created once, out of band.
         run: |
           set -o pipefail
           if [ -z "${FIRESTORE_PIPELINE_E2E_PROJECT_ID:-}" ]; then
             echo "::error::PIPELINE_E2E_PROJECT_ID is empty — the secret is unset or unavailable. Failing early instead of reporting a green run for tests that never executed."
             exit 1
           fi
           log="$RUNNER_TEMP/pipeline-e2e.log"
-          dart test -P prod test/e2e/pipeline_e2e_test.dart \
+          dart test -P prod test/e2e/pipeline/ \
             --concurrency=1 --reporter expanded 2>&1 | tee "$log"
           # `dart test` exits 0 when every test is skipped, which is exactly
           # what a missing project or database ID produces. Treat that as a

.github/workflows/pipeline_api_snapshot.yml

@@ -0,0 +1,55 @@
+name: Pipeline API snapshot
+
+# Checks that test/fixtures/node_pipeline_api.json, which
+# test/pipeline_api_parity_test.dart compares the Dart Pipelines API against,
+# is what tool/pipeline_api_parity extracts from the pinned Node SDK. The Dart
+# test itself runs with the other google_cloud_firestore tests in build.yml.
+#
+# A separate workflow so that it only runs for google_cloud_firestore changes:
+# build.yml cannot filter paths per job.
+on:
+  pull_request:
+    paths:
+      - "packages/google_cloud_firestore/**"
+      - ".github/workflows/pipeline_api_snapshot.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/google_cloud_firestore/**"
+      - ".github/workflows/pipeline_api_snapshot.yml"
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  snapshot:
+    name: Node Pipeline API snapshot is up to date
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    defaults:
+      run:
+        working-directory: packages/google_cloud_firestore/tool/pipeline_api_parity
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          node-version: 24
+
+      - run: npm ci --ignore-scripts --no-audit --no-fund
+
+      - run: npm run extract
+
+      - name: Fail if the snapshot is stale
+        run: |
+          if ! git diff --exit-code -- ../../test/fixtures/node_pipeline_api.json; then
+            echo "::error::test/fixtures/node_pipeline_api.json is stale. Regenerate it with 'npm ci && npm run extract' in packages/google_cloud_firestore/tool/pipeline_api_parity."
+            exit 1
+          fi

.github/workflows/pipeline_golden.yml

@@ -0,0 +1,62 @@
+name: Pipeline Golden Fixtures
+
+on:
+  pull_request:
+    # The fixtures only depend on google_cloud_firestore's generator and the
+    # Node SDK it pins, so there is nothing to check for other changes.
+    paths:
+      - "packages/google_cloud_firestore/**"
+      - ".github/workflows/pipeline_golden.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/google_cloud_firestore/**"
+      - ".github/workflows/pipeline_golden.yml"
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  pipeline-golden:
+    name: Pipeline golden fixtures are up to date
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    defaults:
+      run:
+        working-directory: packages/google_cloud_firestore/tool/pipeline_golden
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          # The pinned @google-cloud/firestore requires Node.js 22 or later.
+          node-version: "24"
+
+      - name: Install the pinned Node SDK
+        run: npm ci --no-audit --no-fund
+
+      - name: Regenerate the fixtures
+        run: npm run generate
+
+      # test/pipeline_golden_test.dart compares against the committed fixtures
+      # in the regular `test` job of build.yml; this only proves they are what
+      # the pinned Node SDK produces for tool/pipeline_golden/cases.js.
+      - name: Fail if the committed fixtures are stale
+        working-directory: .
+        run: |
+          fixtures=packages/google_cloud_firestore/test/fixtures/pipeline_golden
+          if [ -n "$(git status --porcelain -- "$fixtures")" ]; then
+            git status --short -- "$fixtures"
+            git --no-pager diff --stat -- "$fixtures"
+            git --no-pager diff -- "$fixtures" | head -n 200
+            echo "::error::The Pipeline golden fixtures are stale. Run 'npm ci && npm run generate' in packages/google_cloud_firestore/tool/pipeline_golden and commit the result."
+            exit 1
+          fi
+          git diff --exit-code -- "$fixtures"

CONTRIBUTING.md

@@ -137,6 +137,23 @@ No setup needed:
 dart test
 ```
 
+#### `google_cloud_firestore` Pipeline golden fixtures
+
+`test/pipeline_golden_test.dart` is a unit test that compares every Pipeline
+request against the request the Node.js SDK sends for the same Pipeline. The
+expected requests in `test/fixtures/pipeline_golden/` are generated by the
+pinned Node SDK; regenerate them (Node.js 22+) whenever you add a case or bump
+that SDK:
+
+```bash
+# From packages/google_cloud_firestore/tool/pipeline_golden
+npm ci && npm run generate
+```
+
+See
+[`tool/pipeline_golden/README.md`](packages/google_cloud_firestore/tool/pipeline_golden/README.md)
+for how capture works and how to record intended differences.
+
 #### `firebase_admin_sdk` emulator tests
 
 Requires the [Firebase CLI](https://firebase.google.com/docs/cli) and Java 21+.
@@ -307,13 +324,22 @@ in that project:
 
 | Job | Trigger | What it does |
 |-----|---------|--------------|
-| `pipeline-e2e` | PRs (non-fork) touching `packages/google_cloud_firestore/**`, pushes to `main`, schedule & manual | Runs `test/e2e/pipeline_e2e_test.dart` against live Firestore |
+| `pipeline-e2e` | PRs (non-fork) touching `packages/google_cloud_firestore/**`, pushes to `main`, schedule & manual | Runs `test/e2e/pipeline/` against live Firestore |
 
 It is a separate workflow so the live-quota cost is only paid for changes that
 can affect it. See
 [`packages/google_cloud_firestore/test/e2e/README.md`](packages/google_cloud_firestore/test/e2e/README.md)
 for the required secrets and the one-time vector index setup.
 
+**pipeline_golden.yml** keeps the Node-generated Pipeline fixtures honest:
+
+| Job | Trigger | What it does |
+|-----|---------|--------------|
+| `pipeline-golden` | PRs touching `packages/google_cloud_firestore/**`, pushes to `main` & manual | Regenerates `test/fixtures/pipeline_golden/` with the pinned Node SDK and fails if the committed fixtures differ |
+
+The comparison itself, `test/pipeline_golden_test.dart`, runs with the other
+unit tests in `build.yml`.
+
 Tests run against both `stable` and `beta` Dart SDK channels. Coverage is reported as a PR comment and uploaded to Codecov. The minimum threshold is **40%**.
 
 ## License

packages/google_cloud_firestore/CHANGELOG.md

@@ -6,16 +6,37 @@
 - **Breaking:** `PipelineFunctions.join` and `PipelineFunctions.split` now require their delimiter, matching `PipelineExpression` and the Node Admin SDK. A one-argument call was always rejected by the backend with `INVALID_ARGUMENT`.
 - **Breaking:** `PipelineFunctions.mapRemove` and `PipelineExpression.mapRemove` take a single key (a `String` or an expression) and send `map_remove(map, key)`, like the Node Admin SDK, instead of an `Iterable` of keys. Chain `.mapRemove('a').mapRemove('b')` to remove several keys.
 - **Breaking:** `PipelineResult.get` now resolves dot-separated paths such as `'metadata.lang'` as nested fields, like `DocumentSnapshot.get`, and also accepts a `FieldPath`. A top-level field whose name contains a dot, which `get('a.b')` used to return, is now read with `get(FieldPath(['a.b']))`. A string that is empty, starts or ends with `.`, or contains `..` or any of `*~/[]` now throws an `ArgumentError` instead of returning `null`.
+- **Breaking:** `PipelineFunctions.arrayTransform` no longer takes a trailing index alias. Use the new `PipelineFunctions.arrayTransformWithIndex(array, elementAlias, indexAlias, transform)`, which matches `PipelineExpression.arrayTransformWithIndex` and the Node Admin SDK and sends the same `array_transform` function.
 - Fixed `PipelineFunctions.arrayMaximum`, `arrayMaximumN`, `arrayMinimum`, `arrayMinimumN` and `arraySum` sending function names (`array_maximum`, ...) that the backend rejects. They now send `maximum`, `maximum_n`, `minimum`, `minimum_n` and `sum`, like their `PipelineExpression` counterparts.
 - Fixed `Pipeline.addFields` sending one `alias` function per field instead of a single map keyed by alias, which the backend rejected.
 - Fixed `PipelineSource.collectionGroup()`, and `createFrom()` on a collection group query, omitting the leading root ancestor argument of the `collection_group` stage.
 - Fixed `PipelineValueType.double` sending `'double'` instead of `'float64'`, which made `isType` fail. Added the `int32`, `decimal128`, `maxKey`, `minKey`, `objectId` and `regex` value types.
 - Fixed `cosineDistance`, `dotProduct`, `euclideanDistance` and `Pipeline.findNearest` sending a plain list of numbers as an array instead of a vector.
 - Fixed Pipeline functions such as `equalAny`, `notEqualAny`, `arrayContainsAll`, `arrayContainsAny` and `mapMerge` failing when a `List` or `Map` argument holds expressions. Collection arguments are now sent as `array(...)` / `map(...)` functions, like the Node Admin SDK, and the static and fluent forms encode identically.
+- Fixed the top-level `ascending()` and `descending()` sorting by a string constant when given a field name. A `String` now names a field, like everywhere else in the Pipeline API and in the Node Admin SDK.
 - `Pipeline.select`, `addFields`, `aggregate` and `distinct` now throw an `ArgumentError` on a duplicate field name or alias instead of silently keeping the last one, like the Node Admin SDK.
 - `PipelineExpression.substring` and `substringLiteral` now take `(position, [length])` like `PipelineFunctions.substring`: the second argument is a length, not an end index, and can be omitted.
 - `PipelineExpression.round` now accepts the optional `decimalPlaces` argument.
 - Exported the top-level `variable()` Pipeline helper.
+- Every Pipeline function now has a single implementation shared by all of its forms. The fluent `PipelineExpression` method, the top-level helper (`equal`, `and`, `not`, ...) and the `Expression` alias forward to the `PipelineFunctions` function (and `field`, `constant` and `variable` to `Expression`), so every form sends the same request.
+- `PipelineFunctions.arraySlice` now takes an optional `length`, like `PipelineExpression.arraySlice` and the Node Admin SDK.
+- `PipelineFunctions.raw` now accepts `options`, like `Expression.raw`.
+- `PipelineExpression.arrayContainsAll` and `arrayContainsAny` now also accept an array expression, like their `PipelineFunctions` forms and the Node Admin SDK.
+- Fixed Pipeline field names that are not simple identifiers being sent unquoted. They are now backtick-quoted like the Node Admin SDK, so `field('first-name')` sends `` `first-name` ``. `field()` and `Expression.field()` also accept a `FieldPath`. `PipelineField.path` returns the quoted path, and `field('')` and `field('a..b')` now throw.
+- Fixed `Pipeline.select`, `distinct` and the `groups` of `aggregate` keying a `String` field name that is not an identifier, such as `'last name'`, by the raw string, which the backend rejects as an invalid property path. It is now keyed by its quoted path, `` `last name` ``, like a `PipelineField`, so `select(['x-y', field('x-y')])` now throws for a duplicate key. An alias is still keyed as written, and must be a valid field path.
+- Pipeline stages and functions now accept a `FieldPath` wherever they accept a `String` field name, as `field()` does: the target of every `PipelineFunctions` function, `ascending()` and `descending()`, the entries of `select`, `distinct`, `aggregate` groups and `removeFields`, `unnest`, `replaceWith`, and `findNearest`'s `vectorField` and `distanceResultField`. A `FieldPath` there used to be sent as a constant, which failed to encode.
+- Fixed field paths containing a backtick being escaped as a lone backslash, which named a different field, in queries, field masks and Pipelines.
+- Fixed `Pipeline.createFrom()` and `DocumentSnapshot` query cursors leaving `!=` and `not-in` fields out of the implicit ordering, and ordering inequality fields by their quoted names instead of segment by segment, unlike the backend and the Node Admin SDK. A snapshot cursor now needs a value for every inequality field.
+- Fixed `Pipeline.createFrom()` of a query with a cursor on a descending ordering keeping the wrong side of the bound.
+- Fixed `Pipeline.createFrom()` of a `VectorQuery` sending its `distanceThreshold` as an undocumented `find_nearest` option. It is now applied as a filter on the distance.
+- Fixed `indexMode` being sent as an `index_mode` option the backend rejects; it is deprecated. `Pipeline.execute()` and `Transaction.executePipeline()` now ignore `indexMode`, and `PipelineIndexMode` is deprecated too: its only value, `recommended`, is already the backend's default.
+- Fixed `Pipeline.findNearest(distanceThreshold:)` sending a `distance_threshold` option, which the backend rejects. It now adds a `where` stage after `find_nearest` that keeps the results within the threshold, as `Pipeline.createFrom()` does for a `VectorQuery`.
+- Fixed Pipeline results failing to decode when a reference that names no document, such as the database root returned by `parent()`, is nested in a map or array. It now decodes to its resource name at every depth, as it already did for top-level fields.
+- Pipeline raw options now match the Node Admin SDK. Every stage and source takes an optional `rawOptions`. Keys in `rawOptions` and `rawStage(options:)` are dot-separated paths merged into the typed options, and keys with an empty segment throw an `ArgumentError`. `rawStage` sends collections nested in a `Map` argument as `map(...)` / `array(...)` functions.
+- `PipelineFunctions.add` and `multiply` take an optional trailing list of further operands, and `PipelineExpression.add` and `multiply` take `(second, [others])`, like the variadic Node Admin SDK functions. Further operands are sent as nested two-operand calls, `add(add(a, b), c)`, since the backend's `add` and `multiply` take exactly two operands.
+- `PipelineSource.documents()` also accepts document paths such as `'books/book1'`, validated like `Firestore.doc()`.
+- `PipelineFunctions.map()` also accepts a Dart `Map`, as in `map({'title': field('title')})`.
+- Added the `PipelineOrdering.expr` and `PipelineOrdering.direction` getters.
 
 ## 0.5.5