feat(cryo): collect the 16 canonical_execution datasets in-process

.github/workflows/golangci-lint.yml

@@ -24,5 +24,5 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0
         with:
-          version: latest
+          version: v2.12.2
           args: --timeout=10m

.github/workflows/goreleaser.yaml

@@ -56,6 +56,8 @@ jobs:
           docker context create builders
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Expose GitHub Actions cache credentials to buildx
+        uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0
       - name: Login to DockerHub
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:

.golangci.yml

@@ -14,7 +14,6 @@ linters:
     - durationcheck
     - errcheck
     - errname
-    - goconst
     - gocyclo
     - godot
     - goheader
@@ -44,9 +43,6 @@ linters:
   settings:
     errcheck:
       check-type-assertions: true
-    goconst:
-      min-len: 2
-      min-occurrences: 3
     gocritic:
       enabled-tags:
         - diagnostic
@@ -77,9 +73,6 @@ linters:
           - gosec
           - wsl
         path: _test\.go
-      - linters:
-          - goconst
-        path: cmd/tools/
     paths:
       - third_party$
       - builtin$

.goreleaser.yaml

@@ -66,6 +66,8 @@ dockers:
     build_flag_templates:
       - "--platform=linux/amd64"
       - "--provenance=false"
+      - "--cache-from=type=gha,scope=cryo-amd64"
+      - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-amd64"
       - "--build-arg=VERSION={{.Tag}}"
       - "--build-arg=GIT_COMMIT={{.ShortCommit}}"
       - "--label=org.opencontainers.image.created={{.Date}}"
@@ -91,6 +93,8 @@ dockers:
     build_flag_templates:
       - "--platform=linux/arm64/v8"
       - "--provenance=false"
+      - "--cache-from=type=gha,scope=cryo-arm64"
+      - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-arm64"
       - "--build-arg=VERSION={{.Tag}}"
       - "--build-arg=GIT_COMMIT={{.ShortCommit}}"
       - "--label=org.opencontainers.image.created={{.Date}}"

Dockerfile

@@ -1,3 +1,25 @@
+# cryo ships no release binaries at all, so it is built from a pinned commit.
+# The build must be static: cryo links glibc and OpenSSL 3 dynamically by
+# default, and neither exists on the alpine runtime.
+#
+# Pin the Rust version as well as the commit. cryo itself has been stable for
+# ~18 months, but rustc has not: 1.97 fails to compile the ethnum dependency
+# with "cannot transmute between types of different sizes", against both the
+# musl and gnu targets.
+FROM rust:1.83-alpine AS cryo-builder
+
+ARG CRYO_SHA=559b65455d7ef6b03e8e9e96a0e50fd4fe8a9c86
+
+RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig perl make git
+
+ENV OPENSSL_STATIC=1 OPENSSL_DIR=/usr
+
+RUN git clone https://github.com/paradigmxyz/cryo /src && \
+    cd /src && \
+    git checkout "${CRYO_SHA}" && \
+    cargo build --release -p cryo_cli && \
+    strip target/release/cryo
+
 FROM golang:1.25.4-alpine AS builder
 
 RUN apk add --no-cache make git ca-certificates
@@ -21,6 +43,7 @@ RUN apk --no-cache add ca-certificates && \
 
 WORKDIR /app
 
+COPY --from=cryo-builder /src/target/release/cryo /usr/local/bin/cryo
 COPY --from=builder /build/bin/execution-processor .
 
 RUN chown -R appuser:appuser /app

README.md

@@ -251,6 +251,70 @@ StructLogs -> ComputeGasUsed -> ComputeGasSelf -> ComputeMemoryWords -> Classify
                                                                      Finalize -> CallFrameRows
 ```
 
+## cryo Datasets
+
+The `cryo` processors collect the 16 `canonical_execution_*` datasets by
+invoking [cryo](https://github.com/paradigmxyz/cryo) as a subprocess, decoding
+the parquet it writes, and inserting the rows columnar via ch-go.
+
+### Groups
+
+A **group** is one cryo invocation. Because a single invocation can emit many
+datasets at once, grouping is the difference between ~628ms and ~101ms of RPC
+work per block. The group is also the unit of progress: one invocation succeeds
+or fails as a whole, so one `admin.execution_block` row represents one thing
+that actually happened.
+
+Each dataset needs exactly one source RPC method, which gives the natural
+grouping:
+
+| RPC method | datasets |
+| --- | --- |
+| `eth_getBlockByNumber` | blocks, transactions |
+| `eth_getLogs` | logs, erc20_transfers, erc721_transfers |
+| `trace_block` | traces, native_transfers, contracts, address_appearances |
+| `debug_traceBlockByNumber` | balance_reads, nonce_reads, storage_reads, four_byte_counts |
+| `trace_replayBlockTransactions` | balance_diffs, nonce_diffs, storage_diffs |
+
+Collecting all of them in one group is cheapest. Splitting into the five above
+lets datasets be cut over independently, at ~50% more RPC work per block and
+five times the ledger rows.
+
+**The group name is permanent.** It is the `processor` column in
+`admin.execution_block`, so renaming a group orphans its progress and moving a
+datatype between groups requires a re-seed.
+
+### Datatypes are not dataset names
+
+`datatypes` are cryo command-line arguments. Most name one dataset, but
+`state_diffs` is a single argument yielding `balance_diffs`, `nonce_diffs` and
+`storage_diffs` — naming those three individually in one invocation fails inside
+cryo with `Collect failed: schema not provided`.
+
+### internal_index
+
+Fourteen of the sixteen tables carry an `internal_index` column. It is not a
+dedup tiebreak: ClickHouse preserves neither insertion order nor, through a
+Distributed table, any recoverable order at all, so this column is the only
+record of the sequence cryo produced — the order of traces, logs and storage
+reads within a transaction.
+
+It is a 1-based counter per transaction hash assigned in parquet file order.
+Rows are never sorted or reordered during decode, and a row cryo could not
+attribute to a transaction is counted under the key it will be stored with
+rather than skipped.
+
+### Requirements
+
+- The `cryo` binary on `PATH` (bundled into the image; override with
+  `binaryPath`). Its version is checked at startup and exported as
+  `execution_processor_cryo_build_info`.
+- Writable scratch space. Each task creates a temp directory, runs cryo into it,
+  decodes, and removes it. Orphans left by a `SIGKILL` are swept at startup.
+- An execution node exposing `trace_*` and `debug_*` (reth or erigon). Note the
+  `debug_traceBlockByNumber`-derived datasets differ slightly between clients, so
+  keep a deployment on one client family.
+
 ## Architecture
 
 ### Leader Election

example_config.yaml

@@ -90,5 +90,59 @@ processors:
     # bufferMaxRows: 100000       # Max rows before flush (default: 100000)
     # bufferFlushInterval: "1s"   # Max time before flush (default: 1s)
 
+  # cryo processors. One config, one processor per enabled group: a group is a
+  # single cryo invocation, so it is also the unit of success, failure and
+  # progress tracking.
+  #
+  # Requires the `cryo` binary on PATH (bundled in the image).
+  cryo:
+    enabled: false
+    addr: "localhost:9000"
+    database: "default"
+    # binaryPath: "cryo"            # Resolved on PATH unless absolute
+    # tempDir: ""                   # Scratch space parent, empty = OS default
+    # fetchTimeout: "2m"            # Bounds one cryo invocation
+    # bufferMaxRows: 100000         # Per dataset, not per group (default: 100000)
+    # bufferFlushInterval: "3s"     # Per dataset (default: 3s)
+    # maxPendingBlockRange: 2       # Raise substantially for backfill
+    # globalArgs: ["--hex", "--u256-types", "string"]  # Replaces the defaults entirely
+
+    groups:
+      # The group name is the `processor` key in admin.execution_block and is
+      # therefore permanent: renaming it orphans its progress, and moving a
+      # datatype between groups requires a re-seed.
+      - name: all
+        enabled: true
+        # Datatypes are cryo arguments, not always dataset names: `state_diffs`
+        # is one argument that yields balance_diffs, nonce_diffs and
+        # storage_diffs. Collecting those three individually fails inside cryo.
+        datatypes:
+          - blocks
+          - transactions
+          - logs
+          - erc20_transfers
+          - erc721_transfers
+          - traces
+          - native_transfers
+          - contracts
+          - address_appearances
+          - balance_reads
+          - nonce_reads
+          - storage_reads
+          - four_byte_counts
+          - state_diffs
+        # minBlock: 1               # Trace-based datasets cannot trace block 0
+        # tablePrefix: ""           # Prepended to every target table name
+        # maxPendingBlockRange: 20  # Overrides the processor-wide setting
+
+      # Splitting by RPC family instead costs ~50% more per block and 5x the
+      # ledger rows, but lets datasets be cut over independently:
+      #
+      #   - name: block         datatypes: [blocks, transactions]
+      #   - name: logs          datatypes: [logs, erc20_transfers, erc721_transfers]
+      #   - name: trace_block   datatypes: [traces, native_transfers, contracts, address_appearances]
+      #   - name: debug_trace   datatypes: [balance_reads, nonce_reads, storage_reads, four_byte_counts]
+      #   - name: state_diffs   datatypes: [state_diffs]
+
 # Application settings
 shutdownTimeout: 6m

go.mod

@@ -5,6 +5,7 @@ go 1.25.4
 require (
 	github.com/ClickHouse/ch-go v0.69.0
 	github.com/alicebob/miniredis/v2 v2.36.0
+	github.com/apache/arrow-go/v18 v18.7.0
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/creasty/defaults v1.8.0
 	github.com/ethereum/go-ethereum v1.16.7
@@ -20,7 +21,7 @@ require (
 	github.com/testcontainers/testcontainers-go v0.40.0
 	github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0
 	github.com/testcontainers/testcontainers-go/modules/redis v0.40.0
-	golang.org/x/sync v0.18.0
+	golang.org/x/sync v0.22.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -29,6 +30,8 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect
+	github.com/andybalholm/brotli v1.2.2 // indirect
+	github.com/apache/thrift v0.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.24.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -40,7 +43,7 @@ require (
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/deckarep/golang-set/v2 v2.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -58,6 +61,8 @@ require (
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/goccy/go-json v0.10.6 // indirect
+	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect
@@ -67,8 +72,8 @@ require (
 	github.com/holiman/uint256 v1.3.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/klauspost/compress v1.19.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.4.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mdelapenya/tlscert v0.2.0 // indirect
@@ -84,9 +89,9 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pascaldekloe/name v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pierrec/lz4/v4 v4.1.27 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.64.0 // indirect
@@ -103,21 +108,23 @@ require (
 	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
-	go.opentelemetry.io/otel v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b // indirect
-	golang.org/x/mod v0.29.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
+	golang.org/x/mod v0.36.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
-	google.golang.org/grpc v1.78.0 // indirect
+	golang.org/x/tools v0.45.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 )