diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d5017dc..0e4b8bb 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -24,5 +24,5 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 with: - version: latest + version: v2.12.2 args: --timeout=10m diff --git a/.github/workflows/goreleaser.yaml b/.github/workflows/goreleaser.yaml index 4674fcc..c4c04c9 100644 --- a/.github/workflows/goreleaser.yaml +++ b/.github/workflows/goreleaser.yaml @@ -56,6 +56,8 @@ jobs: docker context create builders - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + - name: Expose GitHub Actions cache credentials to buildx + uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0 - name: Login to DockerHub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: diff --git a/.golangci.yml b/.golangci.yml index a189cef..337aff0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -14,7 +14,6 @@ linters: - durationcheck - errcheck - errname - - goconst - gocyclo - godot - goheader @@ -44,9 +43,6 @@ linters: settings: errcheck: check-type-assertions: true - goconst: - min-len: 2 - min-occurrences: 3 gocritic: enabled-tags: - diagnostic @@ -77,9 +73,6 @@ linters: - gosec - wsl path: _test\.go - - linters: - - goconst - path: cmd/tools/ paths: - third_party$ - builtin$ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 87edc0f..296ceab 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -66,6 +66,8 @@ dockers: build_flag_templates: - "--platform=linux/amd64" - "--provenance=false" + - "--cache-from=type=gha,scope=cryo-amd64" + - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-amd64" - "--build-arg=VERSION={{.Tag}}" - "--build-arg=GIT_COMMIT={{.ShortCommit}}" - "--label=org.opencontainers.image.created={{.Date}}" @@ -91,6 +93,8 @@ dockers: build_flag_templates: - "--platform=linux/arm64/v8" - "--provenance=false" + - "--cache-from=type=gha,scope=cryo-arm64" + - "--cache-to=type=gha,mode=max,ignore-error=true,scope=cryo-arm64" - "--build-arg=VERSION={{.Tag}}" - "--build-arg=GIT_COMMIT={{.ShortCommit}}" - "--label=org.opencontainers.image.created={{.Date}}" diff --git a/Dockerfile b/Dockerfile index 2cf3b45..108ec25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,25 @@ +# cryo ships no release binaries at all, so it is built from a pinned commit. +# The build must be static: cryo links glibc and OpenSSL 3 dynamically by +# default, and neither exists on the alpine runtime. +# +# Pin the Rust version as well as the commit. cryo itself has been stable for +# ~18 months, but rustc has not: 1.97 fails to compile the ethnum dependency +# with "cannot transmute between types of different sizes", against both the +# musl and gnu targets. +FROM rust:1.83-alpine AS cryo-builder + +ARG CRYO_SHA=559b65455d7ef6b03e8e9e96a0e50fd4fe8a9c86 + +RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig perl make git + +ENV OPENSSL_STATIC=1 OPENSSL_DIR=/usr + +RUN git clone https://github.com/paradigmxyz/cryo /src && \ + cd /src && \ + git checkout "${CRYO_SHA}" && \ + cargo build --release -p cryo_cli && \ + strip target/release/cryo + FROM golang:1.25.4-alpine AS builder RUN apk add --no-cache make git ca-certificates @@ -21,6 +43,7 @@ RUN apk --no-cache add ca-certificates && \ WORKDIR /app +COPY --from=cryo-builder /src/target/release/cryo /usr/local/bin/cryo COPY --from=builder /build/bin/execution-processor . RUN chown -R appuser:appuser /app diff --git a/README.md b/README.md index 507ac35..6843f3f 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,70 @@ StructLogs -> ComputeGasUsed -> ComputeGasSelf -> ComputeMemoryWords -> Classify Finalize -> CallFrameRows ``` +## cryo Datasets + +The `cryo` processors collect the 16 `canonical_execution_*` datasets by +invoking [cryo](https://github.com/paradigmxyz/cryo) as a subprocess, decoding +the parquet it writes, and inserting the rows columnar via ch-go. + +### Groups + +A **group** is one cryo invocation. Because a single invocation can emit many +datasets at once, grouping is the difference between ~628ms and ~101ms of RPC +work per block. The group is also the unit of progress: one invocation succeeds +or fails as a whole, so one `admin.execution_block` row represents one thing +that actually happened. + +Each dataset needs exactly one source RPC method, which gives the natural +grouping: + +| RPC method | datasets | +| --- | --- | +| `eth_getBlockByNumber` | blocks, transactions | +| `eth_getLogs` | logs, erc20_transfers, erc721_transfers | +| `trace_block` | traces, native_transfers, contracts, address_appearances | +| `debug_traceBlockByNumber` | balance_reads, nonce_reads, storage_reads, four_byte_counts | +| `trace_replayBlockTransactions` | balance_diffs, nonce_diffs, storage_diffs | + +Collecting all of them in one group is cheapest. Splitting into the five above +lets datasets be cut over independently, at ~50% more RPC work per block and +five times the ledger rows. + +**The group name is permanent.** It is the `processor` column in +`admin.execution_block`, so renaming a group orphans its progress and moving a +datatype between groups requires a re-seed. + +### Datatypes are not dataset names + +`datatypes` are cryo command-line arguments. Most name one dataset, but +`state_diffs` is a single argument yielding `balance_diffs`, `nonce_diffs` and +`storage_diffs` — naming those three individually in one invocation fails inside +cryo with `Collect failed: schema not provided`. + +### internal_index + +Fourteen of the sixteen tables carry an `internal_index` column. It is not a +dedup tiebreak: ClickHouse preserves neither insertion order nor, through a +Distributed table, any recoverable order at all, so this column is the only +record of the sequence cryo produced — the order of traces, logs and storage +reads within a transaction. + +It is a 1-based counter per transaction hash assigned in parquet file order. +Rows are never sorted or reordered during decode, and a row cryo could not +attribute to a transaction is counted under the key it will be stored with +rather than skipped. + +### Requirements + +- The `cryo` binary on `PATH` (bundled into the image; override with + `binaryPath`). Its version is checked at startup and exported as + `execution_processor_cryo_build_info`. +- Writable scratch space. Each task creates a temp directory, runs cryo into it, + decodes, and removes it. Orphans left by a `SIGKILL` are swept at startup. +- An execution node exposing `trace_*` and `debug_*` (reth or erigon). Note the + `debug_traceBlockByNumber`-derived datasets differ slightly between clients, so + keep a deployment on one client family. + ## Architecture ### Leader Election diff --git a/example_config.yaml b/example_config.yaml index ea1ae3b..81f34e2 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -90,5 +90,59 @@ processors: # bufferMaxRows: 100000 # Max rows before flush (default: 100000) # bufferFlushInterval: "1s" # Max time before flush (default: 1s) + # cryo processors. One config, one processor per enabled group: a group is a + # single cryo invocation, so it is also the unit of success, failure and + # progress tracking. + # + # Requires the `cryo` binary on PATH (bundled in the image). + cryo: + enabled: false + addr: "localhost:9000" + database: "default" + # binaryPath: "cryo" # Resolved on PATH unless absolute + # tempDir: "" # Scratch space parent, empty = OS default + # fetchTimeout: "2m" # Bounds one cryo invocation + # bufferMaxRows: 100000 # Per dataset, not per group (default: 100000) + # bufferFlushInterval: "3s" # Per dataset (default: 3s) + # maxPendingBlockRange: 2 # Raise substantially for backfill + # globalArgs: ["--hex", "--u256-types", "string"] # Replaces the defaults entirely + + groups: + # The group name is the `processor` key in admin.execution_block and is + # therefore permanent: renaming it orphans its progress, and moving a + # datatype between groups requires a re-seed. + - name: all + enabled: true + # Datatypes are cryo arguments, not always dataset names: `state_diffs` + # is one argument that yields balance_diffs, nonce_diffs and + # storage_diffs. Collecting those three individually fails inside cryo. + datatypes: + - blocks + - transactions + - logs + - erc20_transfers + - erc721_transfers + - traces + - native_transfers + - contracts + - address_appearances + - balance_reads + - nonce_reads + - storage_reads + - four_byte_counts + - state_diffs + # minBlock: 1 # Trace-based datasets cannot trace block 0 + # tablePrefix: "" # Prepended to every target table name + # maxPendingBlockRange: 20 # Overrides the processor-wide setting + + # Splitting by RPC family instead costs ~50% more per block and 5x the + # ledger rows, but lets datasets be cut over independently: + # + # - name: block datatypes: [blocks, transactions] + # - name: logs datatypes: [logs, erc20_transfers, erc721_transfers] + # - name: trace_block datatypes: [traces, native_transfers, contracts, address_appearances] + # - name: debug_trace datatypes: [balance_reads, nonce_reads, storage_reads, four_byte_counts] + # - name: state_diffs datatypes: [state_diffs] + # Application settings shutdownTimeout: 6m diff --git a/go.mod b/go.mod index 8a32246..1d2f106 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.4 require ( github.com/ClickHouse/ch-go v0.69.0 github.com/alicebob/miniredis/v2 v2.36.0 + github.com/apache/arrow-go/v18 v18.7.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/creasty/defaults v1.8.0 github.com/ethereum/go-ethereum v1.16.7 @@ -20,7 +21,7 @@ require ( github.com/testcontainers/testcontainers-go v0.40.0 github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0 github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 - golang.org/x/sync v0.18.0 + golang.org/x/sync v0.22.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -29,6 +30,8 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/apache/thrift v0.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -40,7 +43,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -58,6 +61,8 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect @@ -67,8 +72,8 @@ require ( github.com/holiman/uint256 v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect @@ -84,9 +89,9 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pascaldekloe/name v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pierrec/lz4/v4 v4.1.27 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect @@ -103,21 +108,23 @@ require ( github.com/tklauser/numcpus v0.10.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.43.0 // indirect - golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b // indirect - golang.org/x/mod v0.29.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.38.0 // indirect - google.golang.org/grpc v1.78.0 // indirect + golang.org/x/tools v0.45.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index e165a32..e8f8e65 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,12 @@ github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMG github.com/VictoriaMetrics/fastcache v1.13.0/go.mod h1:hHXhl4DA2fTL2HTZDJFXWgW0LNjo6B+4aj2Wmng3TjU= github.com/alicebob/miniredis/v2 v2.36.0 h1:yKczg+ez0bQYsG/PrgqtMMmCfl820RPu27kVGjP53eY= github.com/alicebob/miniredis/v2 v2.36.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.24.0 h1:H4x4TuulnokZKvHLfzVRTHJfFfnHEeSYJizujEZvmAM= @@ -32,6 +36,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= @@ -68,8 +74,9 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= @@ -132,6 +139,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54= github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -142,6 +151,8 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8= github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -176,10 +187,10 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -203,8 +214,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -243,8 +254,8 @@ github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcM github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -258,8 +269,9 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -276,8 +288,6 @@ github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4= github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA= github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs= github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -338,28 +348,34 @@ github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -373,35 +389,37 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b h1:18qgiDvlvH7kk8Ioa8Ov+K6xCi0GMvmGfGW0sgd/SYA= -golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= -google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= -google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index ce69206..696c9ea 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -79,7 +79,7 @@ type ClickHouseConnection struct { Port int Database string Username string - Password string //nolint:gosec // G117: test config, not a real secret + Password string } // Addr returns the ClickHouse address in host:port format. diff --git a/pkg/api/handler.go b/pkg/api/handler.go index d35bb8d..804cff3 100644 --- a/pkg/api/handler.go +++ b/pkg/api/handler.go @@ -43,7 +43,6 @@ type SingleBlockResponse struct { Processor string `json:"processor"` Queue string `json:"queue"` TransactionCount int `json:"transaction_count"` - TasksCreated int `json:"tasks_created"` } //nolint:tagliatelle // Using snake_case for API backwards compatibility @@ -51,7 +50,6 @@ type BlockResult struct { BlockNumber uint64 `json:"block_number"` Status string `json:"status"` TransactionCount int `json:"transaction_count,omitempty"` - TasksCreated int `json:"tasks_created,omitempty"` Error string `json:"error,omitempty"` } @@ -119,7 +117,6 @@ func (h *Handler) queueSingleBlock(w http.ResponseWriter, r *http.Request) { Processor: processorName, Queue: queue, TransactionCount: result.TransactionCount, - TasksCreated: result.TasksCreated, } h.writeJSON(w, http.StatusOK, response) @@ -177,7 +174,6 @@ func (h *Handler) queueMultipleBlocks(w http.ResponseWriter, r *http.Request) { BlockNumber: blockNumber, Status: "queued", TransactionCount: result.TransactionCount, - TasksCreated: result.TasksCreated, }) response.Summary.Queued++ } diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index 0e2f674..62410e0 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -13,7 +13,7 @@ type Config struct { Addr string `yaml:"addr"` // Native protocol address, e.g., "localhost:9000" Database string `yaml:"database"` // Database name, default: "default" Username string `yaml:"username"` - Password string `yaml:"password"` //nolint:gosec // G117: config field, not a hardcoded secret + Password string `yaml:"password"` // Pool settings MaxConns int32 `yaml:"max_conns"` // Maximum connections in pool, default: 10 diff --git a/pkg/ethereum/execution/embedded_node.go b/pkg/ethereum/execution/embedded_node.go index ab97c2f..8f20384 100644 --- a/pkg/ethereum/execution/embedded_node.go +++ b/pkg/ethereum/execution/embedded_node.go @@ -218,3 +218,9 @@ func (n *EmbeddedNode) IsSynced() bool { func (n *EmbeddedNode) Name() string { return n.name } + +// RPCEndpoint returns an empty string: an embedded node is driven through its +// DataSource and has no network endpoint a subprocess could dial. +func (n *EmbeddedNode) RPCEndpoint() string { + return "" +} diff --git a/pkg/ethereum/execution/geth/rpc_node.go b/pkg/ethereum/execution/geth/rpc_node.go index 6c8e469..5500d40 100644 --- a/pkg/ethereum/execution/geth/rpc_node.go +++ b/pkg/ethereum/execution/geth/rpc_node.go @@ -135,6 +135,9 @@ func (n *RPCNode) Start(ctx context.Context) error { errs := make(chan error, 1) + // Detached from the Start context on purpose: service readiness is awaited + // after Start returns, under its own 30s bound. + //nolint:gosec // G118: see above go func() { wg := sync.WaitGroup{} @@ -271,6 +274,11 @@ func (n *RPCNode) Name() string { return n.config.Name } +// RPCEndpoint returns the configured JSON-RPC URL. +func (n *RPCNode) RPCEndpoint() string { + return n.config.NodeAddress +} + // ChainID returns the chain ID from the metadata service. func (n *RPCNode) ChainID() int64 { if meta := n.Metadata(); meta != nil { diff --git a/pkg/ethereum/execution/interface.go b/pkg/ethereum/execution/interface.go index 06423f0..9fb356a 100644 --- a/pkg/ethereum/execution/interface.go +++ b/pkg/ethereum/execution/interface.go @@ -66,4 +66,8 @@ type Node interface { // Name returns the configured name for this node. Name() string + + // RPCEndpoint returns the node's JSON-RPC URL, or an empty string for + // nodes that are not reachable over the network. + RPCEndpoint() string } diff --git a/pkg/ethereum/pool_test.go b/pkg/ethereum/pool_test.go index 66a72f6..62ae7b8 100644 --- a/pkg/ethereum/pool_test.go +++ b/pkg/ethereum/pool_test.go @@ -140,6 +140,10 @@ func (m *MockNode) Name() string { return m.name } +func (m *MockNode) RPCEndpoint() string { + return "http://" + m.name + ":8545" +} + // Compile-time check that MockNode implements execution.Node. var _ execution.Node = (*MockNode)(nil) diff --git a/pkg/processor/config.go b/pkg/processor/config.go index 3daf998..6a63a66 100644 --- a/pkg/processor/config.go +++ b/pkg/processor/config.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/ethpandaops/execution-processor/pkg/processor/cryo" "github.com/ethpandaops/execution-processor/pkg/processor/tracker" "github.com/ethpandaops/execution-processor/pkg/processor/transaction/simple" "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" @@ -53,6 +54,7 @@ type Config struct { TransactionStructlog structlog.Config `yaml:"transactionStructlog"` TransactionSimple simple.Config `yaml:"transactionSimple"` TransactionStructlogAgg structlog_agg.Config `yaml:"transactionStructlogAgg"` + Cryo cryo.Config `yaml:"cryo"` } // StaleBlockDetectionConfig holds configuration for stale block detection. @@ -175,5 +177,9 @@ func (c *Config) Validate() error { return fmt.Errorf("transaction structlog_agg config validation failed: %w", err) } + if err := c.Cryo.Validate(); err != nil { + return fmt.Errorf("cryo config validation failed: %w", err) + } + return nil } diff --git a/pkg/processor/cryo/bench_test.go b/pkg/processor/cryo/bench_test.go new file mode 100644 index 0000000..85243a5 --- /dev/null +++ b/pkg/processor/cryo/bench_test.go @@ -0,0 +1,251 @@ +package cryo + +import ( + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// benchBlock is the fixture the benchmarks run against. 23000026 is the +// heaviest of the three, so it is a closer analogue of a real mainnet block +// than the lighter 23000000. +const benchBlock = "b23000026" + +// benchSink exposes the stages of a sink separately so a benchmark can +// attribute cost. The signatures are deliberately non-generic so one interface +// covers every dataset's datasetSink[R]. +type benchSink interface { + mapOnly(t *decode.Table, meta rowMeta) (int, error) + encodeOnly(t *decode.Table, meta rowMeta) error +} + +func (s *datasetSink[R]) mapOnly(t *decode.Table, meta rowMeta) (int, error) { + rows, err := s.decode(t, meta) + + return len(rows), err +} + +func (s *datasetSink[R]) encodeOnly(t *decode.Table, meta rowMeta) error { + rows, err := s.decode(t, meta) + if err != nil { + return err + } + + cols := s.acquireCols() + defer s.releaseCols(cols) + + for i := range rows { + if err := cols.Append(rows[i]); err != nil { + return err + } + } + + _ = cols.Input() + + return nil +} + +// resetChecker exposes a fill/Reset/refill comparison without leaking the +// row type, so one assertion covers all sixteen datasets. +type resetChecker interface { + checkReset(t *decode.Table, meta rowMeta) (names []string, fresh, reused []int, err error) +} + +func (s *datasetSink[R]) checkReset(t *decode.Table, meta rowMeta) ([]string, []int, []int, error) { + rows, err := s.decode(t, meta) + if err != nil { + return nil, nil, nil, err + } + + fill := func(cols columnar[R]) ([]string, []int, error) { + for i := range rows { + if appendErr := cols.Append(rows[i]); appendErr != nil { + return nil, nil, appendErr + } + } + + input := cols.Input() + names := make([]string, 0, len(input)) + counts := make([]int, 0, len(input)) + + for _, col := range input { + names = append(names, col.Name) + counts = append(counts, col.Data.Rows()) + } + + return names, counts, nil + } + + names, fresh, err := fill(s.newCols()) + if err != nil { + return nil, nil, nil, err + } + + recycled := s.newCols() + + if _, _, fillErr := fill(recycled); fillErr != nil { + return nil, nil, nil, fillErr + } + + recycled.Reset() + + _, reused, err := fill(recycled) + if err != nil { + return nil, nil, nil, err + } + + return names, fresh, reused, nil +} + +func benchLogger() logrus.FieldLogger { + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + return log +} + +func benchSinkFor(ds *Dataset) benchSink { + s, ok := ds.newSink(sinkDeps{ + log: benchLogger(), + dataset: ds, + table: ds.Table, + network: "mainnet", + processor: "cryo_bench", + bufferMaxRows: DefaultBufferMaxRows, + bufferFlushInterval: DefaultBufferFlushInterval, + }).(benchSink) + if !ok { + panic("datasetSink does not implement benchSink") + } + + return s +} + +func benchFixturePath(tb testing.TB, dataset string) string { + tb.Helper() + + matches, err := filepath.Glob(filepath.Join("testdata", benchBlock, "ethereum__"+dataset+"__*.parquet")) + if err != nil || len(matches) != 1 { + tb.Fatalf("expected one %s fixture, got %v (%v)", dataset, matches, err) + } + + return matches[0] +} + +func benchTable(tb testing.TB, dataset string) *decode.Table { + tb.Helper() + + table, err := decode.File(benchFixturePath(tb, dataset)) + if err != nil { + tb.Fatal(err) + } + + return table +} + +// BenchmarkDecodeFile measures parquet decode alone, per dataset. +func BenchmarkDecodeFile(b *testing.B) { + for _, ds := range datasets { + path := benchFixturePath(b, ds.Name) + rows := benchTable(b, ds.Name).Rows() + + b.Run(ds.Name, func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + if _, err := decode.File(path); err != nil { + b.Fatal(err) + } + } + + reportRowRate(b, rows) + }) + } +} + +// BenchmarkMapRows measures turning a decoded table into row structs, which is +// where internal_index and every null and hex decision happens. +func BenchmarkMapRows(b *testing.B) { + for _, ds := range datasets { + table := benchTable(b, ds.Name) + sink := benchSinkFor(ds) + + b.Run(ds.Name, func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + if _, err := sink.mapOnly(table, fixtureMeta); err != nil { + b.Fatal(err) + } + } + + reportRowRate(b, table.Rows()) + }) + } +} + +// BenchmarkEncodeColumns measures map plus the ch-go column block build, which +// together are the work done on every flush. +func BenchmarkEncodeColumns(b *testing.B) { + for _, ds := range datasets { + table := benchTable(b, ds.Name) + sink := benchSinkFor(ds) + + b.Run(ds.Name, func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + if err := sink.encodeOnly(table, fixtureMeta); err != nil { + b.Fatal(err) + } + } + + reportRowRate(b, table.Rows()) + }) + } +} + +// BenchmarkBlockPipeline measures everything a task does except the ClickHouse +// round trip: decode, map and encode all 16 datasets of one block. +func BenchmarkBlockPipeline(b *testing.B) { + paths := make([]string, 0, len(datasets)) + sinks := make([]benchSink, 0, len(datasets)) + total := 0 + + for _, ds := range datasets { + paths = append(paths, benchFixturePath(b, ds.Name)) + sinks = append(sinks, benchSinkFor(ds)) + total += benchTable(b, ds.Name).Rows() + } + + b.ReportAllocs() + + for b.Loop() { + for i, path := range paths { + table, err := decode.File(path) + if err != nil { + b.Fatal(err) + } + + if err := sinks[i].encodeOnly(table, fixtureMeta); err != nil { + b.Fatal(err) + } + } + } + + b.ReportMetric(float64(b.Elapsed().Milliseconds())/float64(b.N), "ms/block") + reportRowRate(b, total) +} + +func reportRowRate(b *testing.B, rows int) { + b.Helper() + + if rows == 0 || b.Elapsed() == 0 { + return + } + + b.ReportMetric(float64(rows)*float64(b.N)/b.Elapsed().Seconds(), "rows/s") +} diff --git a/pkg/processor/cryo/block_processing.go b/pkg/processor/cryo/block_processing.go new file mode 100644 index 0000000..6a9210d --- /dev/null +++ b/pkg/processor/cryo/block_processing.go @@ -0,0 +1,262 @@ +package cryo + +import ( + "context" + "errors" + "fmt" + "math/big" + + "github.com/hibiken/asynq" + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/execution-processor/pkg/common" + "github.com/ethpandaops/execution-processor/pkg/ethereum/execution" + "github.com/ethpandaops/execution-processor/pkg/processor/tracker" + "github.com/ethpandaops/execution-processor/pkg/state" +) + +// ProcessNextBlock discovers and enqueues the next block(s) for this group. +func (p *Processor) ProcessNextBlock(ctx context.Context) error { + node := p.pool.GetHealthyExecutionNode() + if node == nil { + return fmt.Errorf("no healthy execution node available") + } + + var chainHead *big.Int + + if latest, err := node.BlockNumber(ctx); err == nil && latest != nil { + chainHead = new(big.Int).SetUint64(*latest) + } + + nextBlock, err := p.stateManager.NextBlock(ctx, p.Name(), p.network.Name, p.processingMode, chainHead) + if err != nil { + if errors.Is(err, state.ErrNoMoreBlocks) { + p.log.Debug("no more blocks to process") + + return nil + } + + return fmt.Errorf("failed to get next block: %w", err) + } + + if nextBlock == nil { + p.log.Debug("no more blocks to process") + + return nil + } + + // A group cannot collect below the highest floor among its datasets, so + // descending past it in backwards mode is the end of the run rather than + // a stall. + if p.processingMode == tracker.BACKWARDS_MODE && nextBlock.Uint64() < p.group.MinBlock { + p.log.WithFields(logrus.Fields{ + "next_block": nextBlock.Uint64(), + "min_block": p.group.MinBlock, + }).Debug("Reached the group's floor, backwards processing complete") + + return nil + } + + blocked, err := p.handleBackpressure(ctx, nextBlock.Uint64()) + if err != nil { + return err + } + + if blocked { + return nil + } + + capacity, err := p.GetAvailableCapacity(ctx, nextBlock.Uint64(), p.processingMode) + if err != nil { + p.log.WithError(err).Warn("Failed to get available capacity, falling back to single block") + + capacity = 1 + } + + if capacity <= 0 { + p.log.Debug("No capacity available, waiting for tasks to complete") + + return nil + } + + blockNumbers, err := p.stateManager.NextBlocks(ctx, p.Name(), p.network.Name, p.processingMode, chainHead, capacity) + if err != nil { + p.log.WithError(err).Warn("Failed to get batch of block numbers, falling back to single block") + + blockNumbers = []*big.Int{nextBlock} + } + + if len(blockNumbers) == 0 { + return nil + } + + if leashErr := p.ValidateBatchWithinLeash(ctx, blockNumbers[0].Uint64(), len(blockNumbers), p.processingMode); leashErr != nil { + p.log.WithError(leashErr).Warn("Batch validation failed, reducing to single block") + + blockNumbers = blockNumbers[:1] + } + + blocks, err := node.BlocksByNumbers(ctx, blockNumbers) + if err != nil { + p.log.WithError(err).WithField("network", p.network.Name).Error("could not fetch blocks") + + return err + } + + if len(blocks) == 0 { + return fmt.Errorf("block %s not yet available", nextBlock.String()) + } + + for _, block := range blocks { + if err := p.ProcessBlock(ctx, block); err != nil { + return err + } + } + + return nil +} + +// handleBackpressure reports whether the pending-block window is full, and +// re-enqueues the blocking block if it has lost its Redis tracking. +func (p *Processor) handleBackpressure(ctx context.Context, nextBlock uint64) (bool, error) { + blocked, blockingBlock, err := p.IsBlockedByIncompleteBlocks(ctx, nextBlock, p.processingMode) + if err != nil { + p.log.WithError(err).Warn("Failed to check incomplete blocks distance, proceeding anyway") + + return false, nil + } + + if !blocked { + return false, nil + } + + if blockingBlock == nil { + return true, nil + } + + hasTracking, err := p.completionTracker.HasBlockTracking(ctx, *blockingBlock, p.network.Name, p.Name(), p.processingMode) + if err != nil { + p.log.WithError(err).Warn("Failed to check block tracking") + + return true, nil + } + + if hasTracking { + return true, nil + } + + p.log.WithFields(logrus.Fields{ + "blocking_block": *blockingBlock, + "next_block": nextBlock, + }).Warn("Detected orphaned block blocking progress, reprocessing") + + if err := p.ReprocessBlock(ctx, *blockingBlock); err != nil { + p.log.WithError(err).Error("Failed to reprocess orphaned block") + } + + return true, nil +} + +// ProcessBlock marks a block enqueued and queues its single collection task. +func (p *Processor) ProcessBlock(ctx context.Context, block execution.Block) error { + blockNumber := block.Number().Uint64() + + recentlyProcessed, err := p.stateManager.IsBlockRecentlyProcessed(ctx, blockNumber, p.network.Name, p.Name(), 10) + if err != nil { + p.log.WithError(err).Warn("Failed to check if block was recently processed") + } + + if recentlyProcessed { + common.BlockProcessingSkipped.WithLabelValues(p.network.Name, p.Name(), "recently_processed").Inc() + + return fmt.Errorf("block %d was recently processed", blockNumber) + } + + // Unlike the transaction processors there is no empty-block shortcut: cryo + // emits per-block rows regardless of transaction count, and a block with no + // transactions still has a header, reward traces and state diffs. + return p.enqueue(ctx, blockNumber, p.queueFor(p.processingMode), "Enqueued block for processing") +} + +// ReprocessBlock re-enqueues a block that is recorded as incomplete but has no +// Redis tracking, using the high-priority queue. +func (p *Processor) ReprocessBlock(ctx context.Context, blockNum uint64) error { + return p.enqueue(ctx, blockNum, p.reprocessQueueFor(p.processingMode), "Reprocessed orphaned block to high-priority queue") +} + +// enqueue records the block as in flight and queues its task. The ledger row is +// written before the task so that a crash between the two leaves a block the +// gap detector can see, rather than a task with nothing tracking it. +func (p *Processor) enqueue(ctx context.Context, blockNumber uint64, queue, logMessage string) error { + if err := p.stateManager.MarkBlockEnqueued(ctx, blockNumber, 1, p.network.Name, p.Name()); err != nil { + p.log.WithError(err).WithFields(logrus.Fields{ + "network": p.network.Name, + "block_number": blockNumber, + }).Error("could not mark block as enqueued") + + return err + } + + if err := p.completionTracker.RegisterBlock( + ctx, blockNumber, 1, p.network.Name, p.Name(), p.processingMode, queue, + ); err != nil { + return fmt.Errorf("failed to register block %d for completion tracking: %w", blockNumber, err) + } + + payload := &ProcessPayload{ + BlockNumber: *new(big.Int).SetUint64(blockNumber), + NetworkName: p.network.Name, + } + + task, taskID, err := p.newTask(payload, p.processingMode) + if err != nil { + return err + } + + p.deleteTaskFromMainQueue(taskID) + + err = p.EnqueueTask(ctx, task, asynq.Queue(queue), asynq.TaskID(taskID)) + + switch { + case errors.Is(err, asynq.ErrTaskIDConflict): + p.log.WithFields(logrus.Fields{ + "task_id": taskID, + "block_number": blockNumber, + }).Debug("Task already exists (TaskID conflict), skipping") + case err != nil: + return fmt.Errorf("failed to enqueue task: %w", err) + default: + common.TasksEnqueued.WithLabelValues(p.network.Name, p.Name(), queue, task.Type()).Inc() + } + + common.BlocksProcessed.WithLabelValues(p.network.Name, p.Name()).Inc() + + p.log.WithFields(logrus.Fields{ + "block_number": blockNumber, + "queue": queue, + }).Info(logMessage) + + return nil +} + +// deleteTaskFromMainQueue drops any pending copy of a task before it is +// re-enqueued elsewhere. An active task cannot be deleted, which is fine: it +// will finish and the TaskID deduplication keeps the outcome idempotent. +func (p *Processor) deleteTaskFromMainQueue(taskID string) { + if p.asynqInspector == nil { + return + } + + mainQueue := p.queueFor(p.processingMode) + + err := p.asynqInspector.DeleteTask(mainQueue, taskID) + if err == nil || errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) { + return + } + + p.log.WithFields(logrus.Fields{ + "task_id": taskID, + "queue": mainQueue, + "error": err, + }).Debug("Could not delete task from main queue (may be active)") +} diff --git a/pkg/processor/cryo/config.go b/pkg/processor/cryo/config.go new file mode 100644 index 0000000..28da9d6 --- /dev/null +++ b/pkg/processor/cryo/config.go @@ -0,0 +1,266 @@ +package cryo + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/ethpandaops/execution-processor/pkg/clickhouse" +) + +// Default buffer and execution configuration values. +const ( + DefaultBufferMaxRows = 100000 + DefaultBufferFlushInterval = 3 * time.Second + DefaultBinaryPath = "cryo" + DefaultFetchTimeout = 2 * time.Minute + + // DefaultMaxPendingBlockRange is deliberately far above the shared + // tracker.DefaultMaxPendingBlockRange. That default sizes the transaction + // processors, which fan out one task per transaction, so a handful of + // blocks already saturates the fleet. Cryo emits one task per block, so the + // pending range is the entire fan-out: at the shared default only two tasks + // would ever be in flight and every other worker would idle. + DefaultMaxPendingBlockRange = 50 +) + +// defaultGlobalArgs are passed to every cryo invocation. +// +// --hex emits 0x-prefixed lowercase strings for every byte column, and +// --u256-types string emits U256 values only as decimal strings. Both are safe +// for all datasets and remove the encoding work the target schema would +// otherwise need done after decoding. +// +// Every dataset mapping assumes this encoding, so these are applied before any +// operator arguments rather than being replaceable by them. A replacement would +// change how cryo encodes values without changing any table, so the startup +// schema check would still pass and every mapping would misread the output. +var defaultGlobalArgs = []string{"--hex", "--u256-types", "string"} + +// groupNamePattern constrains a group name to what is safe in a Redis key, four +// asynq queue names, a Prometheus label and a ClickHouse column value. +var groupNamePattern = regexp.MustCompile(`^[a-z0-9_]+$`) + +// reservedArgs are the arguments the fetcher sets itself. clap resolves +// repeated arguments last-wins, so an operator repeating one of these would +// silently take over the invocation: --output-dir would write the parquet +// outside the scratch directory, where the fetch cannot find it and the sweep +// cannot remove it, and --blocks would decouple the output from the block the +// task believes it collected. +var reservedArgs = map[string]struct{}{ + "--blocks": {}, + "-b": {}, + "--output-dir": {}, + "-o": {}, + "--no-report": {}, + "--rpc": {}, + "-r": {}, +} + +// Config holds configuration for the cryo processors. +// +// Unlike the other processors this is one config to many processors: each +// enabled group becomes its own processor with its own checkpoint, because a +// group is one cryo invocation and therefore one unit of success or failure. +type Config struct { + clickhouse.Config `yaml:",inline"` + + Enabled bool `yaml:"enabled"` + + // BinaryPath is the cryo executable, resolved on PATH when not absolute. + BinaryPath string `yaml:"binaryPath"` + + // TempDir is the parent directory for per-task scratch space. Empty uses + // the operating system default. + TempDir string `yaml:"tempDir"` + + // GlobalArgs are appended to the default cryo arguments applied to every + // group. Validate resolves this to the effective list, defaults first. + GlobalArgs []string `yaml:"globalArgs"` + + // FetchTimeout bounds a single cryo invocation. + FetchTimeout time.Duration `yaml:"fetchTimeout"` + + // BufferMaxRows and BufferFlushInterval apply per dataset, not per group, + // because one invocation writes to several tables with volumes an order of + // magnitude apart. + BufferMaxRows int `yaml:"bufferMaxRows"` + BufferFlushInterval time.Duration `yaml:"bufferFlushInterval"` + + // MaxPendingBlockRange bounds how far ahead of the oldest incomplete block + // the processor may enqueue. It defaults to this package's + // DefaultMaxPendingBlockRange rather than the shared tracker default, + // because one cryo task is a whole block instead of a single transaction, + // so this value is the fan-out rather than a multiplier on it. + MaxPendingBlockRange int `yaml:"maxPendingBlockRange"` + + Groups []GroupConfig `yaml:"groups"` +} + +// GroupConfig describes one cryo invocation and the datasets it writes. +type GroupConfig struct { + // Name is the checkpoint key in admin.execution_block, via the processor + // column, and is therefore permanent: renaming a group orphans its + // progress, and moving a datatype between groups is a re-seed. + Name string `yaml:"name"` + + Enabled bool `yaml:"enabled"` + + // Datatypes are the cryo arguments for this invocation. They are not + // always dataset names: state_diffs is one argument yielding three. + Datatypes []string `yaml:"datatypes"` + + // ExtraArgs are appended to the invocation after the global arguments. + ExtraArgs []string `yaml:"extraArgs"` + + // TablePrefix is prepended to each dataset's table name. + TablePrefix string `yaml:"tablePrefix"` + + // MinBlock overrides the lowest block this group will collect. The + // effective floor is the highest of this and its datasets' own floors. + MinBlock uint64 `yaml:"minBlock"` + + // MaxPendingBlockRange overrides the processor-wide setting. + MaxPendingBlockRange int `yaml:"maxPendingBlockRange"` +} + +// Validate checks the configuration and applies defaults. +func (c *Config) Validate() error { + if !c.Enabled { + return nil + } + + if err := c.Config.Validate(); err != nil { + return fmt.Errorf("clickhouse config validation failed: %w", err) + } + + if c.BinaryPath == "" { + c.BinaryPath = DefaultBinaryPath + } + + if err := checkReservedArgs("globalArgs", c.GlobalArgs); err != nil { + return err + } + + // Additive, not a replacement: the defaults describe the encoding every + // mapping reads, and a fresh slice keeps defaultGlobalArgs immutable. + globalArgs := make([]string, 0, len(defaultGlobalArgs)+len(c.GlobalArgs)) + globalArgs = append(globalArgs, defaultGlobalArgs...) + globalArgs = append(globalArgs, c.GlobalArgs...) + c.GlobalArgs = globalArgs + + if c.FetchTimeout <= 0 { + c.FetchTimeout = DefaultFetchTimeout + } + + // Replicas can share a temp filesystem, and the startup sweep deletes any + // scratch directory older than sweepMinAge. A fetch allowed to run that long + // would have its working directory removed underneath it by a booting + // replica. + if c.FetchTimeout >= sweepMinAge { + return fmt.Errorf( + "fetchTimeout %s must be below the orphan sweep age %s, or a restarting replica will delete an in-flight fetch's scratch directory", + c.FetchTimeout, sweepMinAge, + ) + } + + if c.MaxPendingBlockRange <= 0 { + c.MaxPendingBlockRange = DefaultMaxPendingBlockRange + } + + if c.BufferMaxRows <= 0 { + c.BufferMaxRows = DefaultBufferMaxRows + } + + if c.BufferFlushInterval <= 0 { + c.BufferFlushInterval = DefaultBufferFlushInterval + } + + enabled := 0 + seen := make(map[string]struct{}, len(c.Groups)) + claimed := make(map[string]string, len(datasets)) + + for i := range c.Groups { + group := &c.Groups[i] + + if err := group.validate(); err != nil { + return fmt.Errorf("group %d: %w", i, err) + } + + if _, dup := seen[group.Name]; dup { + return fmt.Errorf("duplicate cryo group name %q", group.Name) + } + + seen[group.Name] = struct{}{} + + if !group.Enabled { + continue + } + + enabled++ + + if group.MaxPendingBlockRange <= 0 { + group.MaxPendingBlockRange = c.MaxPendingBlockRange + } + + members, err := datasetsFor(group.Datatypes) + if err != nil { + return fmt.Errorf("group %q: %w", group.Name, err) + } + + // Two groups writing the same table would race on the same rows while + // tracking progress under separate checkpoints. + for _, ds := range members { + if owner, taken := claimed[ds.Name]; taken { + return fmt.Errorf("dataset %q is produced by both group %q and group %q", ds.Name, owner, group.Name) + } + + claimed[ds.Name] = group.Name + } + } + + if enabled == 0 { + return fmt.Errorf("cryo is enabled but no group is") + } + + return nil +} + +func (g *GroupConfig) validate() error { + if g.Name == "" { + return fmt.Errorf("name is required") + } + + if !groupNamePattern.MatchString(g.Name) { + return fmt.Errorf( + "name %q must match %s: it becomes the permanent checkpoint key in admin.execution_block, a Redis key component, four queue names and a metrics label", + g.Name, groupNamePattern, + ) + } + + if !g.Enabled { + return nil + } + + if len(g.Datatypes) == 0 { + return fmt.Errorf("group %q must list at least one datatype", g.Name) + } + + return checkReservedArgs(fmt.Sprintf("group %q extraArgs", g.Name), g.ExtraArgs) +} + +// checkReservedArgs rejects operator arguments that would override what the +// fetcher sets itself. Both the bare flag and its --flag=value form are +// matched, in long and short form. +func checkReservedArgs(field string, args []string) error { + for _, arg := range args { + flag, _, _ := strings.Cut(arg, "=") + + if _, reserved := reservedArgs[flag]; reserved { + return fmt.Errorf("%s must not set %s: the fetcher sets it and cryo takes the last value", field, flag) + } + } + + return nil +} diff --git a/pkg/processor/cryo/config_test.go b/pkg/processor/cryo/config_test.go new file mode 100644 index 0000000..91f6220 --- /dev/null +++ b/pkg/processor/cryo/config_test.go @@ -0,0 +1,211 @@ +package cryo + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// testConfig is the smallest configuration that validates: one enabled group +// collecting one datatype. +func testConfig() *Config { + cfg := &Config{ + Enabled: true, + Groups: []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}}, + } + cfg.Addr = "localhost:9000" + + return cfg +} + +func TestValidateAppliesDefaults(t *testing.T) { + t.Parallel() + + cfg := testConfig() + require.NoError(t, cfg.Validate()) + + require.Equal(t, DefaultBinaryPath, cfg.BinaryPath) + require.Equal(t, DefaultFetchTimeout, cfg.FetchTimeout) + require.Equal(t, DefaultBufferMaxRows, cfg.BufferMaxRows) + require.Equal(t, DefaultBufferFlushInterval, cfg.BufferFlushInterval) +} + +// TestValidateDefaultsMaxPendingBlockRange covers the setting that decides how +// many blocks the fleet may work on at once: one cryo task is a whole block, so +// the shared tracker default would leave nearly every worker idle. +func TestValidateDefaultsMaxPendingBlockRange(t *testing.T) { + t.Parallel() + + cfg := testConfig() + require.NoError(t, cfg.Validate()) + + require.Equal(t, DefaultMaxPendingBlockRange, cfg.MaxPendingBlockRange) + require.Equal(t, DefaultMaxPendingBlockRange, cfg.Groups[0].MaxPendingBlockRange, + "a group without an override inherits the processor-wide value") + require.Greater(t, DefaultMaxPendingBlockRange, 2, "the shared default is sized for per-transaction fan-out") +} + +func TestValidateKeepsMaxPendingBlockRangeOverrides(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.MaxPendingBlockRange = 20 + cfg.Groups = append(cfg.Groups, GroupConfig{ + Name: "u", Enabled: true, Datatypes: []string{"logs"}, MaxPendingBlockRange: 5, + }) + require.NoError(t, cfg.Validate()) + + require.Equal(t, 20, cfg.MaxPendingBlockRange) + require.Equal(t, 20, cfg.Groups[0].MaxPendingBlockRange) + require.Equal(t, 5, cfg.Groups[1].MaxPendingBlockRange) +} + +// TestValidateGlobalArgsAreAdditive is the one that matters for correctness of +// the data: dropping --hex or --u256-types leaves the tables unchanged, so the +// startup schema check still passes while every mapping misreads the output. +func TestValidateGlobalArgsAreAdditive(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.GlobalArgs = []string{"--verbose"} + require.NoError(t, cfg.Validate()) + + require.Equal(t, []string{"--hex", "--u256-types", "string", "--verbose"}, cfg.GlobalArgs) +} + +func TestValidateGlobalArgsDefaultWhenUnset(t *testing.T) { + t.Parallel() + + cfg := testConfig() + require.NoError(t, cfg.Validate()) + + require.Equal(t, defaultGlobalArgs, cfg.GlobalArgs) +} + +// TestValidateDoesNotMutateDefaultGlobalArgs guards the package-level slice: a +// merge that appended in place would leak one operator's arguments into the +// defaults every later caller sees. +func TestValidateDoesNotMutateDefaultGlobalArgs(t *testing.T) { + t.Parallel() + + before := append([]string(nil), defaultGlobalArgs...) + + cfg := testConfig() + cfg.GlobalArgs = []string{"--verbose"} + require.NoError(t, cfg.Validate()) + + cfg.GlobalArgs[0] = "--clobbered" + + require.Equal(t, before, defaultGlobalArgs) +} + +func TestValidateRejectsReservedArgs(t *testing.T) { + t.Parallel() + + for _, arg := range []string{ + "--blocks", "-b", "--output-dir", "-o", "--no-report", "--rpc", "-r", + "--blocks=1:2", "--output-dir=/tmp/elsewhere", "--rpc=http://other", + } { + t.Run("globalArgs "+arg, func(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.GlobalArgs = []string{arg} + + err := cfg.Validate() + + require.Error(t, err) + require.Contains(t, err.Error(), "globalArgs") + }) + + t.Run("extraArgs "+arg, func(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.Groups[0].ExtraArgs = []string{arg} + + err := cfg.Validate() + + require.Error(t, err) + require.Contains(t, err.Error(), "extraArgs") + }) + } +} + +func TestValidateAllowsUnreservedArgs(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.GlobalArgs = []string{"--verbose"} + cfg.Groups[0].ExtraArgs = []string{"--include-columns", "gas_limit", "--requests-per-second=50"} + + require.NoError(t, cfg.Validate()) +} + +func TestValidateGroupNames(t *testing.T) { + t.Parallel() + + valid := []string{"blocks", "state_diffs", "erc20", "a1_b2"} + for _, name := range valid { + t.Run("valid "+name, func(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.Groups[0].Name = name + + require.NoError(t, cfg.Validate()) + }) + } + + invalid := []string{"State Diffs", "state:diffs", "state-diffs", "StateDiffs", "stäte", "", "state.diffs", "state/diffs"} + for _, name := range invalid { + t.Run("invalid "+name, func(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.Groups[0].Name = name + + require.Error(t, cfg.Validate(), "a group name becomes a queue name and a permanent checkpoint key") + }) + } +} + +// TestValidateRejectsFetchTimeoutAtSweepAge covers the cross-replica hazard: a +// fetch allowed to outlive the orphan sweep age would have its scratch +// directory deleted by a booting replica sharing the temp filesystem. +func TestValidateRejectsFetchTimeoutAtSweepAge(t *testing.T) { + t.Parallel() + + for _, timeout := range []time.Duration{sweepMinAge, sweepMinAge + time.Second, 3 * sweepMinAge} { + t.Run(timeout.String(), func(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.FetchTimeout = timeout + + err := cfg.Validate() + + require.Error(t, err) + require.Contains(t, err.Error(), timeout.String()) + require.Contains(t, err.Error(), sweepMinAge.String()) + }) + } +} + +func TestValidateAllowsFetchTimeoutBelowSweepAge(t *testing.T) { + t.Parallel() + + cfg := testConfig() + cfg.FetchTimeout = sweepMinAge - time.Second + + require.NoError(t, cfg.Validate()) +} + +func TestValidateSkipsDisabledConfig(t *testing.T) { + t.Parallel() + + cfg := &Config{Groups: []GroupConfig{{Name: "not valid at all"}}} + + require.NoError(t, cfg.Validate(), "a disabled processor is not configured at all") +} diff --git a/pkg/processor/cryo/coverage.go b/pkg/processor/cryo/coverage.go new file mode 100644 index 0000000..edcf5a9 --- /dev/null +++ b/pkg/processor/cryo/coverage.go @@ -0,0 +1,75 @@ +package cryo + +import ( + "errors" + "fmt" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// ErrIncompleteCoverage reports that cryo returned successfully but its output +// does not cover the blocks that were asked for. +// +// This is the failure the per-block ledger otherwise cannot see. A node serving +// empty traces for a range it has pruned, or a cryo that skips a chunk and +// still exits zero, produces an empty parquet that is indistinguishable from a +// block with genuinely no rows — and the block is then checkpointed complete, +// so nothing ever revisits it. Production lost whole blocks of storage_reads +// exactly this way. +var ErrIncompleteCoverage = errors.New("cryo output does not cover the requested blocks") + +// verifyCoverage checks a group's output against the range it was asked for. +func verifyCoverage(g *Group, tables map[string]*decode.Table, from, to uint64) error { + expected := to - from + 1 + + for _, ds := range g.collected() { + table, ok := tables[ds.Name] + if !ok { + return fmt.Errorf("%w: dataset %s is absent", ErrIncompleteCoverage, ds.Name) + } + + if err := verifyBlockNumbers(ds.Name, table, from, to); err != nil { + return err + } + } + + // blocks is the one dataset with a knowable row count: exactly one per + // block, whatever the block contains. Every group collects it for this + // reason, so the assertion is always available. + table, ok := tables[blocksDataset.Name] + if !ok { + return fmt.Errorf("%w: blocks is absent", ErrIncompleteCoverage) + } + + if got := table.Rows(); got < 0 || uint64(got) != expected { + return fmt.Errorf("%w: blocks has %d rows, expected %d", ErrIncompleteCoverage, got, expected) + } + + return nil +} + +// verifyBlockNumbers checks that every row belongs to the requested range, so +// that output written for the wrong blocks is never attributed to these. +func verifyBlockNumbers(dataset string, table *decode.Table, from, to uint64) error { + if table.Rows() == 0 { + return nil + } + + col, err := table.Int("block_number") + if err != nil { + return fmt.Errorf("dataset %s: %w", dataset, err) + } + + for i := range table.Rows() { + if col.IsNull(i) { + return fmt.Errorf("%w: dataset %s row %d has no block number", ErrIncompleteCoverage, dataset, i) + } + + if got := col.Int(i); got < from || got > to { + return fmt.Errorf("%w: dataset %s row %d is block %d, outside %d-%d", + ErrIncompleteCoverage, dataset, i, got, from, to) + } + } + + return nil +} diff --git a/pkg/processor/cryo/coverage_test.go b/pkg/processor/cryo/coverage_test.go new file mode 100644 index 0000000..0f2d96e --- /dev/null +++ b/pkg/processor/cryo/coverage_test.go @@ -0,0 +1,175 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +func fixtureTables(t *testing.T, block string, datasets ...string) map[string]*decode.Table { + t.Helper() + + tables := make(map[string]*decode.Table, len(datasets)) + for _, ds := range datasets { + tables[ds] = fixtureTable(t, block, ds) + } + + return tables +} + +func TestVerifyCoverageAcceptsRealOutput(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks", "transactions", "logs", "state_diffs") + + tables := fixtureTables(t, "b23000000", + "blocks", "transactions", "logs", "balance_diffs", "nonce_diffs", "storage_diffs") + + require.NoError(t, verifyCoverage(group, tables, 23000000, 23000000)) +} + +// TestVerifyCoverageAcceptsGenuinelyEmptyDataset guards against the check +// becoming so strict it rejects normal output: erc721_transfers really does +// have no rows at block 23000000. +func TestVerifyCoverageAcceptsGenuinelyEmptyDataset(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks", "erc721_transfers") + tables := fixtureTables(t, "b23000000", "blocks", "erc721_transfers") + + require.Zero(t, tables["erc721_transfers"].Rows()) + require.NoError(t, verifyCoverage(group, tables, 23000000, 23000000)) +} + +// TestVerifyCoverageRejectsSilentlyEmptyBlocks is the case that lost whole +// blocks of storage_reads in production: cryo succeeds, writes an empty +// parquet, and nothing distinguishes that from a block with no rows. +func TestVerifyCoverageRejectsSilentlyEmptyBlocks(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks") + + // b23000000's blocks fixture holds block 23000000, so asking for a + // different block is the same shape as cryo returning the wrong window. + tables := fixtureTables(t, "b23000000", "blocks") + + err := verifyCoverage(group, tables, 23000001, 23000001) + + require.ErrorIs(t, err, ErrIncompleteCoverage) + require.Contains(t, err.Error(), "outside 23000001-23000001") +} + +func TestVerifyCoverageRejectsShortBlockCount(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks") + tables := fixtureTables(t, "b23000000", "blocks") + + // A range request that comes back with one block is a skipped chunk. + err := verifyCoverage(group, tables, 23000000, 23000009) + + require.ErrorIs(t, err, ErrIncompleteCoverage) + require.Contains(t, err.Error(), "blocks has 1 rows, expected 10") +} + +func TestVerifyCoverageRejectsAbsentDataset(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks", "logs") + tables := fixtureTables(t, "b23000000", "blocks") + + err := verifyCoverage(group, tables, 23000000, 23000000) + + require.ErrorIs(t, err, ErrIncompleteCoverage) + require.Contains(t, err.Error(), "dataset logs is absent") +} + +// TestVerifyCoverageCanaryCoversGroupsThatDoNotWriteBlocks is the blind spot +// that made this necessary. cryo asked for a block it cannot serve writes an +// empty parquet and exits zero, so a group of datasets that can all legitimately +// be empty has nothing to distinguish success from silence. Every group +// therefore collects blocks, whose row count is knowable, even when it does not +// write it. +func TestVerifyCoverageCanaryCoversGroupsThatDoNotWriteBlocks(t *testing.T) { + t.Parallel() + + group := testGroup(t, "logs", "erc20_transfers", "erc721_transfers") + + require.NotNil(t, group.Canary, "a group without blocks must collect it as a canary") + require.Contains(t, group.Datatypes, "blocks", "the canary must be asked of cryo") + require.NotContains(t, datasetNames(group.Datasets), "blocks", "the canary must not be written") + + tables := fixtureTables(t, "b23000000", "logs", "erc20_transfers", "erc721_transfers", "blocks") + require.NoError(t, verifyCoverage(group, tables, 23000000, 23000000)) + + // Every dataset empty, which is exactly what a node that cannot serve the + // range produces, and what the old pipeline recorded as success. + delete(tables, "blocks") + require.ErrorIs(t, verifyCoverage(group, tables, 23000000, 23000000), ErrIncompleteCoverage) +} + +func TestNewGroupDoesNotDuplicateBlocksCanary(t *testing.T) { + t.Parallel() + + group := testGroup(t, "blocks", "transactions") + + require.Nil(t, group.Canary, "a group that already writes blocks needs no canary") + require.Equal(t, []string{"blocks", "transactions"}, group.Datatypes) +} + +func datasetNames(datasets []*Dataset) []string { + names := make([]string, 0, len(datasets)) + for _, ds := range datasets { + names = append(names, ds.Name) + } + + return names +} + +// TestGroupRequiresGasLimitColumn pins the one column cryo omits unless asked. +// Losing it does not fail: ClickHouse writes zero for every row, which is how +// staging silently held gas_limit = 0. +func TestGroupRequiresGasLimitColumn(t *testing.T) { + t.Parallel() + + require.Equal(t, []string{"gas_limit"}, testGroup(t, "blocks").RequiredColumns()) + + // Reached through the canary too, so no group can lose it. + require.Equal(t, []string{"gas_limit"}, testGroup(t, "logs").RequiredColumns()) +} + +// TestColumnBlocksResetCompletely guards the column pool. Reset was dead code +// until blocks started being recycled; a Reset that misses a column now leaves +// the previous batch's rows in place, so the next flush builds a block whose +// columns disagree on length. +func TestColumnBlocksResetCompletely(t *testing.T) { + t.Parallel() + + for _, ds := range datasets { + t.Run(ds.Name, func(t *testing.T) { + t.Parallel() + + table := fixtureTable(t, "b23000026", ds.Name) + if table.Rows() == 0 { + t.Skipf("%s has no rows at this block", ds.Name) + } + + checker, ok := ds.newSink(sinkDeps{ + log: benchLogger(), dataset: ds, table: ds.Table, network: "mainnet", + }).(resetChecker) + require.True(t, ok) + + names, fresh, reused, err := checker.checkReset(table, fixtureMeta) + require.NoError(t, err) + require.NotEmpty(t, fresh) + + for i := range fresh { + require.Equal(t, fresh[i], reused[i], + "column %q holds %d rows after Reset+refill but %d in a fresh block: Reset is incomplete", + names[i], reused[i], fresh[i]) + } + }) + } +} diff --git a/pkg/processor/cryo/dataset.go b/pkg/processor/cryo/dataset.go new file mode 100644 index 0000000..62b0479 --- /dev/null +++ b/pkg/processor/cryo/dataset.go @@ -0,0 +1,119 @@ +package cryo + +import ( + "fmt" + "sort" +) + +// Dataset describes one cryo datatype and how its rows land in ClickHouse. +type Dataset struct { + // Name is the cryo dataset name, which is also the parquet file's infix. + Name string + // Table is the target ClickHouse table. + Table string + // InternalIndex marks datasets whose target table carries internal_index. + InternalIndex bool + // MinBlock is the lowest block the dataset can be collected for. Trace + // based datasets cannot trace the genesis block. + MinBlock uint64 + // RequiredColumns are parquet columns cryo only emits when explicitly + // asked, which the group must therefore request. + RequiredColumns []string + // newSink builds the per-dataset row buffer and column mapping. + newSink func(sinkDeps) sink +} + +// Datatype is a cryo command-line argument, which is not always a dataset: the +// state diff datatypes can only be collected through the state_diffs +// meta-datatype, which yields three datasets from one collection. +type Datatype struct { + // Name is the argument passed to cryo. + Name string + // Datasets are the dataset names cryo writes for this datatype. + Datasets []string +} + +// datatypes lists every cryo datatype the processor knows how to collect. +// +// Naming two members of the state diff family in one invocation fails inside +// cryo with "Collect failed: schema not provided", so the meta-datatype is the +// only way to collect them alongside anything else. +var datatypes = []Datatype{ + {Name: "blocks", Datasets: []string{"blocks"}}, + {Name: "transactions", Datasets: []string{"transactions"}}, + {Name: "logs", Datasets: []string{"logs"}}, + {Name: "erc20_transfers", Datasets: []string{"erc20_transfers"}}, + {Name: "erc721_transfers", Datasets: []string{"erc721_transfers"}}, + {Name: "traces", Datasets: []string{"traces"}}, + {Name: "native_transfers", Datasets: []string{"native_transfers"}}, + {Name: "contracts", Datasets: []string{"contracts"}}, + {Name: "address_appearances", Datasets: []string{"address_appearances"}}, + {Name: "balance_reads", Datasets: []string{"balance_reads"}}, + {Name: "nonce_reads", Datasets: []string{"nonce_reads"}}, + {Name: "storage_reads", Datasets: []string{"storage_reads"}}, + {Name: "four_byte_counts", Datasets: []string{"four_byte_counts"}}, + {Name: "state_diffs", Datasets: []string{"balance_diffs", "nonce_diffs", "storage_diffs"}}, +} + +// datatypeByName indexes datatypes for config validation. +var datatypeByName = func() map[string]Datatype { + byName := make(map[string]Datatype, len(datatypes)) + for _, dt := range datatypes { + byName[dt.Name] = dt + } + + return byName +}() + +// datasetByName indexes every dataset the processor can write. +var datasetByName = func() map[string]*Dataset { + byName := make(map[string]*Dataset, len(datasets)) + for _, ds := range datasets { + byName[ds.Name] = ds + } + + return byName +}() + +// DatatypeNames returns every collectable cryo datatype, sorted. +func DatatypeNames() []string { + names := make([]string, 0, len(datatypes)) + for _, dt := range datatypes { + names = append(names, dt.Name) + } + + sort.Strings(names) + + return names +} + +// datasetsFor expands datatype names into the datasets they produce, in the +// order the datatypes were listed. +func datasetsFor(datatypeNames []string) ([]*Dataset, error) { + out := make([]*Dataset, 0, len(datatypeNames)) + seen := make(map[string]struct{}, len(datatypeNames)) + + for _, name := range datatypeNames { + dt, ok := datatypeByName[name] + if !ok { + return nil, fmt.Errorf("unknown cryo datatype %q, expected one of %v", name, DatatypeNames()) + } + + if _, dup := seen[name]; dup { + return nil, fmt.Errorf("cryo datatype %q listed more than once", name) + } + + seen[name] = struct{}{} + + for _, dsName := range dt.Datasets { + ds, known := datasetByName[dsName] + if !known { + return nil, fmt.Errorf("datatype %q yields unregistered dataset %q", name, dsName) + } + + out = append(out, ds) + } + } + + return out, nil +} diff --git a/pkg/processor/cryo/datasets.go b/pkg/processor/cryo/datasets.go new file mode 100644 index 0000000..16b4502 --- /dev/null +++ b/pkg/processor/cryo/datasets.go @@ -0,0 +1,22 @@ +package cryo + +// datasets is every dataset the processor can write. Each descriptor lives +// beside its column mapping in the matching ds_*.go file. +var datasets = []*Dataset{ + blocksDataset, + transactionsDataset, + logsDataset, + erc20TransfersDataset, + erc721TransfersDataset, + tracesDataset, + nativeTransfersDataset, + contractsDataset, + addressAppearancesDataset, + balanceReadsDataset, + nonceReadsDataset, + storageReadsDataset, + fourByteCountsDataset, + balanceDiffsDataset, + nonceDiffsDataset, + storageDiffsDataset, +} diff --git a/pkg/processor/cryo/decode/decode.go b/pkg/processor/cryo/decode/decode.go new file mode 100644 index 0000000..899f60a --- /dev/null +++ b/pkg/processor/cryo/decode/decode.go @@ -0,0 +1,334 @@ +// Package decode reads cryo's parquet output into name-keyed columns. +// +// cryo writes flat, single-level parquet via Polars: every column is an +// OPTIONAL leaf of physical type BYTE_ARRAY, INT32, INT64, DOUBLE or BOOLEAN, +// LZ4_RAW compressed. There are no nested, list, struct or INT96 columns. +// +// Row order is preserved exactly as written, because it carries information +// the target schema encodes as internal_index and cannot recover otherwise. +package decode + +import ( + "errors" + "fmt" + + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" +) + +const ( + // batchSize is the number of values requested per ReadBatch call. + batchSize = 8192 + + // maxPrealloc caps how much a file's declared row count may reserve up + // front, so a corrupt footer costs an allocation rather than the process. + maxPrealloc = 1 << 20 +) + +// Kind is the decoded Go representation of a parquet column. +type Kind uint8 + +const ( + // KindString holds BYTE_ARRAY values as Go strings. + KindString Kind = iota + // KindInt holds INT32/INT64 values widened to uint64. + KindInt + // KindFloat holds DOUBLE values. + KindFloat + // KindBool holds BOOLEAN values. + KindBool +) + +func (k Kind) String() string { + switch k { + case KindString: + return "string" + case KindInt: + return "int" + case KindFloat: + return "float" + case KindBool: + return "bool" + default: + return "unknown" + } +} + +// ErrColumnMissing is returned when a requested column is absent from the file. +var ErrColumnMissing = errors.New("column missing from parquet file") + +// Column holds one decoded parquet column. Only the slice matching Kind is +// populated; valid carries the per-row null mask. +type Column struct { + name string + kind Kind + valid []bool + strs []string + ints []uint64 + flts []float64 + bools []bool +} + +// Name returns the parquet column name. +func (c *Column) Name() string { return c.name } + +// Kind returns the decoded representation of the column. +func (c *Column) Kind() Kind { return c.kind } + +// IsNull reports whether row i is null. +func (c *Column) IsNull(i int) bool { return !c.valid[i] } + +// Str returns the string value at row i. The result is meaningless when the +// row is null; callers must check IsNull first. +func (c *Column) Str(i int) string { return c.strs[i] } + +// Int returns the integer value at row i. +func (c *Column) Int(i int) uint64 { return c.ints[i] } + +// Float returns the float value at row i. +func (c *Column) Float(i int) float64 { return c.flts[i] } + +// Bool returns the boolean value at row i. +func (c *Column) Bool(i int) bool { return c.bools[i] } + +// Table holds the decoded columns of one parquet file. +type Table struct { + rows int + cols map[string]*Column +} + +// Rows returns the row count. +func (t *Table) Rows() int { return t.rows } + +// Names returns the decoded column names. +func (t *Table) Names() []string { + names := make([]string, 0, len(t.cols)) + for name := range t.cols { + names = append(names, name) + } + + return names +} + +func (t *Table) column(name string, kind Kind) (*Column, error) { + col, ok := t.cols[name] + if !ok { + return nil, fmt.Errorf("%w: %q", ErrColumnMissing, name) + } + + if col.kind != kind { + return nil, fmt.Errorf("column %q is %s, want %s", name, col.kind, kind) + } + + return col, nil +} + +// Str returns the named column, requiring it to hold strings. +func (t *Table) Str(name string) (*Column, error) { return t.column(name, KindString) } + +// Int returns the named column, requiring it to hold integers. +func (t *Table) Int(name string) (*Column, error) { return t.column(name, KindInt) } + +// Float returns the named column, requiring it to hold floats. +func (t *Table) Float(name string) (*Column, error) { return t.column(name, KindFloat) } + +// Bool returns the named column, requiring it to hold booleans. +func (t *Table) Bool(name string) (*Column, error) { return t.column(name, KindBool) } + +// File decodes every column of a parquet file. +func File(path string) (*Table, error) { + rdr, err := file.OpenParquetFile(path, false) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + + defer func() { _ = rdr.Close() }() + + sch := rdr.MetaData().Schema + + indexes := make(map[string]int, sch.NumColumns()) + for i := range sch.NumColumns() { + indexes[sch.Column(i).Name()] = i + } + + rows := int(rdr.NumRows()) + + table := &Table{rows: rows, cols: make(map[string]*Column, len(indexes))} + + for name, idx := range indexes { + col, colErr := newColumn(sch.Column(idx), rows) + if colErr != nil { + return nil, fmt.Errorf("%s: %w", path, colErr) + } + + table.cols[name] = col + } + + for rg := range rdr.NumRowGroups() { + group := rdr.RowGroup(rg) + + for name, idx := range indexes { + reader, colErr := group.Column(idx) + if colErr != nil { + return nil, fmt.Errorf("%s: column %q row group %d: %w", path, name, rg, colErr) + } + + if readErr := table.cols[name].read(reader); readErr != nil { + return nil, fmt.Errorf("%s: column %q row group %d: %w", path, name, rg, readErr) + } + } + } + + for name, col := range table.cols { + if got := len(col.valid); got != rows { + return nil, fmt.Errorf("%s: column %q decoded %d rows, file declares %d", path, name, got, rows) + } + } + + return table, nil +} + +// newColumn allocates a column sized for the whole file from its schema entry. +func newColumn(desc *schema.Column, rows int) (*Column, error) { + if desc.MaxRepetitionLevel() != 0 { + return nil, fmt.Errorf("column %q is repeated, which cryo never emits", desc.Name()) + } + + prealloc := min(rows, maxPrealloc) + + col := &Column{name: desc.Name(), valid: make([]bool, 0, prealloc)} + + switch desc.PhysicalType() { + case parquet.Types.ByteArray: + col.kind = KindString + col.strs = make([]string, 0, prealloc) + case parquet.Types.Int32, parquet.Types.Int64: + // Parquet stores unsigned integers in signed physical types, so a set + // sign bit is data rather than an error. Only the logical type says + // which it is, and cryo emits nothing signed. + if lt, ok := desc.LogicalType().(schema.IntLogicalType); ok && lt.IsSigned() { + return nil, fmt.Errorf("column %q is a signed integer, which cryo never emits", desc.Name()) + } + + col.kind = KindInt + col.ints = make([]uint64, 0, prealloc) + case parquet.Types.Double: + col.kind = KindFloat + col.flts = make([]float64, 0, prealloc) + case parquet.Types.Boolean: + col.kind = KindBool + col.bools = make([]bool, 0, prealloc) + case parquet.Types.Float, parquet.Types.Int96, parquet.Types.FixedLenByteArray, parquet.Types.Undefined: + return nil, fmt.Errorf("column %q has unsupported physical type %s", desc.Name(), desc.PhysicalType()) + default: + return nil, fmt.Errorf("column %q has unsupported physical type %s", desc.Name(), desc.PhysicalType()) + } + + return col, nil +} + +// read appends one row group's worth of values. +func (c *Column) read(reader file.ColumnChunkReader) error { + switch typed := reader.(type) { + case *file.ByteArrayColumnChunkReader: + return drain(c, typed, make([]parquet.ByteArray, batchSize), typed.ReadBatch, + func(col *Column, v parquet.ByteArray) error { + col.strs = append(col.strs, string(v)) + + return nil + }, + func(col *Column) { col.strs = append(col.strs, "") }) + case *file.Int32ColumnChunkReader: + return drain(c, typed, make([]int32, batchSize), typed.ReadBatch, appendInt32, padInt) + case *file.Int64ColumnChunkReader: + return drain(c, typed, make([]int64, batchSize), typed.ReadBatch, appendInt64, padInt) + case *file.Float64ColumnChunkReader: + return drain(c, typed, make([]float64, batchSize), typed.ReadBatch, + func(col *Column, v float64) error { + col.flts = append(col.flts, v) + + return nil + }, + func(col *Column) { col.flts = append(col.flts, 0) }) + case *file.BooleanColumnChunkReader: + return drain(c, typed, make([]bool, batchSize), typed.ReadBatch, + func(col *Column, v bool) error { + col.bools = append(col.bools, v) + + return nil + }, + func(col *Column) { col.bools = append(col.bools, false) }) + default: + return fmt.Errorf("column %q has unsupported reader %T", c.name, reader) + } +} + +// readBatchFunc matches the generated ReadBatch method of every typed reader. +type readBatchFunc[T any] func(batchSize int64, values []T, defLvls, repLvls []int16) (int64, int, error) + +// drain reads a column chunk to exhaustion. ReadBatch returns values densely +// packed — nulls are absent — so each batch is expanded against its definition +// levels to keep every row at its original index. +func drain[T any]( + c *Column, + reader file.ColumnChunkReader, + values []T, + readBatch readBatchFunc[T], + appendValue func(*Column, T) error, + appendNull func(*Column), +) error { + maxDef := reader.Descriptor().MaxDefinitionLevel() + defLvls := make([]int16, batchSize) + + for reader.HasNext() { + total, read, err := readBatch(batchSize, values, defLvls, nil) + if err != nil { + return err + } + + if total == 0 { + break + } + + var next int + + for _, def := range defLvls[:total] { + if def < maxDef { + c.valid = append(c.valid, false) + appendNull(c) + + continue + } + + if next >= read { + return fmt.Errorf("column %q: batch declared %d values but only %d were read", c.name, next+1, read) + } + + if err := appendValue(c, values[next]); err != nil { + return err + } + + c.valid = append(c.valid, true) + next++ + } + } + + return reader.Err() +} + +func appendInt32(c *Column, v int32) error { + //nolint:gosec // G115: reinterpreting the bits is the point; the logical type is unsigned + c.ints = append(c.ints, uint64(uint32(v))) + + return nil +} + +func appendInt64(c *Column, v int64) error { + //nolint:gosec // G115: reinterpreting the bits is the point; the logical type is unsigned + c.ints = append(c.ints, uint64(v)) + + return nil +} + +func padInt(c *Column) { c.ints = append(c.ints, 0) } diff --git a/pkg/processor/cryo/decode/decode_test.go b/pkg/processor/cryo/decode/decode_test.go new file mode 100644 index 0000000..9937873 --- /dev/null +++ b/pkg/processor/cryo/decode/decode_test.go @@ -0,0 +1,150 @@ +package decode_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/stretchr/testify/require" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// writeUnsignedParquet builds a file shaped like cryo's output: optional, +// unsigned integer leaves. +func writeUnsignedParquet(t *testing.T, u32 []int32, u64 []int64) string { + t.Helper() + + u32Type, err := schema.NewPrimitiveNodeLogical( + "u32", parquet.Repetitions.Optional, schema.NewIntLogicalType(32, false), + parquet.Types.Int32, 0, 1) + require.NoError(t, err) + + u64Type, err := schema.NewPrimitiveNodeLogical( + "u64", parquet.Repetitions.Optional, schema.NewIntLogicalType(64, false), + parquet.Types.Int64, 0, 2) + require.NoError(t, err) + + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Required, + schema.FieldList{u32Type, u64Type}, -1) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "unsigned.parquet") + + f, err := os.Create(path) + require.NoError(t, err) + + writer := file.NewParquetWriter(f, root) + + rg, err := writer.AppendRowGroupChecked() + require.NoError(t, err) + + defLvls := make([]int16, len(u32)) + for i := range defLvls { + defLvls[i] = 1 + } + + col32, err := rg.NextColumn() + require.NoError(t, err) + + writer32, ok := col32.(*file.Int32ColumnChunkWriter) + require.True(t, ok) + + _, err = writer32.WriteBatch(u32, defLvls, nil) + require.NoError(t, err) + require.NoError(t, col32.Close()) + + col64, err := rg.NextColumn() + require.NoError(t, err) + + writer64, ok := col64.(*file.Int64ColumnChunkWriter) + require.True(t, ok) + + _, err = writer64.WriteBatch(u64, defLvls, nil) + require.NoError(t, err) + require.NoError(t, col64.Close()) + + require.NoError(t, rg.Close()) + require.NoError(t, writer.Close()) + + return path +} + +// TestFileReadsUnsignedValuesWithTheSignBitSet is the 2038 case. Parquet stores +// unsigned integers in signed physical types, so cryo's uint32 block timestamps +// carry a set sign bit from 2038-01-19 onward, and every uint64 column does the +// same beyond 2^63. Treating that as corruption would wedge the processor on +// every block from that date. +func TestFileReadsUnsignedValuesWithTheSignBitSet(t *testing.T) { + t.Parallel() + + // 2038-01-19T03:14:08Z is the first second that does not fit in an int32. + var ( + afterY2038 = uint64(2147483648) + maxUint32 = uint64(4294967295) + beyondInt64 = uint64(1) << 63 + maxUint64 = ^uint64(0) + ) + + path := writeUnsignedParquet(t, + []int32{1, int32(afterY2038), int32(maxUint32)}, + []int64{1, int64(beyondInt64), int64(maxUint64)}, + ) + + table, err := decode.File(path) + require.NoError(t, err, "an unsigned value with the sign bit set must decode, not error") + + u32, err := table.Int("u32") + require.NoError(t, err) + + require.Equal(t, uint64(1), u32.Int(0)) + require.Equal(t, afterY2038, u32.Int(1)) + require.Equal(t, maxUint32, u32.Int(2)) + + u64, err := table.Int("u64") + require.NoError(t, err) + + require.Equal(t, uint64(1), u64.Int(0)) + require.Equal(t, beyondInt64, u64.Int(1)) + require.Equal(t, maxUint64, u64.Int(2)) +} + +func TestFileRejectsSignedIntegerColumn(t *testing.T) { + t.Parallel() + + signed, err := schema.NewPrimitiveNodeLogical( + "s32", parquet.Repetitions.Optional, schema.NewIntLogicalType(32, true), + parquet.Types.Int32, 0, 1) + require.NoError(t, err) + + root, err := schema.NewGroupNode("schema", parquet.Repetitions.Required, schema.FieldList{signed}, -1) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "signed.parquet") + + f, err := os.Create(path) + require.NoError(t, err) + + writer := file.NewParquetWriter(f, root) + + rg, err := writer.AppendRowGroupChecked() + require.NoError(t, err) + + col, err := rg.NextColumn() + require.NoError(t, err) + + colWriter, ok := col.(*file.Int32ColumnChunkWriter) + require.True(t, ok) + + _, err = colWriter.WriteBatch([]int32{-1}, []int16{1}, nil) + require.NoError(t, err) + require.NoError(t, col.Close()) + require.NoError(t, rg.Close()) + require.NoError(t, writer.Close()) + + _, err = decode.File(path) + require.ErrorContains(t, err, "signed integer") +} diff --git a/pkg/processor/cryo/ds_address_appearances.go b/pkg/processor/cryo/ds_address_appearances.go new file mode 100644 index 0000000..7b27116 --- /dev/null +++ b/pkg/processor/cryo/ds_address_appearances.go @@ -0,0 +1,121 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var addressAppearancesDataset = &Dataset{ + Name: "address_appearances", + Table: "canonical_execution_address_appearances", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeAddressAppearances, func() columnar[addressAppearanceRow] { return newAddressAppearanceColumns() }) + }, +} + +type addressAppearanceRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionHash string + InternalIndex uint32 + Address string + Relationship string + MetaNetworkName string +} + +func decodeAddressAppearances(t *decode.Table, meta rowMeta) ([]addressAppearanceRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionHash = p.str("transaction_hash") + address = p.str("address") + relationship = p.str("relationship") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]addressAppearanceRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, addressAppearanceRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + Relationship: strOrEmpty(relationship, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type addressAppearanceColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + Relationship *proto.ColLowCardinality[string] + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newAddressAppearanceColumns() *addressAppearanceColumns { + return &addressAppearanceColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + Relationship: new(proto.ColStr).LowCardinality(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *addressAppearanceColumns) Append(r addressAppearanceRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.Relationship.Append(r.Relationship) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *addressAppearanceColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.Relationship.Reset() + c.MetaNetworkName.Reset() +} + +func (c *addressAppearanceColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *addressAppearanceColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "relationship", Data: c.Relationship}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_address_appearances_test.go b/pkg/processor/cryo/ds_address_appearances_test.go new file mode 100644 index 0000000..8f8ee15 --- /dev/null +++ b/pkg/processor/cryo/ds_address_appearances_test.go @@ -0,0 +1,101 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeAddressAppearances(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "address_appearances", decodeAddressAppearances) + require.NoError(t, err) + require.Len(t, rows, 2309) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, "0x0511b790404361753f71d85f71714f05bdf5459255960856bd8fcb37e0f7af5a", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", first.Address) + require.Equal(t, "miner_fee", first.Relationship) + require.Equal(t, "mainnet", first.MetaNetworkName) + + // One address appears four times in this transaction under four + // relationships, each a distinct row. + for i, relationship := range []string{"call_from", "erc20_transfer_from", "erc20_transfer_from_to", "tx_from"} { + row := rows[i+1] + require.Equal(t, first.TransactionHash, row.TransactionHash) + require.Equal(t, uint32(i+2), row.InternalIndex) + require.Equal(t, "0x68770d610c6cb38493416eca5fdbd7e053c12c86", row.Address) + require.Equal(t, relationship, row.Relationship) + } + + // The seventh row closes the first transaction; the eighth starts the next. + require.Equal(t, first.TransactionHash, rows[6].TransactionHash) + require.Equal(t, uint32(7), rows[6].InternalIndex) + require.Equal(t, "tx_to", rows[6].Relationship) + require.Equal(t, "0x0637faa52c470dd7ea0c434db9cc82b7f0f059fd1123c8e914de5d434db0d1c3", rows[7].TransactionHash) + require.Equal(t, uint32(1), rows[7].InternalIndex) + + last := rows[2308] + require.Equal(t, "0xffbd484e846b720325ef6fa8302cb2b73b4df87a9412e8a298e25c1876daa6ec", last.TransactionHash) + require.Equal(t, "0x4656e33ce80748db2b2816e00eb115c0f6183f63", last.Address) + require.Equal(t, "tx_from", last.Relationship) +} + +// TestDecodeAddressAppearancesInternalIndex pins the one column that carries +// information nothing else in the row does: 1047 of the 2309 rows repeat a +// (transaction, address, relationship) triple, so without internal_index the +// ReplacingMergeTree sort key would collapse them. +func TestDecodeAddressAppearancesInternalIndex(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "address_appearances", decodeAddressAppearances) + require.NoError(t, err) + + type triple struct { + hash string + address string + relationship string + } + + type sortKey struct { + hash string + index uint32 + } + + var ( + next = make(map[string]uint32) + triples = make(map[triple]struct{}) + keys = make(map[sortKey]struct{}) + ) + + for i, r := range rows { + next[r.TransactionHash]++ + require.Equal(t, next[r.TransactionHash], r.InternalIndex, "row %d", i) + + triples[triple{r.TransactionHash, r.Address, r.Relationship}] = struct{}{} + keys[sortKey{r.TransactionHash, r.InternalIndex}] = struct{}{} + } + + require.Len(t, next, 137) + require.Len(t, triples, 1262) + require.Len(t, keys, len(rows), "internal_index must make every row's sort key unique") +} + +func TestAddressAppearancesColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "address_appearances", decodeAddressAppearances) + require.NoError(t, err) + + cols := newAddressAppearanceColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_address_appearances", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_balance_diffs.go b/pkg/processor/cryo/ds_balance_diffs.go new file mode 100644 index 0000000..a33c376 --- /dev/null +++ b/pkg/processor/cryo/ds_balance_diffs.go @@ -0,0 +1,147 @@ +package cryo + +import ( + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var balanceDiffsDataset = &Dataset{ + Name: "balance_diffs", + Table: "canonical_execution_balance_diffs", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeBalanceDiffs, func() columnar[balanceDiffRow] { return newBalanceDiffColumns() }) + }, +} + +type balanceDiffRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Address string + // FromValue and ToValue stay decimal strings until Append, because balances + // exceed uint64 and only proto.UInt256 can hold them. + FromValue string + ToValue string + MetaNetworkName string +} + +func decodeBalanceDiffs(t *decode.Table, meta rowMeta) ([]balanceDiffRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + address = p.str("address") + fromValue = p.str("from_value_string") + toValue = p.str("to_value_string") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]balanceDiffRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, balanceDiffRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + FromValue: strOrEmpty(fromValue, i), + ToValue: strOrEmpty(toValue, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type balanceDiffColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + FromValue proto.ColUInt256 + ToValue proto.ColUInt256 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newBalanceDiffColumns() *balanceDiffColumns { + return &balanceDiffColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *balanceDiffColumns) Append(r balanceDiffRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + fromValue, err := uint256(r.FromValue) + if err != nil { + return fmt.Errorf("from_value: %w", err) + } + + toValue, err := uint256(r.ToValue) + if err != nil { + return fmt.Errorf("to_value: %w", err) + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.FromValue.Append(fromValue) + c.ToValue.Append(toValue) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *balanceDiffColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.FromValue.Reset() + c.ToValue.Reset() + c.MetaNetworkName.Reset() +} + +func (c *balanceDiffColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *balanceDiffColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "from_value", Data: &c.FromValue}, + {Name: "to_value", Data: &c.ToValue}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_balance_diffs_test.go b/pkg/processor/cryo/ds_balance_diffs_test.go new file mode 100644 index 0000000..24ada37 --- /dev/null +++ b/pkg/processor/cryo/ds_balance_diffs_test.go @@ -0,0 +1,65 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeBalanceDiffs(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "balance_diffs", decodeBalanceDiffs) + require.NoError(t, err) + require.Len(t, rows, 377) + + r := rows[0] + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", r.Address) + require.Equal(t, "mainnet", r.MetaNetworkName) + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + + // Balances routinely exceed uint64, so they stay decimal strings until the + // UInt256 column takes them. + require.Equal(t, "38432751121515279399317", r.FromValue) + require.Equal(t, "38432750982043899049813", r.ToValue) + + require.Equal(t, uint32(2), rows[1].InternalIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", rows[1].Address) + require.Equal(t, uint32(1), rows[2].InternalIndex) + require.Equal(t, "0x6747dea6d070127292e2a600d879ac67b986253088e82de01d6bfa92e7ffb90c", rows[2].TransactionHash) + require.Equal(t, uint32(3), rows[4].InternalIndex) + + last := rows[376] + require.Equal(t, uint64(136), last.TransactionIndex) + require.Equal(t, "0x1894e4033d2ecb3f3b3bfa8b2c82c213593965754b40042a4ec392b2733ba76c", last.TransactionHash) + require.Equal(t, uint32(2), last.InternalIndex) + require.Equal(t, "0x5995510b29924a0c68e5e21cb95ac426519c43bf", last.Address) + require.Equal(t, "162525574300734667111", last.FromValue) + require.Equal(t, "162549100389846748107", last.ToValue) +} + +func TestBalanceDiffsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "balance_diffs", decodeBalanceDiffs) + require.NoError(t, err) + + cols := newBalanceDiffColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + // 38432751121515279399317 does not fit in a uint64, and a float would lose + // its low digits. + require.Equal(t, uint64(0x71909f9361e1b995), cols.FromValue[0].Low.Low) + require.Equal(t, uint64(0x823), cols.FromValue[0].Low.High) + require.Equal(t, uint64(0), cols.FromValue[0].High.Low) + + requireInputMatchesDDL(t, "canonical_execution_balance_diffs", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_balance_reads.go b/pkg/processor/cryo/ds_balance_reads.go new file mode 100644 index 0000000..fcb89a1 --- /dev/null +++ b/pkg/processor/cryo/ds_balance_reads.go @@ -0,0 +1,132 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var balanceReadsDataset = &Dataset{ + Name: "balance_reads", + Table: "canonical_execution_balance_reads", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeBalanceReads, func() columnar[balanceReadRow] { return newBalanceReadColumns() }) + }, +} + +type balanceReadRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Address string + Balance string + MetaNetworkName string +} + +func decodeBalanceReads(t *decode.Table, meta rowMeta) ([]balanceReadRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + address = p.str("address") + balance = p.str("balance_string") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]balanceReadRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, balanceReadRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + Balance: strOrEmpty(balance, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type balanceReadColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + Balance proto.ColUInt256 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newBalanceReadColumns() *balanceReadColumns { + return &balanceReadColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *balanceReadColumns) Append(r balanceReadRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + balance, err := uint256(r.Balance) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.Balance.Append(balance) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *balanceReadColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.Balance.Reset() + c.MetaNetworkName.Reset() +} + +func (c *balanceReadColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *balanceReadColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "balance", Data: &c.Balance}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_balance_reads_test.go b/pkg/processor/cryo/ds_balance_reads_test.go new file mode 100644 index 0000000..1a4d766 --- /dev/null +++ b/pkg/processor/cryo/ds_balance_reads_test.go @@ -0,0 +1,73 @@ +package cryo + +import ( + "testing" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" +) + +func TestDecodeBalanceReads(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "balance_reads", decodeBalanceReads) + require.NoError(t, err) + require.Len(t, rows, 641) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", first.Address) + require.Equal(t, "38432751121515279399317", first.Balance) + require.Equal(t, "mainnet", first.MetaNetworkName) + + second := rows[1] + require.Equal(t, uint32(2), second.InternalIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", second.Address) + require.Equal(t, "9456062291357014287", second.Balance) + + last := rows[640] + require.Equal(t, uint64(136), last.TransactionIndex) + require.Equal(t, "0x1894e4033d2ecb3f3b3bfa8b2c82c213593965754b40042a4ec392b2733ba76c", last.TransactionHash) + require.Equal(t, uint32(2), last.InternalIndex) + require.Equal(t, "0x5995510b29924a0c68e5e21cb95ac426519c43bf", last.Address) + require.Equal(t, "162525574300734667111", last.Balance) + + // 137 transactions read balances, and every row is attributed to one of + // them, so no row falls back to the "0x" key. + seen := make(map[string]uint32, len(rows)) + + for _, r := range rows { + require.NotEqual(t, emptyHex, r.TransactionHash) + seen[r.TransactionHash]++ + require.Equal(t, seen[r.TransactionHash], r.InternalIndex) + } + + require.Len(t, seen, 137) +} + +func TestBalanceReadsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "balance_reads", decodeBalanceReads) + require.NoError(t, err) + + cols := newBalanceReadColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + // A balance beyond uint64 must survive as 256-bit limbs rather than losing + // precision through a float. + require.Equal(t, proto.UInt256{ + Low: proto.UInt128{Low: 14951621711058254183, High: 8}, + }, cols.Balance[640]) + require.Equal(t, proto.UInt256FromUInt64(9456062291357014287), cols.Balance[1]) + + requireInputMatchesDDL(t, "canonical_execution_balance_reads", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_blocks.go b/pkg/processor/cryo/ds_blocks.go new file mode 100644 index 0000000..062b1ab --- /dev/null +++ b/pkg/processor/cryo/ds_blocks.go @@ -0,0 +1,187 @@ +package cryo + +import ( + "encoding/hex" + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var blocksDataset = &Dataset{ + Name: "blocks", + Table: "canonical_execution_block", + InternalIndex: false, + MinBlock: 0, + // cryo omits gas_limit from blocks unless asked, and ClickHouse would fill + // the gap with zero rather than failing, which is how staging came to hold + // gas_limit = 0 for every row. + RequiredColumns: []string{"gas_limit"}, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeBlocks, func() columnar[blockRow] { return newBlockColumns() }) + }, +} + +type blockRow struct { + UpdatedDateTime time.Time + BlockDateTime time.Time + BlockNumber uint64 + BlockHash string + Author proto.Nullable[string] + GasUsed proto.Nullable[uint64] + GasLimit uint64 + ExtraData proto.Nullable[string] + ExtraDataString proto.Nullable[string] + BaseFeePerGas proto.Nullable[uint64] + MetaNetworkName string +} + +func decodeBlocks(t *decode.Table, meta rowMeta) ([]blockRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + blockHash = p.str("block_hash") + author = p.str("author") + gasUsed = p.int("gas_used") + gasLimit = p.int("gas_limit") + extraData = p.str("extra_data") + timestamp = p.int("timestamp") + baseFeePerGas = p.int("base_fee_per_gas") + ) + + if p.err != nil { + return nil, p.err + } + + rows := make([]blockRow, 0, t.Rows()) + + for i := range t.Rows() { + extraHex, extraString, decodeErr := extraDataPair(extraData, i) + if decodeErr != nil { + return nil, fmt.Errorf("row %d: %w", i, decodeErr) + } + + rows = append(rows, blockRow{ + UpdatedDateTime: meta.updated, + //nolint:gosec // cryo emits the block timestamp as an unsigned second count + BlockDateTime: time.Unix(int64(uintOrZero(timestamp, i)), 0).UTC(), + BlockNumber: uintOrZero(blockNumber, i), + BlockHash: hexOrEmpty(blockHash, i), + Author: nullableHex(author, i), + GasUsed: nullableUint(gasUsed, i), + GasLimit: uintOrZero(gasLimit, i), + ExtraData: extraHex, + ExtraDataString: extraString, + BaseFeePerGas: nullableUint(baseFeePerGas, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +// extraDataPair derives both extra_data columns from cryo's single hex field: +// the hex form is stored verbatim, and the string form is the raw bytes it +// encodes, which are arbitrary and need not be valid UTF-8. +func extraDataPair(c *decode.Column, i int) (hexForm, stringForm proto.Nullable[string], err error) { + null := proto.Null[string]() + + if c.IsNull(i) { + return null, null, nil + } + + v := c.Str(i) + if v == "" || v == emptyHex { + return null, null, nil + } + + raw, err := hex.DecodeString(v[len(emptyHex):]) + if err != nil { + return null, null, fmt.Errorf("extra_data %q is not hex: %w", v, err) + } + + return proto.NewNullable(v), proto.NewNullable(string(raw)), nil +} + +type blockColumns struct { + UpdatedDateTime proto.ColDateTime + BlockDateTime *proto.ColDateTime64 + BlockNumber proto.ColUInt64 + BlockHash proto.ColFixedStr + Author *proto.ColNullable[string] + GasUsed *proto.ColNullable[uint64] + GasLimit proto.ColUInt64 + ExtraData *proto.ColNullable[string] + ExtraDataString *proto.ColNullable[string] + BaseFeePerGas *proto.ColNullable[uint64] + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newBlockColumns() *blockColumns { + return &blockColumns{ + BlockDateTime: new(proto.ColDateTime64).WithPrecision(proto.PrecisionMilli), + BlockHash: proto.ColFixedStr{Size: hashSize}, + Author: new(proto.ColStr).Nullable(), + GasUsed: new(proto.ColUInt64).Nullable(), + ExtraData: new(proto.ColStr).Nullable(), + ExtraDataString: new(proto.ColStr).Nullable(), + BaseFeePerGas: new(proto.ColUInt64).Nullable(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *blockColumns) Append(r blockRow) error { + hash, err := fixedHash(r.BlockHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockDateTime.Append(r.BlockDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.BlockHash.Append(hash) + c.Author.Append(r.Author) + c.GasUsed.Append(r.GasUsed) + c.GasLimit.Append(r.GasLimit) + c.ExtraData.Append(r.ExtraData) + c.ExtraDataString.Append(r.ExtraDataString) + c.BaseFeePerGas.Append(r.BaseFeePerGas) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *blockColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockDateTime.Reset() + c.BlockNumber.Reset() + c.BlockHash.Reset() + c.Author.Reset() + c.GasUsed.Reset() + c.GasLimit.Reset() + c.ExtraData.Reset() + c.ExtraDataString.Reset() + c.BaseFeePerGas.Reset() + c.MetaNetworkName.Reset() +} + +func (c *blockColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *blockColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_date_time", Data: c.BlockDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "block_hash", Data: &c.BlockHash}, + {Name: "author", Data: c.Author}, + {Name: "gas_used", Data: c.GasUsed}, + {Name: "gas_limit", Data: &c.GasLimit}, + {Name: "extra_data", Data: c.ExtraData}, + {Name: "extra_data_string", Data: c.ExtraDataString}, + {Name: "base_fee_per_gas", Data: c.BaseFeePerGas}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_blocks_test.go b/pkg/processor/cryo/ds_blocks_test.go new file mode 100644 index 0000000..b5fbd38 --- /dev/null +++ b/pkg/processor/cryo/ds_blocks_test.go @@ -0,0 +1,46 @@ +package cryo + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDecodeBlocks(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "blocks", decodeBlocks) + require.NoError(t, err) + require.Len(t, rows, 1) + + r := rows[0] + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, "0xe368c631c74a82c3043e6d44c4bef6e6139a6501b39c7700c2552554d10e6c3b", r.BlockHash) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", r.Author.Value) + require.Equal(t, uint64(11210841), r.GasUsed.Value) + require.Equal(t, uint64(45000000), r.GasLimit) + require.Equal(t, uint64(239712557), r.BaseFeePerGas.Value) + require.Equal(t, "mainnet", r.MetaNetworkName) + + // extra_data is stored twice: verbatim hex, and the raw bytes it encodes. + require.Equal(t, "0xe29ca82051756173617220287175617361722e77696e2920e29ca8", r.ExtraData.Value) + require.Equal(t, "✨ Quasar (quasar.win) ✨", r.ExtraDataString.Value) + + require.Equal(t, time.Unix(1753492931, 0).UTC(), r.BlockDateTime) +} + +func TestBlockColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "blocks", decodeBlocks) + require.NoError(t, err) + + cols := newBlockColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_block", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_contracts.go b/pkg/processor/cryo/ds_contracts.go new file mode 100644 index 0000000..a915891 --- /dev/null +++ b/pkg/processor/cryo/ds_contracts.go @@ -0,0 +1,180 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var contractsDataset = &Dataset{ + Name: "contracts", + Table: "canonical_execution_contracts", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeContracts, func() columnar[contractRow] { return newContractColumns() }) + }, +} + +type contractRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionHash string + InternalIndex uint32 + CreateIndex uint32 + ContractAddress string + Deployer string + Factory string + InitCode string + Code proto.Nullable[string] + InitCodeHash string + NInitCodeBytes uint32 + NCodeBytes uint32 + CodeHash string + MetaNetworkName string +} + +func decodeContracts(t *decode.Table, meta rowMeta) ([]contractRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + createIndex = p.int("create_index") + transactionHash = p.str("transaction_hash") + contractAddress = p.str("contract_address") + deployer = p.str("deployer") + factory = p.str("factory") + initCode = p.str("init_code") + code = p.str("code") + initCodeHash = p.str("init_code_hash") + nInitCodeBytes = p.int("n_init_code_bytes") + nCodeBytes = p.int("n_code_bytes") + codeHash = p.str("code_hash") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]contractRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, contractRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + //nolint:gosec // cryo emits create_index as a parquet uint32 + CreateIndex: uint32(uintOrZero(createIndex, i)), + ContractAddress: hexOrEmpty(contractAddress, i), + Deployer: hexOrEmpty(deployer, i), + Factory: hexOrEmpty(factory, i), + InitCode: hexOrEmpty(initCode, i), + Code: nullableHex(code, i), + InitCodeHash: hexOrEmpty(initCodeHash, i), + //nolint:gosec // cryo emits n_init_code_bytes as a parquet uint32 + NInitCodeBytes: uint32(uintOrZero(nInitCodeBytes, i)), + //nolint:gosec // cryo emits n_code_bytes as a parquet uint32 + NCodeBytes: uint32(uintOrZero(nCodeBytes, i)), + CodeHash: hexOrEmpty(codeHash, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type contractColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + CreateIndex proto.ColUInt32 + ContractAddress proto.ColStr + Deployer proto.ColStr + Factory proto.ColStr + InitCode proto.ColStr + Code *proto.ColNullable[string] + InitCodeHash proto.ColStr + NInitCodeBytes proto.ColUInt32 + NCodeBytes proto.ColUInt32 + CodeHash proto.ColStr + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newContractColumns() *contractColumns { + return &contractColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + Code: new(proto.ColStr).Nullable(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *contractColumns) Append(r contractRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.CreateIndex.Append(r.CreateIndex) + c.ContractAddress.Append(r.ContractAddress) + c.Deployer.Append(r.Deployer) + c.Factory.Append(r.Factory) + c.InitCode.Append(r.InitCode) + c.Code.Append(r.Code) + c.InitCodeHash.Append(r.InitCodeHash) + c.NInitCodeBytes.Append(r.NInitCodeBytes) + c.NCodeBytes.Append(r.NCodeBytes) + c.CodeHash.Append(r.CodeHash) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *contractColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.CreateIndex.Reset() + c.ContractAddress.Reset() + c.Deployer.Reset() + c.Factory.Reset() + c.InitCode.Reset() + c.Code.Reset() + c.InitCodeHash.Reset() + c.NInitCodeBytes.Reset() + c.NCodeBytes.Reset() + c.CodeHash.Reset() + c.MetaNetworkName.Reset() +} + +func (c *contractColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *contractColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "create_index", Data: &c.CreateIndex}, + {Name: "contract_address", Data: &c.ContractAddress}, + {Name: "deployer", Data: &c.Deployer}, + {Name: "factory", Data: &c.Factory}, + {Name: "init_code", Data: &c.InitCode}, + {Name: "code", Data: c.Code}, + {Name: "init_code_hash", Data: &c.InitCodeHash}, + {Name: "n_init_code_bytes", Data: &c.NInitCodeBytes}, + {Name: "n_code_bytes", Data: &c.NCodeBytes}, + {Name: "code_hash", Data: &c.CodeHash}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_contracts_test.go b/pkg/processor/cryo/ds_contracts_test.go new file mode 100644 index 0000000..0c6691f --- /dev/null +++ b/pkg/processor/cryo/ds_contracts_test.go @@ -0,0 +1,89 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const contractsTable = "canonical_execution_contracts" + +func TestContractsDataset(t *testing.T) { + t.Parallel() + + require.Equal(t, "contracts", contractsDataset.Name) + require.Equal(t, contractsTable, contractsDataset.Table) + require.True(t, contractsDataset.InternalIndex) + require.Zero(t, contractsDataset.MinBlock) +} + +func TestDecodeContracts(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "contracts", decodeContracts) + require.NoError(t, err) + require.Len(t, rows, 4) + + r := rows[0] + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + require.Equal(t, uint64(23000026), r.BlockNumber) + require.Equal(t, "0xd8a375f59e03b9ea056afeb3257cd7e805322d34015320f4a4104b96332cbb28", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, uint32(0), r.CreateIndex) + require.Equal(t, "0x70b3f08431e7d7ba1ca202017d0b19caad8d1fe9", r.ContractAddress) + require.Equal(t, "0x4e565f63257d90f988e5ec9d065bab00f94d2dfd", r.Deployer) + require.Equal(t, "0x9fa5c5733b53814692de4fb31fd592070de5f5f0", r.Factory) + require.Equal(t, uint32(785), r.NInitCodeBytes) + require.Equal(t, uint32(0), r.NCodeBytes) + require.Equal(t, "mainnet", r.MetaNetworkName) + + // This contract self-destructs in its constructor, so cryo reports its code + // as the literal "0x" and the target column stores NULL. + require.False(t, r.Code.Set) + + last := rows[3] + require.Equal(t, uint32(3), last.CreateIndex) + require.Equal(t, uint32(1), last.InternalIndex) + require.Equal(t, "0x9f1eb7bf7b105bd7baa144286c836ccb85d41417", last.ContractAddress) + require.Equal(t, "0x67e0409b97fa2a41c9105b2adee931c8649ea0dc", last.Deployer) + require.Equal(t, "0x2971adfa57b20e5a416ae5a708a8655a9c74f723", last.Factory) + require.Equal(t, + "0x3d602d80600a3d3981f3363d3d373d3d3d363d73fe02a32cbe0cb9ad9a945576a5bb53a3c123a3a35af43d82803e903d91602b57fd5bf3", + last.InitCode) + require.Equal(t, + "0x363d3d373d3d3d363d73fe02a32cbe0cb9ad9a945576a5bb53a3c123a3a35af43d82803e903d91602b57fd5bf3", + last.Code.Value) + require.Equal(t, "0xf51217697e54d3e7c4d961223d0ec7a0f4962b08eb43d9e27c22df8e30b6b3e1", last.InitCodeHash) + require.Equal(t, "0x269179116bc54c44db2053d1b6076ac2eacd45b921cf397525f967ff87095344", last.CodeHash) + require.Equal(t, uint32(55), last.NInitCodeBytes) + require.Equal(t, uint32(45), last.NCodeBytes) +} + +// TestDecodeContractsEmpty covers a block that deploys nothing: cryo still +// writes the parquet file, with zero rows. +func TestDecodeContractsEmpty(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, preMergeBlock, "contracts", decodeContracts) + require.NoError(t, err) + require.Empty(t, rows) + + cols := newContractColumns() + require.Equal(t, 0, cols.Rows()) + requireInputMatchesDDL(t, contractsTable, cols.Input()) +} + +func TestContractsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "contracts", decodeContracts) + require.NoError(t, err) + + cols := newContractColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, contractsTable, cols.Input()) +} diff --git a/pkg/processor/cryo/ds_erc20_transfers.go b/pkg/processor/cryo/ds_erc20_transfers.go new file mode 100644 index 0000000..0b8d610 --- /dev/null +++ b/pkg/processor/cryo/ds_erc20_transfers.go @@ -0,0 +1,154 @@ +package cryo + +import ( + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var erc20TransfersDataset = &Dataset{ + Name: "erc20_transfers", + Table: "canonical_execution_erc20_transfers", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeErc20Transfers, func() columnar[erc20TransferRow] { return newErc20TransferColumns() }) + }, +} + +type erc20TransferRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + LogIndex uint64 + Erc20 string + FromAddress string + ToAddress string + Value string + MetaNetworkName string +} + +func decodeErc20Transfers(t *decode.Table, meta rowMeta) ([]erc20TransferRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + logIndex = p.int("log_index") + erc20 = p.str("erc20") + fromAddress = p.str("from_address") + toAddress = p.str("to_address") + value = p.str("value_string") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]erc20TransferRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, erc20TransferRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + LogIndex: uintOrZero(logIndex, i), + Erc20: hexOrEmpty(erc20, i), + FromAddress: hexOrEmpty(fromAddress, i), + ToAddress: hexOrEmpty(toAddress, i), + Value: strOrEmpty(value, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type erc20TransferColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + LogIndex proto.ColUInt64 + Erc20 proto.ColStr + FromAddress proto.ColStr + ToAddress proto.ColStr + Value proto.ColUInt256 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newErc20TransferColumns() *erc20TransferColumns { + return &erc20TransferColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *erc20TransferColumns) Append(r erc20TransferRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + value, err := uint256(r.Value) + if err != nil { + return fmt.Errorf("value: %w", err) + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.LogIndex.Append(r.LogIndex) + c.Erc20.Append(r.Erc20) + c.FromAddress.Append(r.FromAddress) + c.ToAddress.Append(r.ToAddress) + c.Value.Append(value) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *erc20TransferColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.LogIndex.Reset() + c.Erc20.Reset() + c.FromAddress.Reset() + c.ToAddress.Reset() + c.Value.Reset() + c.MetaNetworkName.Reset() +} + +func (c *erc20TransferColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *erc20TransferColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "log_index", Data: &c.LogIndex}, + {Name: "erc20", Data: &c.Erc20}, + {Name: "from_address", Data: &c.FromAddress}, + {Name: "to_address", Data: &c.ToAddress}, + {Name: "value", Data: &c.Value}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_erc20_transfers_test.go b/pkg/processor/cryo/ds_erc20_transfers_test.go new file mode 100644 index 0000000..4517745 --- /dev/null +++ b/pkg/processor/cryo/ds_erc20_transfers_test.go @@ -0,0 +1,69 @@ +package cryo + +import ( + "testing" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" +) + +func TestDecodeErc20Transfers(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "erc20_transfers", decodeErc20Transfers) + require.NoError(t, err) + require.Len(t, rows, 152) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, uint64(0), first.LogIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", first.Erc20) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", first.FromAddress) + require.Equal(t, "0xf2abf514dfbaf3323a5cb95bb2e4ab180a5ba3e8", first.ToAddress) + require.Equal(t, "4996500000", first.Value) + require.Equal(t, "mainnet", first.MetaNetworkName) + + // Rows 1 and 2 share a transaction, so internal_index counts within it while + // log_index keeps counting across the block. + require.Equal(t, "0xcb51cc795fccee5df8468c88475be47dda46ba014c4d649a2ecd44d39d5876f2", rows[1].TransactionHash) + require.Equal(t, uint32(1), rows[1].InternalIndex) + require.Equal(t, uint64(4), rows[1].LogIndex) + require.Equal(t, "5264232533978631", rows[1].Value) + + require.Equal(t, rows[1].TransactionHash, rows[2].TransactionHash) + require.Equal(t, uint32(2), rows[2].InternalIndex) + require.Equal(t, uint64(5), rows[2].LogIndex) + require.Equal(t, "12370946454849782151684", rows[2].Value) + + last := rows[151] + require.Equal(t, uint64(131), last.TransactionIndex) + require.Equal(t, uint64(321), last.LogIndex) + require.Equal(t, "0x6c49082790288faf4c4b4a49dd7428817a8ce142cb7d474941229b1ca0b0b992", last.TransactionHash) + require.Equal(t, "42848622000", last.Value) +} + +func TestErc20TransferColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "erc20_transfers", decodeErc20Transfers) + require.NoError(t, err) + + cols := newErc20TransferColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + // 12370946454849782151684 does not fit a uint64, so the limbs prove the + // decimal string reached the U256 column without passing through one. + require.Equal(t, proto.UInt256{ + Low: proto.UInt128{Low: 11627925464382568964, High: 670}, + }, cols.Value[2]) + + requireInputMatchesDDL(t, "canonical_execution_erc20_transfers", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_erc721_transfers.go b/pkg/processor/cryo/ds_erc721_transfers.go new file mode 100644 index 0000000..253dfd3 --- /dev/null +++ b/pkg/processor/cryo/ds_erc721_transfers.go @@ -0,0 +1,155 @@ +package cryo + +import ( + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var erc721TransfersDataset = &Dataset{ + Name: "erc721_transfers", + Table: "canonical_execution_erc721_transfers", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeErc721Transfers, func() columnar[erc721TransferRow] { return newErc721TransferColumns() }) + }, +} + +type erc721TransferRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + LogIndex uint64 + Erc721 string + FromAddress string + ToAddress string + Token string + MetaNetworkName string +} + +func decodeErc721Transfers(t *decode.Table, meta rowMeta) ([]erc721TransferRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + logIndex = p.int("log_index") + // cryo names the token contract column erc20 in its erc721 output too. + erc721 = p.str("erc20") + fromAddress = p.str("from_address") + toAddress = p.str("to_address") + token = p.str("token_id_string") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]erc721TransferRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, erc721TransferRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + LogIndex: uintOrZero(logIndex, i), + Erc721: hexOrEmpty(erc721, i), + FromAddress: hexOrEmpty(fromAddress, i), + ToAddress: hexOrEmpty(toAddress, i), + Token: strOrEmpty(token, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type erc721TransferColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + LogIndex proto.ColUInt64 + Erc721 proto.ColStr + FromAddress proto.ColStr + ToAddress proto.ColStr + Token proto.ColUInt256 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newErc721TransferColumns() *erc721TransferColumns { + return &erc721TransferColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *erc721TransferColumns) Append(r erc721TransferRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + token, err := uint256(r.Token) + if err != nil { + return fmt.Errorf("token: %w", err) + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.LogIndex.Append(r.LogIndex) + c.Erc721.Append(r.Erc721) + c.FromAddress.Append(r.FromAddress) + c.ToAddress.Append(r.ToAddress) + c.Token.Append(token) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *erc721TransferColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.LogIndex.Reset() + c.Erc721.Reset() + c.FromAddress.Reset() + c.ToAddress.Reset() + c.Token.Reset() + c.MetaNetworkName.Reset() +} + +func (c *erc721TransferColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *erc721TransferColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "log_index", Data: &c.LogIndex}, + {Name: "erc721", Data: &c.Erc721}, + {Name: "from_address", Data: &c.FromAddress}, + {Name: "to_address", Data: &c.ToAddress}, + {Name: "token", Data: &c.Token}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_erc721_transfers_test.go b/pkg/processor/cryo/ds_erc721_transfers_test.go new file mode 100644 index 0000000..10704b0 --- /dev/null +++ b/pkg/processor/cryo/ds_erc721_transfers_test.go @@ -0,0 +1,102 @@ +package cryo + +import ( + "testing" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" +) + +// erc721TokenLarge is the token id of the first row of the b23000026 fixture, +// an ENS label hash. At 255 bits it fills all four U256 limbs, which is the +// case a float or uint64 intermediate would silently destroy. +const erc721TokenLarge = "54878141629548454963394928233127112722036498901245006402027164760877303839156" + +func TestDecodeErc721Transfers(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "erc721_transfers", decodeErc721Transfers) + require.NoError(t, err) + require.Len(t, rows, 42) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000026), first.BlockNumber) + require.Equal(t, uint64(102), first.TransactionIndex) + require.Equal(t, "0x8591f4c01a8849e191bc10101c7de815fa7f73a972d1c732823b9759977a5baf", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, uint64(463), first.LogIndex) + // cryo emits the token contract under the column name erc20 even here. + require.Equal(t, "0x57f1887a8bf19b14fc0df6fd9b2acc9af147ea85", first.Erc721) + require.Equal(t, "0xc0f7ee3fc687173d15f1d20182a04b22d3157fb4", first.FromAddress) + require.Equal(t, "0x0000000000000000000000000000000000000000", first.ToAddress) + require.Equal(t, erc721TokenLarge, first.Token) + require.Equal(t, "mainnet", first.MetaNetworkName) + + require.Equal(t, first.TransactionHash, rows[1].TransactionHash) + require.Equal(t, uint32(2), rows[1].InternalIndex) + require.Equal(t, uint64(464), rows[1].LogIndex) + require.Equal(t, erc721TokenLarge, rows[1].Token) + + // The block's second transaction restarts internal_index at 1. + require.Equal(t, "0x5940e7800479d3fd08490428e8f74160a54e09bd77f1c8824c97e0c689903e51", rows[40].TransactionHash) + require.Equal(t, uint32(1), rows[40].InternalIndex) + require.Equal(t, uint64(921), rows[40].LogIndex) + + last := rows[41] + require.Equal(t, rows[40].TransactionHash, last.TransactionHash) + require.Equal(t, uint32(2), last.InternalIndex) + require.Equal(t, uint64(201), last.TransactionIndex) + require.Equal(t, uint64(928), last.LogIndex) + require.Equal(t, "0x283af0b28c62c092c9727f1ee09c02ca627eb7f5", last.FromAddress) + require.Equal(t, "0xed976ca9036bc2d4e25ba8219fada1be503a09c7", last.ToAddress) + require.Equal(t, "56460438191091326310504608336249342392432960708470807119089512209041103960200", last.Token) +} + +// TestDecodeErc721TransfersEmptyBlock covers a block with no erc721 activity, +// which cryo still writes as a parquet file carrying only the schema. +func TestDecodeErc721TransfersEmptyBlock(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "erc721_transfers", decodeErc721Transfers) + require.NoError(t, err) + require.Empty(t, rows) +} + +func TestErc721TransferColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "erc721_transfers", decodeErc721Transfers) + require.NoError(t, err) + + cols := newErc721TransferColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + require.Equal(t, proto.UInt256{ + Low: proto.UInt128{ + Low: 8239877381706201524, + High: 11825367184831116616, + }, + High: proto.UInt128{ + Low: 4942523148105832171, + High: 8742592352801473700, + }, + }, cols.Token[0]) + + require.Equal(t, proto.UInt256{ + Low: proto.UInt128{ + Low: 8071277476105599112, + High: 1821699312930359461, + }, + High: proto.UInt128{ + Low: 14729306524578318847, + High: 8994666738122137780, + }, + }, cols.Token[41]) + + requireInputMatchesDDL(t, "canonical_execution_erc721_transfers", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_four_byte_counts.go b/pkg/processor/cryo/ds_four_byte_counts.go new file mode 100644 index 0000000..f68aa1e --- /dev/null +++ b/pkg/processor/cryo/ds_four_byte_counts.go @@ -0,0 +1,137 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var fourByteCountsDataset = &Dataset{ + Name: "four_byte_counts", + Table: "canonical_execution_four_byte_counts", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeFourByteCounts, func() columnar[fourByteCountRow] { return newFourByteCountColumns() }) + }, +} + +type fourByteCountRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Signature string + Size uint64 + Count uint64 + MetaNetworkName string +} + +func decodeFourByteCounts(t *decode.Table, meta rowMeta) ([]fourByteCountRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + signature = p.str("signature") + size = p.int("size") + count = p.int("count") + ) + + if p.err != nil { + return nil, p.err + } + + // cryo counts each selector once per calldata size, so a transaction can + // carry the same signature more than once; internal_index is what keeps + // those rows apart under the table's sort key. + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]fourByteCountRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, fourByteCountRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Signature: hexOrEmpty(signature, i), + Size: uintOrZero(size, i), + Count: uintOrZero(count, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type fourByteCountColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Signature proto.ColStr + Size proto.ColUInt64 + Count proto.ColUInt64 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newFourByteCountColumns() *fourByteCountColumns { + return &fourByteCountColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *fourByteCountColumns) Append(r fourByteCountRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Signature.Append(r.Signature) + c.Size.Append(r.Size) + c.Count.Append(r.Count) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *fourByteCountColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Signature.Reset() + c.Size.Reset() + c.Count.Reset() + c.MetaNetworkName.Reset() +} + +func (c *fourByteCountColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *fourByteCountColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "signature", Data: &c.Signature}, + {Name: "size", Data: &c.Size}, + {Name: "count", Data: &c.Count}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_four_byte_counts_test.go b/pkg/processor/cryo/ds_four_byte_counts_test.go new file mode 100644 index 0000000..fe5c76f --- /dev/null +++ b/pkg/processor/cryo/ds_four_byte_counts_test.go @@ -0,0 +1,142 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeFourByteCounts(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "four_byte_counts", decodeFourByteCounts) + require.NoError(t, err) + require.Len(t, rows, 278) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, "0xa9059cbb", first.Signature) + require.Equal(t, uint64(64), first.Size) + require.Equal(t, uint64(2), first.Count) + require.Equal(t, "mainnet", first.MetaNetworkName) + + second := rows[1] + require.Equal(t, uint64(3), second.TransactionIndex) + require.Equal(t, "0x3a49a2dcaf21efef0db0aeb21dba2b335a502078c8d5d8b9fb32a078e506d274", second.TransactionHash) + require.Equal(t, uint32(1), second.InternalIndex) + require.Equal(t, "0x5c43fcf6", second.Signature) + + last := rows[277] + require.Equal(t, uint64(131), last.TransactionIndex) + require.Equal(t, "0x6c49082790288faf4c4b4a49dd7428817a8ce142cb7d474941229b1ca0b0b992", last.TransactionHash) + require.Equal(t, uint32(1), last.InternalIndex) + require.Equal(t, "0xa9059cbb", last.Signature) + require.Equal(t, uint64(64), last.Size) + require.Equal(t, uint64(2), last.Count) +} + +// TestFourByteCountsInternalIndexIsUnique guards the reason this table carries +// internal_index at all: cryo counts a selector once per calldata size, so a +// transaction can repeat a signature, and without the extra sort key those rows +// collapse into one another. +func TestFourByteCountsInternalIndexIsUnique(t *testing.T) { + t.Parallel() + + for block, want := range map[string]int{"b23000000": 278, "b23000026": 1094} { + t.Run(block, func(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, block, "four_byte_counts", decodeFourByteCounts) + require.NoError(t, err) + require.Len(t, rows, want) + + type key struct { + hash string + index uint32 + } + + seen := make(map[key]struct{}, len(rows)) + perTx := make(map[string]uint32, len(rows)) + + for i, r := range rows { + k := key{hash: r.TransactionHash, index: r.InternalIndex} + + _, dup := seen[k] + + require.NotZero(t, r.InternalIndex, "row %d has a zero internal_index", i) + require.False(t, dup, "row %d repeats (%s, %d)", i, r.TransactionHash, r.InternalIndex) + + seen[k] = struct{}{} + perTx[r.TransactionHash]++ + + require.Equal(t, perTx[r.TransactionHash], r.InternalIndex, + "row %d is not the next index for %s", i, r.TransactionHash) + } + + require.Len(t, seen, len(rows)) + }) + } +} + +// TestFourByteCountsRepeatedSignature pins the transaction that motivates the +// column: the same selector appears twice under two calldata sizes. +func TestFourByteCountsRepeatedSignature(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "four_byte_counts", decodeFourByteCounts) + require.NoError(t, err) + + const hash = "0x50c8d7b157abe4c0515adab242a25bd39f815fcb51f1e03c21650e38afb887e0" + + var ( + tx []fourByteCountRow + repeats []fourByteCountRow + ) + + for _, r := range rows { + if r.TransactionHash != hash { + continue + } + + tx = append(tx, r) + + if r.Signature == "0x7bd243de" { + repeats = append(repeats, r) + } + } + + require.Len(t, tx, 12) + + for i, r := range tx { + require.Equal(t, uint32(i+1), r.InternalIndex) + require.Equal(t, uint64(122), r.TransactionIndex) + } + + require.Len(t, repeats, 2) + require.Equal(t, uint32(9), repeats[0].InternalIndex) + require.Equal(t, uint64(192), repeats[0].Size) + require.Equal(t, uint32(10), repeats[1].InternalIndex) + require.Equal(t, uint64(224), repeats[1].Size) +} + +func TestFourByteCountColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "four_byte_counts", decodeFourByteCounts) + require.NoError(t, err) + + cols := newFourByteCountColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + require.Equal(t, "0xa9059cbb", cols.Signature.Row(0)) + require.Equal(t, uint32(1), cols.InternalIndex[0]) + + requireInputMatchesDDL(t, "canonical_execution_four_byte_counts", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_logs.go b/pkg/processor/cryo/ds_logs.go new file mode 100644 index 0000000..71127a0 --- /dev/null +++ b/pkg/processor/cryo/ds_logs.go @@ -0,0 +1,168 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var logsDataset = &Dataset{ + Name: "logs", + Table: "canonical_execution_logs", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeLogs, func() columnar[logRow] { return newLogColumns() }) + }, +} + +type logRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + LogIndex uint32 + Address string + Topic0 proto.Nullable[string] + Topic1 proto.Nullable[string] + Topic2 proto.Nullable[string] + Topic3 proto.Nullable[string] + Data proto.Nullable[string] + MetaNetworkName string +} + +func decodeLogs(t *decode.Table, meta rowMeta) ([]logRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + logIndex = p.int("log_index") + address = p.str("address") + topic0 = p.str("topic0") + topic1 = p.str("topic1") + topic2 = p.str("topic2") + topic3 = p.str("topic3") + data = p.str("data") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]logRow, 0, t.Rows()) + + for i := range t.Rows() { + //nolint:gosec // log_index is uint32 in cryo's schema, widened by the decoder + rows = append(rows, logRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + LogIndex: uint32(uintOrZero(logIndex, i)), + Address: hexOrEmpty(address, i), + Topic0: nullableHex(topic0, i), + Topic1: nullableHex(topic1, i), + Topic2: nullableHex(topic2, i), + Topic3: nullableHex(topic3, i), + Data: nullableHex(data, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type logColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + LogIndex proto.ColUInt32 + Address proto.ColStr + Topic0 *proto.ColNullable[string] + Topic1 *proto.ColNullable[string] + Topic2 *proto.ColNullable[string] + Topic3 *proto.ColNullable[string] + Data *proto.ColNullable[string] + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newLogColumns() *logColumns { + return &logColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + Topic0: new(proto.ColStr).Nullable(), + Topic1: new(proto.ColStr).Nullable(), + Topic2: new(proto.ColStr).Nullable(), + Topic3: new(proto.ColStr).Nullable(), + Data: new(proto.ColStr).Nullable(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *logColumns) Append(r logRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.LogIndex.Append(r.LogIndex) + c.Address.Append(r.Address) + c.Topic0.Append(r.Topic0) + c.Topic1.Append(r.Topic1) + c.Topic2.Append(r.Topic2) + c.Topic3.Append(r.Topic3) + c.Data.Append(r.Data) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *logColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.LogIndex.Reset() + c.Address.Reset() + c.Topic0.Reset() + c.Topic1.Reset() + c.Topic2.Reset() + c.Topic3.Reset() + c.Data.Reset() + c.MetaNetworkName.Reset() +} + +func (c *logColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *logColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "log_index", Data: &c.LogIndex}, + {Name: "address", Data: &c.Address}, + {Name: "topic0", Data: c.Topic0}, + {Name: "topic1", Data: c.Topic1}, + {Name: "topic2", Data: c.Topic2}, + {Name: "topic3", Data: c.Topic3}, + {Name: "data", Data: c.Data}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_logs_test.go b/pkg/processor/cryo/ds_logs_test.go new file mode 100644 index 0000000..0082e73 --- /dev/null +++ b/pkg/processor/cryo/ds_logs_test.go @@ -0,0 +1,92 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeLogs(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "logs", decodeLogs) + require.NoError(t, err) + require.Len(t, rows, 322) + + r := rows[0] + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, uint32(0), r.LogIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", r.Address) + require.Equal(t, "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", r.Topic0.Value) + require.Equal(t, "0x00000000000000000000000028c6c06298d514db089934071355e5743bf21d60", r.Topic1.Value) + require.Equal(t, "0x000000000000000000000000f2abf514dfbaf3323a5cb95bb2e4ab180a5ba3e8", r.Topic2.Value) + require.False(t, r.Topic3.Set) + require.Equal(t, "0x0000000000000000000000000000000000000000000000000000000129d08a20", r.Data.Value) + require.Equal(t, "mainnet", r.MetaNetworkName) + + // internal_index restarts at 1 for each transaction and counts every log + // the transaction emitted, in the order cryo wrote them. + require.Equal(t, uint32(1), rows[3].InternalIndex) + require.Equal(t, uint32(5), rows[7].InternalIndex) + require.Equal(t, uint32(11), rows[26].InternalIndex) + require.Equal(t, uint32(47), rows[62].InternalIndex) + require.Equal(t, "0xce3d9c0c9dc7a3db4f613a9a8f56be5b7a858b60fd758d2b4109e9db2b96dc5d", rows[62].TransactionHash) + require.Equal(t, uint64(6), rows[62].TransactionIndex) + require.Equal(t, uint32(1), rows[321].InternalIndex) + + // A log with no data at all arrives as "0x" and is stored as NULL. + require.False(t, rows[26].Data.Set) + require.False(t, rows[62].Data.Set) +} + +func TestDecodeLogsAnonymous(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "logs", decodeLogs) + require.NoError(t, err) + require.Len(t, rows, 990) + + anonymous := 0 + + for _, r := range rows { + if !r.Topic0.Set { + anonymous++ + + require.False(t, r.Topic1.Set) + require.False(t, r.Topic2.Set) + require.False(t, r.Topic3.Set) + } + } + + // LOG0 emits no topics whatsoever, so topic0 is genuinely absent rather + // than zero. + require.Equal(t, 10, anonymous) + + r := rows[26] + require.False(t, r.Topic0.Set) + require.Equal(t, uint32(26), r.LogIndex) + require.Equal(t, uint64(1), r.TransactionIndex) + require.Equal(t, "0xffbf7de7b5c7740694ec15a8dfd2b6c0e42fde82fd70db8eb8122a4f10c68257", r.TransactionHash) + require.Equal(t, uint32(17), r.InternalIndex) + require.Equal(t, "0xe0e0e08a6a4b9dc7bd67bcb7aade5cf48157d444", r.Address) + require.True(t, r.Data.Set) +} + +func TestLogsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000026", "logs", decodeLogs) + require.NoError(t, err) + + cols := newLogColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_logs", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_native_transfers.go b/pkg/processor/cryo/ds_native_transfers.go new file mode 100644 index 0000000..de106ac --- /dev/null +++ b/pkg/processor/cryo/ds_native_transfers.go @@ -0,0 +1,147 @@ +package cryo + +import ( + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var nativeTransfersDataset = &Dataset{ + Name: "native_transfers", + Table: "canonical_execution_native_transfers", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeNativeTransfers, func() columnar[nativeTransferRow] { return newNativeTransferColumns() }) + }, +} + +type nativeTransferRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + TransferIndex uint64 + FromAddress string + ToAddress string + Value string + MetaNetworkName string +} + +func decodeNativeTransfers(t *decode.Table, meta rowMeta) ([]nativeTransferRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + transferIndex = p.int("transfer_index") + fromAddress = p.str("from_address") + toAddress = p.str("to_address") + value = p.str("value_string") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]nativeTransferRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, nativeTransferRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + TransferIndex: uintOrZero(transferIndex, i), + FromAddress: hexOrEmpty(fromAddress, i), + ToAddress: hexOrEmpty(toAddress, i), + Value: strOrEmpty(value, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type nativeTransferColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + TransferIndex proto.ColUInt64 + FromAddress proto.ColStr + ToAddress proto.ColStr + Value proto.ColUInt256 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newNativeTransferColumns() *nativeTransferColumns { + return &nativeTransferColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *nativeTransferColumns) Append(r nativeTransferRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + value, err := uint256(r.Value) + if err != nil { + return fmt.Errorf("value: %w", err) + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.TransferIndex.Append(r.TransferIndex) + c.FromAddress.Append(r.FromAddress) + c.ToAddress.Append(r.ToAddress) + c.Value.Append(value) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *nativeTransferColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.TransferIndex.Reset() + c.FromAddress.Reset() + c.ToAddress.Reset() + c.Value.Reset() + c.MetaNetworkName.Reset() +} + +func (c *nativeTransferColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *nativeTransferColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "transfer_index", Data: &c.TransferIndex}, + {Name: "from_address", Data: &c.FromAddress}, + {Name: "to_address", Data: &c.ToAddress}, + {Name: "value", Data: &c.Value}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_native_transfers_test.go b/pkg/processor/cryo/ds_native_transfers_test.go new file mode 100644 index 0000000..aa631e6 --- /dev/null +++ b/pkg/processor/cryo/ds_native_transfers_test.go @@ -0,0 +1,75 @@ +package cryo + +import ( + "testing" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" +) + +func TestDecodeNativeTransfers(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "native_transfers", decodeNativeTransfers) + require.NoError(t, err) + require.Len(t, rows, 797) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, uint64(0), first.TransferIndex) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", first.FromAddress) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", first.ToAddress) + require.Equal(t, "0", first.Value) + require.Equal(t, "mainnet", first.MetaNetworkName) + + require.Equal(t, first.TransactionHash, rows[1].TransactionHash) + require.Equal(t, uint32(2), rows[1].InternalIndex) + require.Equal(t, uint64(1), rows[1].TransferIndex) + + // transfer_index counts across the whole block, so a new transaction resets + // internal_index without resetting it. + require.Equal(t, "0x6747dea6d070127292e2a600d879ac67b986253088e82de01d6bfa92e7ffb90c", rows[2].TransactionHash) + require.Equal(t, uint32(1), rows[2].InternalIndex) + require.Equal(t, uint64(2), rows[2].TransferIndex) + require.Equal(t, "1085803440000000000", rows[2].Value) + + big := rows[470] + require.Equal(t, "0xc55c306596f2d46cf0658b7432834e40ddb0056d5d3083b2bae93de9753824ad", big.TransactionHash) + require.Equal(t, uint64(56), big.TransactionIndex) + require.Equal(t, uint64(470), big.TransferIndex) + require.Equal(t, uint32(1), big.InternalIndex) + require.Equal(t, "82520860000000000000", big.Value) + + last := rows[796] + require.Equal(t, "0x1894e4033d2ecb3f3b3bfa8b2c82c213593965754b40042a4ec392b2733ba76c", last.TransactionHash) + require.Equal(t, uint64(136), last.TransactionIndex) + require.Equal(t, uint64(796), last.TransferIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", last.FromAddress) + require.Equal(t, "23526089112080996", last.Value) +} + +func TestNativeTransferColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "native_transfers", decodeNativeTransfers) + require.NoError(t, err) + + cols := newNativeTransferColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + // 82.52 ETH in wei overflows a uint64, so the limbs prove the decimal string + // reached the U256 column without passing through one. + require.Equal(t, proto.UInt256{ + Low: proto.UInt128{Low: 8733883705161793536, High: 4}, + }, cols.Value[470]) + + requireInputMatchesDDL(t, "canonical_execution_native_transfers", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_nonce_diffs.go b/pkg/processor/cryo/ds_nonce_diffs.go new file mode 100644 index 0000000..7f77913 --- /dev/null +++ b/pkg/processor/cryo/ds_nonce_diffs.go @@ -0,0 +1,134 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var nonceDiffsDataset = &Dataset{ + Name: "nonce_diffs", + Table: "canonical_execution_nonce_diffs", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeNonceDiffs, func() columnar[nonceDiffRow] { return newNonceDiffColumns() }) + }, +} + +type nonceDiffRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Address string + FromValue uint64 + ToValue uint64 + MetaNetworkName string +} + +func decodeNonceDiffs(t *decode.Table, meta rowMeta) ([]nonceDiffRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + address = p.str("address") + fromValue = p.int("from_value") + toValue = p.int("to_value") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]nonceDiffRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, nonceDiffRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + FromValue: uintOrZero(fromValue, i), + ToValue: uintOrZero(toValue, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type nonceDiffColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + FromValue proto.ColUInt64 + ToValue proto.ColUInt64 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newNonceDiffColumns() *nonceDiffColumns { + return &nonceDiffColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *nonceDiffColumns) Append(r nonceDiffRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.FromValue.Append(r.FromValue) + c.ToValue.Append(r.ToValue) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *nonceDiffColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.FromValue.Reset() + c.ToValue.Reset() + c.MetaNetworkName.Reset() +} + +func (c *nonceDiffColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *nonceDiffColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "from_value", Data: &c.FromValue}, + {Name: "to_value", Data: &c.ToValue}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_nonce_diffs_test.go b/pkg/processor/cryo/ds_nonce_diffs_test.go new file mode 100644 index 0000000..6c95c4b --- /dev/null +++ b/pkg/processor/cryo/ds_nonce_diffs_test.go @@ -0,0 +1,62 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeNonceDiffs(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "nonce_diffs", decodeNonceDiffs) + require.NoError(t, err) + require.Len(t, rows, 139) + + r := rows[0] + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", r.Address) + require.Equal(t, uint64(12963272), r.FromValue) + require.Equal(t, uint64(12963273), r.ToValue) + require.Equal(t, "mainnet", r.MetaNetworkName) + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + + require.Equal(t, uint64(1), rows[1].TransactionIndex) + require.Equal(t, uint32(1), rows[1].InternalIndex) + require.Equal(t, uint64(12963273), rows[1].FromValue) + + // A transaction that bumps a second account's nonce produces a second row + // under the same hash. + second := rows[36] + require.Equal(t, "0xe99d668cb5ea5a2c06bf5d7ff12a75b8f7331d21e8ffeaf00c43156b57b9a7e0", second.TransactionHash) + require.Equal(t, uint32(2), second.InternalIndex) + require.Equal(t, "0x9fa5c5733b53814692de4fb31fd592070de5f5f0", second.Address) + require.Equal(t, uint64(75977), second.FromValue) + require.Equal(t, uint64(75978), second.ToValue) + + last := rows[138] + require.Equal(t, uint64(136), last.TransactionIndex) + require.Equal(t, "0x1894e4033d2ecb3f3b3bfa8b2c82c213593965754b40042a4ec392b2733ba76c", last.TransactionHash) + require.Equal(t, uint32(1), last.InternalIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", last.Address) + require.Equal(t, uint64(75878), last.FromValue) + require.Equal(t, uint64(75879), last.ToValue) +} + +func TestNonceDiffsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "nonce_diffs", decodeNonceDiffs) + require.NoError(t, err) + + cols := newNonceDiffColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_nonce_diffs", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_nonce_reads.go b/pkg/processor/cryo/ds_nonce_reads.go new file mode 100644 index 0000000..1d04636 --- /dev/null +++ b/pkg/processor/cryo/ds_nonce_reads.go @@ -0,0 +1,127 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var nonceReadsDataset = &Dataset{ + Name: "nonce_reads", + Table: "canonical_execution_nonce_reads", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeNonceReads, func() columnar[nonceReadRow] { return newNonceReadColumns() }) + }, +} + +type nonceReadRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Address string + Nonce uint64 + MetaNetworkName string +} + +func decodeNonceReads(t *decode.Table, meta rowMeta) ([]nonceReadRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + address = p.str("address") + nonce = p.int("nonce") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]nonceReadRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, nonceReadRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + Nonce: uintOrZero(nonce, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type nonceReadColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + Nonce proto.ColUInt64 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newNonceReadColumns() *nonceReadColumns { + return &nonceReadColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *nonceReadColumns) Append(r nonceReadRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.Nonce.Append(r.Nonce) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *nonceReadColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.Nonce.Reset() + c.MetaNetworkName.Reset() +} + +func (c *nonceReadColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *nonceReadColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "nonce", Data: &c.Nonce}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_nonce_reads_test.go b/pkg/processor/cryo/ds_nonce_reads_test.go new file mode 100644 index 0000000..fd91efa --- /dev/null +++ b/pkg/processor/cryo/ds_nonce_reads_test.go @@ -0,0 +1,64 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeNonceReads(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "nonce_reads", decodeNonceReads) + require.NoError(t, err) + require.Len(t, rows, 592) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", first.Address) + require.Equal(t, uint64(12963272), first.Nonce) + require.Equal(t, "mainnet", first.MetaNetworkName) + + second := rows[1] + require.Equal(t, uint32(2), second.InternalIndex) + require.Equal(t, "0x396343362be2a4da1ce0c1c210945346fb82aa49", second.Address) + require.Equal(t, uint64(75876), second.Nonce) + + last := rows[591] + require.Equal(t, uint64(136), last.TransactionIndex) + require.Equal(t, "0x1894e4033d2ecb3f3b3bfa8b2c82c213593965754b40042a4ec392b2733ba76c", last.TransactionHash) + require.Equal(t, uint32(2), last.InternalIndex) + require.Equal(t, "0x5995510b29924a0c68e5e21cb95ac426519c43bf", last.Address) + require.Equal(t, uint64(1), last.Nonce) + + seen := make(map[string]uint32, len(rows)) + + for _, r := range rows { + require.NotEqual(t, emptyHex, r.TransactionHash) + seen[r.TransactionHash]++ + require.Equal(t, seen[r.TransactionHash], r.InternalIndex) + } + + require.Len(t, seen, 137) +} + +func TestNonceReadsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "nonce_reads", decodeNonceReads) + require.NoError(t, err) + + cols := newNonceReadColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + require.Equal(t, uint64(12963272), cols.Nonce[0]) + + requireInputMatchesDDL(t, "canonical_execution_nonce_reads", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_storage_diffs.go b/pkg/processor/cryo/ds_storage_diffs.go new file mode 100644 index 0000000..9c933a8 --- /dev/null +++ b/pkg/processor/cryo/ds_storage_diffs.go @@ -0,0 +1,143 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var storageDiffsDataset = &Dataset{ + Name: "storage_diffs", + Table: "canonical_execution_storage_diffs", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeStorageDiffs, func() columnar[storageDiffRow] { return newStorageDiffColumns() }) + }, +} + +type storageDiffRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + Address string + // Slot, FromValue and ToValue are 32-byte words the target stores verbatim + // as hex rather than as numbers. + Slot string + FromValue string + ToValue string + MetaNetworkName string +} + +func decodeStorageDiffs(t *decode.Table, meta rowMeta) ([]storageDiffRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + address = p.str("address") + slot = p.str("slot") + fromValue = p.str("from_value") + toValue = p.str("to_value") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]storageDiffRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, storageDiffRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + Address: hexOrEmpty(address, i), + Slot: hexOrEmpty(slot, i), + FromValue: hexOrEmpty(fromValue, i), + ToValue: hexOrEmpty(toValue, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type storageDiffColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + Address proto.ColStr + Slot proto.ColStr + FromValue proto.ColStr + ToValue proto.ColStr + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newStorageDiffColumns() *storageDiffColumns { + return &storageDiffColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *storageDiffColumns) Append(r storageDiffRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.Address.Append(r.Address) + c.Slot.Append(r.Slot) + c.FromValue.Append(r.FromValue) + c.ToValue.Append(r.ToValue) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *storageDiffColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.Address.Reset() + c.Slot.Reset() + c.FromValue.Reset() + c.ToValue.Reset() + c.MetaNetworkName.Reset() +} + +func (c *storageDiffColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *storageDiffColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "address", Data: &c.Address}, + {Name: "slot", Data: &c.Slot}, + {Name: "from_value", Data: &c.FromValue}, + {Name: "to_value", Data: &c.ToValue}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_storage_diffs_test.go b/pkg/processor/cryo/ds_storage_diffs_test.go new file mode 100644 index 0000000..c7e236f --- /dev/null +++ b/pkg/processor/cryo/ds_storage_diffs_test.go @@ -0,0 +1,64 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeStorageDiffs(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "storage_diffs", decodeStorageDiffs) + require.NoError(t, err) + require.Len(t, rows, 359) + + r := rows[0] + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", r.Address) + require.Equal(t, "mainnet", r.MetaNetworkName) + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + + // The slot and both values are 32-byte words stored verbatim, not numbers. + require.Equal(t, "0x07081a045c3dbf2e63b62a407ef205e7586e2629d2e2b95ff093308ca0ff3727", r.Slot) + require.Equal(t, "0x00000000000000000000000000000000000000000000000000035412376833d6", r.FromValue) + require.Equal(t, "0x000000000000000000000000000000000000000000000000000354110d97a9b6", r.ToValue) + + // A leading-zero word keeps every one of its 64 hex digits. + require.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000000", rows[1].FromValue) + require.Equal(t, "0x0000000000000000000000000000000000000000000000000000000129d08a20", rows[1].ToValue) + require.Equal(t, uint32(2), rows[1].InternalIndex) + + require.Equal(t, uint64(5), rows[2].TransactionIndex) + require.Equal(t, "0xcb51cc795fccee5df8468c88475be47dda46ba014c4d649a2ecd44d39d5876f2", rows[2].TransactionHash) + require.Equal(t, uint32(1), rows[2].InternalIndex) + require.Equal(t, "0x000000000004444c5dc75cb358380d2e3de08a90", rows[2].Address) + require.Equal(t, uint32(3), rows[4].InternalIndex) + + last := rows[358] + require.Equal(t, uint64(131), last.TransactionIndex) + require.Equal(t, "0x6c49082790288faf4c4b4a49dd7428817a8ce142cb7d474941229b1ca0b0b992", last.TransactionHash) + require.Equal(t, uint32(2), last.InternalIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", last.Address) + require.Equal(t, "0xd7c2ca1b2001ebdfb058bef6a6231a3c83cff193a748eecbefc3e36fdd8bc0d6", last.Slot) + require.Equal(t, "0x00000000000000000000000000000000000000000000000000000107e59fa4b1", last.FromValue) + require.Equal(t, "0x000000000000000000000000000000000000000000000000000000fdeba58f01", last.ToValue) +} + +func TestStorageDiffsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "storage_diffs", decodeStorageDiffs) + require.NoError(t, err) + + cols := newStorageDiffColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_storage_diffs", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_storage_reads.go b/pkg/processor/cryo/ds_storage_reads.go new file mode 100644 index 0000000..83122a0 --- /dev/null +++ b/pkg/processor/cryo/ds_storage_reads.go @@ -0,0 +1,136 @@ +package cryo + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var storageReadsDataset = &Dataset{ + Name: "storage_reads", + Table: "canonical_execution_storage_reads", + InternalIndex: true, + MinBlock: 1, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeStorageReads, func() columnar[storageReadRow] { return newStorageReadColumns() }) + }, +} + +type storageReadRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + ContractAddress string + Slot string + Value string + MetaNetworkName string +} + +func decodeStorageReads(t *decode.Table, meta rowMeta) ([]storageReadRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + contractAddress = p.str("contract_address") + slot = p.str("slot") + value = p.str("value") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]storageReadRow, 0, t.Rows()) + + for i := range t.Rows() { + rows = append(rows, storageReadRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + ContractAddress: hexOrEmpty(contractAddress, i), + // The slot key and the word it holds are stored as the hex cryo + // emits, not as numbers: the target columns are String. + Slot: hexOrEmpty(slot, i), + Value: hexOrEmpty(value, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type storageReadColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + ContractAddress proto.ColStr + Slot proto.ColStr + Value proto.ColStr + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newStorageReadColumns() *storageReadColumns { + return &storageReadColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *storageReadColumns) Append(r storageReadRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.ContractAddress.Append(r.ContractAddress) + c.Slot.Append(r.Slot) + c.Value.Append(r.Value) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *storageReadColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.ContractAddress.Reset() + c.Slot.Reset() + c.Value.Reset() + c.MetaNetworkName.Reset() +} + +func (c *storageReadColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *storageReadColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "contract_address", Data: &c.ContractAddress}, + {Name: "slot", Data: &c.Slot}, + {Name: "value", Data: &c.Value}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_storage_reads_test.go b/pkg/processor/cryo/ds_storage_reads_test.go new file mode 100644 index 0000000..d2f4d56 --- /dev/null +++ b/pkg/processor/cryo/ds_storage_reads_test.go @@ -0,0 +1,75 @@ +package cryo + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeStorageReads(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "storage_reads", decodeStorageReads) + require.NoError(t, err) + require.Len(t, rows, 900) + + first := rows[0] + require.Equal(t, fixtureMeta.updated, first.UpdatedDateTime) + require.Equal(t, uint64(23000000), first.BlockNumber) + require.Equal(t, uint64(0), first.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", first.TransactionHash) + require.Equal(t, uint32(1), first.InternalIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", first.ContractAddress) + require.Equal(t, "mainnet", first.MetaNetworkName) + + // The slot and the word it holds stay verbatim hex; leading zero nibbles + // would be lost if either were read as a number. + require.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000001", first.Slot) + require.Equal(t, "0x0000000000000000000000004914f61d25e5c567143774b76edbf4d5109a8566", first.Value) + + second := rows[1] + require.Equal(t, uint32(2), second.InternalIndex) + require.Equal(t, "0x07081a045c3dbf2e63b62a407ef205e7586e2629d2e2b95ff093308ca0ff3727", second.Slot) + require.Equal(t, "0x00000000000000000000000000000000000000000000000000035412376833d6", second.Value) + + last := rows[899] + require.Equal(t, uint64(131), last.TransactionIndex) + require.Equal(t, "0x6c49082790288faf4c4b4a49dd7428817a8ce142cb7d474941229b1ca0b0b992", last.TransactionHash) + require.Equal(t, uint32(5), last.InternalIndex) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", last.ContractAddress) + require.Equal(t, "0xd7c2ca1b2001ebdfb058bef6a6231a3c83cff193a748eecbefc3e36fdd8bc0d6", last.Slot) + require.Equal(t, "0x00000000000000000000000000000000000000000000000000000107e59fa4b1", last.Value) + + seen := make(map[string]uint32, len(rows)) + + for _, r := range rows { + require.Len(t, r.Slot, 66) + require.Len(t, r.Value, 66) + require.True(t, strings.HasPrefix(r.Slot, emptyHex)) + require.True(t, strings.HasPrefix(r.Value, emptyHex)) + require.Len(t, r.ContractAddress, 42) + + seen[r.TransactionHash]++ + require.Equal(t, seen[r.TransactionHash], r.InternalIndex) + } + + require.Len(t, seen, 76) +} + +func TestStorageReadsColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "storage_reads", decodeStorageReads) + require.NoError(t, err) + + cols := newStorageReadColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + require.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000001", cols.Slot.Row(0)) + + requireInputMatchesDDL(t, "canonical_execution_storage_reads", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_traces.go b/pkg/processor/cryo/ds_traces.go new file mode 100644 index 0000000..1860df5 --- /dev/null +++ b/pkg/processor/cryo/ds_traces.go @@ -0,0 +1,246 @@ +package cryo + +import ( + "fmt" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var tracesDataset = &Dataset{ + Name: "traces", + Table: "canonical_execution_traces", + InternalIndex: true, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeTraces, func() columnar[traceRow] { return newTraceColumns() }) + }, +} + +type traceRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + InternalIndex uint32 + ActionFrom string + ActionTo proto.Nullable[string] + ActionValue proto.UInt256 + ActionGas proto.Nullable[uint64] + ActionInput proto.Nullable[string] + ActionCallType string + ActionInit proto.Nullable[string] + ActionRewardType string + ActionType string + ResultGasUsed proto.Nullable[uint64] + ResultOutput proto.Nullable[string] + ResultCode proto.Nullable[string] + ResultAddress proto.Nullable[string] + TraceAddress proto.Nullable[string] + Subtraces uint32 + Error proto.Nullable[string] + MetaNetworkName string +} + +func decodeTraces(t *decode.Table, meta rowMeta) ([]traceRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + actionFrom = p.str("action_from") + actionTo = p.str("action_to") + actionValue = p.str("action_value") + actionGas = p.int("action_gas") + actionInput = p.str("action_input") + actionCallType = p.str("action_call_type") + actionInit = p.str("action_init") + actionRewardType = p.str("action_reward_type") + actionType = p.str("action_type") + resultGasUsed = p.int("result_gas_used") + resultOutput = p.str("result_output") + resultCode = p.str("result_code") + resultAddress = p.str("result_address") + traceAddress = p.str("trace_address") + subtraces = p.int("subtraces") + traceError = p.str("error") + ) + + if p.err != nil { + return nil, p.err + } + + idx := internalIndex(transactionHash, t.Rows()) + + rows := make([]traceRow, 0, t.Rows()) + + for i := range t.Rows() { + // action_value is the one U256 cryo names without a _string suffix. + value, err := uint256(strOrEmpty(actionValue, i)) + if err != nil { + return nil, fmt.Errorf("row %d: action_value: %w", i, err) + } + + rows = append(rows, traceRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + InternalIndex: idx[i], + ActionFrom: hexOrEmpty(actionFrom, i), + ActionTo: nullableHex(actionTo, i), + ActionValue: value, + ActionGas: nullableUint(actionGas, i), + ActionInput: nullableHex(actionInput, i), + ActionCallType: strOrEmpty(actionCallType, i), + ActionInit: nullableHex(actionInit, i), + ActionRewardType: strOrEmpty(actionRewardType, i), + ActionType: strOrEmpty(actionType, i), + ResultGasUsed: nullableUint(resultGasUsed, i), + ResultOutput: nullableHex(resultOutput, i), + ResultCode: nullableHex(resultCode, i), + ResultAddress: nullableHex(resultAddress, i), + // trace_address is an underscore-joined path of child indices, not hex. + TraceAddress: nullableStr(traceAddress, i), + //nolint:gosec // cryo emits subtraces as a parquet uint32 + Subtraces: uint32(uintOrZero(subtraces, i)), + Error: nullableStr(traceError, i), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type traceColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + InternalIndex proto.ColUInt32 + ActionFrom proto.ColStr + ActionTo *proto.ColNullable[string] + ActionValue proto.ColUInt256 + ActionGas *proto.ColNullable[uint64] + ActionInput *proto.ColNullable[string] + ActionCallType *proto.ColLowCardinality[string] + ActionInit *proto.ColNullable[string] + ActionRewardType proto.ColStr + ActionType *proto.ColLowCardinality[string] + ResultGasUsed *proto.ColNullable[uint64] + ResultOutput *proto.ColNullable[string] + ResultCode *proto.ColNullable[string] + ResultAddress *proto.ColNullable[string] + TraceAddress *proto.ColNullable[string] + Subtraces proto.ColUInt32 + Error *proto.ColNullable[string] + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newTraceColumns() *traceColumns { + return &traceColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + ActionTo: new(proto.ColStr).Nullable(), + ActionGas: new(proto.ColUInt64).Nullable(), + ActionInput: new(proto.ColStr).Nullable(), + ActionCallType: new(proto.ColStr).LowCardinality(), + ActionInit: new(proto.ColStr).Nullable(), + ActionType: new(proto.ColStr).LowCardinality(), + ResultGasUsed: new(proto.ColUInt64).Nullable(), + ResultOutput: new(proto.ColStr).Nullable(), + ResultCode: new(proto.ColStr).Nullable(), + ResultAddress: new(proto.ColStr).Nullable(), + TraceAddress: new(proto.ColStr).Nullable(), + Error: new(proto.ColStr).Nullable(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *traceColumns) Append(r traceRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.InternalIndex.Append(r.InternalIndex) + c.ActionFrom.Append(r.ActionFrom) + c.ActionTo.Append(r.ActionTo) + c.ActionValue.Append(r.ActionValue) + c.ActionGas.Append(r.ActionGas) + c.ActionInput.Append(r.ActionInput) + c.ActionCallType.Append(r.ActionCallType) + c.ActionInit.Append(r.ActionInit) + c.ActionRewardType.Append(r.ActionRewardType) + c.ActionType.Append(r.ActionType) + c.ResultGasUsed.Append(r.ResultGasUsed) + c.ResultOutput.Append(r.ResultOutput) + c.ResultCode.Append(r.ResultCode) + c.ResultAddress.Append(r.ResultAddress) + c.TraceAddress.Append(r.TraceAddress) + c.Subtraces.Append(r.Subtraces) + c.Error.Append(r.Error) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *traceColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.InternalIndex.Reset() + c.ActionFrom.Reset() + c.ActionTo.Reset() + c.ActionValue.Reset() + c.ActionGas.Reset() + c.ActionInput.Reset() + c.ActionCallType.Reset() + c.ActionInit.Reset() + c.ActionRewardType.Reset() + c.ActionType.Reset() + c.ResultGasUsed.Reset() + c.ResultOutput.Reset() + c.ResultCode.Reset() + c.ResultAddress.Reset() + c.TraceAddress.Reset() + c.Subtraces.Reset() + c.Error.Reset() + c.MetaNetworkName.Reset() +} + +func (c *traceColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *traceColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "internal_index", Data: &c.InternalIndex}, + {Name: "action_from", Data: &c.ActionFrom}, + {Name: "action_to", Data: c.ActionTo}, + {Name: "action_value", Data: &c.ActionValue}, + {Name: "action_gas", Data: c.ActionGas}, + {Name: "action_input", Data: c.ActionInput}, + {Name: "action_call_type", Data: c.ActionCallType}, + {Name: "action_init", Data: c.ActionInit}, + {Name: "action_reward_type", Data: &c.ActionRewardType}, + {Name: "action_type", Data: c.ActionType}, + {Name: "result_gas_used", Data: c.ResultGasUsed}, + {Name: "result_output", Data: c.ResultOutput}, + {Name: "result_code", Data: c.ResultCode}, + {Name: "result_address", Data: c.ResultAddress}, + {Name: "trace_address", Data: c.TraceAddress}, + {Name: "subtraces", Data: &c.Subtraces}, + {Name: "error", Data: c.Error}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_traces_test.go b/pkg/processor/cryo/ds_traces_test.go new file mode 100644 index 0000000..87d3799 --- /dev/null +++ b/pkg/processor/cryo/ds_traces_test.go @@ -0,0 +1,159 @@ +package cryo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// preMergeBlock has an uncle, so it carries the two reward traces that share a +// NULL transaction hash and must not collapse onto one sort key. +const preMergeBlock = "b15000051" + +func TestTracesDataset(t *testing.T) { + t.Parallel() + + require.Equal(t, "traces", tracesDataset.Name) + require.Equal(t, "canonical_execution_traces", tracesDataset.Table) + require.True(t, tracesDataset.InternalIndex) + require.Zero(t, tracesDataset.MinBlock) +} + +func TestDecodeTraces(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, preMergeBlock, "traces", decodeTraces) + require.NoError(t, err) + require.Len(t, rows, 818) + + // Row 2 is the first trace of the block's first transaction; rows 0 and 1 + // are the reward traces. + r := rows[2] + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + require.Equal(t, uint64(15000051), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x763827e5a5197faf46175f7a83460028926cc6a2721acc9b50b5cceba824fe20", r.TransactionHash) + require.Equal(t, uint32(1), r.InternalIndex) + require.Equal(t, "0x2b4cdbcf60f009889e88b1982f1d1f9ce8fd2233", r.ActionFrom) + require.Equal(t, "0xbeefbabeea323f07c59926295205d3b7a17e8638", r.ActionTo.Value) + require.Zero(t, r.ActionValue) + require.Equal(t, uint64(476176), r.ActionGas.Value) + require.Equal(t, uint64(221260), r.ResultGasUsed.Value) + require.Equal(t, "call", r.ActionCallType) + require.Equal(t, "call", r.ActionType) + require.Empty(t, r.ActionRewardType) + require.False(t, r.ActionInit.Set) + require.False(t, r.ResultCode.Set) + require.False(t, r.ResultAddress.Set) + require.Equal(t, uint32(3), r.Subtraces) + require.False(t, r.Error.Set) + require.Equal(t, "mainnet", r.MetaNetworkName) + + // cryo emits an empty return as the literal "0x", which the target column + // stores as NULL. + require.False(t, r.ResultOutput.Set) + + // trace_address is an underscore-joined path of child indices, never hex. + require.False(t, r.TraceAddress.Set) + require.Equal(t, "0", rows[3].TraceAddress.Value) + require.Equal(t, "2_0_0", rows[7].TraceAddress.Value) + + require.Equal(t, uint32(2), rows[3].InternalIndex) + require.Equal(t, "static_call", rows[3].ActionCallType) + require.Equal(t, "0x0902f1ac", rows[3].ActionInput.Value) +} + +// TestDecodeTracesRewards pins the behaviour the mapping exists to protect: a +// pre-merge block with an uncle produces two reward traces with no transaction +// hash, and both must survive under distinct internal indices. +func TestDecodeTracesRewards(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, preMergeBlock, "traces", decodeTraces) + require.NoError(t, err) + + var rewards []traceRow + + for _, r := range rows { + if r.TransactionHash == emptyHex { + rewards = append(rewards, r) + } + } + + require.Len(t, rewards, 2) + require.Equal(t, uint32(1), rewards[0].InternalIndex) + require.Equal(t, uint32(2), rewards[1].InternalIndex) + + blockReward, err := uint256("2062500000000000000") + require.NoError(t, err) + require.Equal(t, "reward", rewards[0].ActionRewardType) + require.Equal(t, blockReward, rewards[0].ActionValue) + require.Equal(t, "0x1ad91ee08f21be3de0ba2ba6918e714da6b45836", rewards[0].ActionFrom) + + uncleReward, err := uint256("1750000000000000000") + require.NoError(t, err) + require.Equal(t, "uncle", rewards[1].ActionRewardType) + require.Equal(t, uncleReward, rewards[1].ActionValue) + require.Equal(t, "0xea674fdde714fd979de3edf0f56aa9716b898ec8", rewards[1].ActionFrom) + + for _, r := range rewards { + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "reward", r.ActionType) + require.False(t, r.ActionTo.Set) + require.False(t, r.ActionGas.Set) + require.False(t, r.ResultGasUsed.Set) + require.False(t, r.TraceAddress.Set) + require.Empty(t, r.ActionCallType) + } +} + +func TestDecodeTracesPostMerge(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "traces", decodeTraces) + require.NoError(t, err) + require.Len(t, rows, 797) + + var suicides, creates int + + for _, r := range rows { + // Post-merge blocks have no reward traces at all, so every row's + // reward type is NULL in parquet and empty in the non-nullable target. + require.Empty(t, r.ActionRewardType) + require.NotEqual(t, emptyHex, r.TransactionHash) + + switch r.ActionType { + case "suicide": + suicides++ + + require.False(t, r.ActionGas.Set) + require.False(t, r.ResultGasUsed.Set) + require.Empty(t, r.ActionCallType) + case "create": + creates++ + + require.False(t, r.ActionTo.Set) + require.True(t, r.ActionInit.Set) + require.True(t, r.ResultAddress.Set) + require.False(t, r.ResultCode.Set) + } + } + + require.Equal(t, 2, suicides) + require.Equal(t, 2, creates) +} + +func TestTracesColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, preMergeBlock, "traces", decodeTraces) + require.NoError(t, err) + + cols := newTraceColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + requireInputMatchesDDL(t, "canonical_execution_traces", cols.Input()) +} diff --git a/pkg/processor/cryo/ds_transactions.go b/pkg/processor/cryo/ds_transactions.go new file mode 100644 index 0000000..745630e --- /dev/null +++ b/pkg/processor/cryo/ds_transactions.go @@ -0,0 +1,232 @@ +package cryo + +import ( + "fmt" + "math" + "time" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +var transactionsDataset = &Dataset{ + Name: "transactions", + Table: "canonical_execution_transaction", + InternalIndex: false, + MinBlock: 0, + newSink: func(d sinkDeps) sink { + return newDatasetSink(d, decodeTransactions, func() columnar[transactionRow] { return newTransactionColumns() }) + }, +} + +type transactionRow struct { + UpdatedDateTime time.Time + BlockNumber uint64 + TransactionIndex uint64 + TransactionHash string + Nonce uint64 + FromAddress string + ToAddress proto.Nullable[string] + Value string + Input proto.Nullable[string] + GasLimit uint64 + GasUsed uint64 + GasPrice uint64 + TransactionType uint8 + MaxPriorityFeePerGas proto.Nullable[uint64] + MaxFeePerGas proto.Nullable[uint64] + Success bool + NInputBytes uint32 + NInputZeroBytes uint32 + NInputNonzeroBytes uint32 + MetaNetworkName string +} + +func decodeTransactions(t *decode.Table, meta rowMeta) ([]transactionRow, error) { + p := newPicker(t) + + var ( + blockNumber = p.int("block_number") + transactionIndex = p.int("transaction_index") + transactionHash = p.str("transaction_hash") + nonce = p.int("nonce") + fromAddress = p.str("from_address") + toAddress = p.str("to_address") + value = p.str("value_string") + input = p.str("input") + gasLimit = p.int("gas_limit") + gasUsed = p.int("gas_used") + gasPrice = p.int("gas_price") + transactionType = p.int("transaction_type") + maxPriorityFeePerGas = p.int("max_priority_fee_per_gas") + maxFeePerGas = p.int("max_fee_per_gas") + success = p.bool("success") + nInputBytes = p.int("n_input_bytes") + nInputZeroBytes = p.int("n_input_zero_bytes") + nInputNonzeroBytes = p.int("n_input_nonzero_bytes") + ) + + if p.err != nil { + return nil, p.err + } + + rows := make([]transactionRow, 0, t.Rows()) + + for i := range t.Rows() { + // cryo types the envelope as uint32 while the target stores a single + // byte, which is all EIP-2718 allows; a wider value is a protocol + // change rather than a value to truncate. + txType := uintOrZero(transactionType, i) + if txType > math.MaxUint8 { + return nil, fmt.Errorf("row %d: transaction_type %d does not fit UInt8", i, txType) + } + + //nolint:gosec // the n_input_* counts are uint32 in cryo's schema, widened by the decoder + rows = append(rows, transactionRow{ + UpdatedDateTime: meta.updated, + BlockNumber: uintOrZero(blockNumber, i), + TransactionIndex: uintOrZero(transactionIndex, i), + TransactionHash: hexOrEmpty(transactionHash, i), + Nonce: uintOrZero(nonce, i), + FromAddress: hexOrEmpty(fromAddress, i), + ToAddress: nullableHex(toAddress, i), + Value: strOrEmpty(value, i), + Input: nullableHex(input, i), + GasLimit: uintOrZero(gasLimit, i), + GasUsed: uintOrZero(gasUsed, i), + GasPrice: uintOrZero(gasPrice, i), + TransactionType: uint8(txType), + MaxPriorityFeePerGas: nullableUint(maxPriorityFeePerGas, i), + MaxFeePerGas: nullableUint(maxFeePerGas, i), + Success: boolOrFalse(success, i), + NInputBytes: uint32(uintOrZero(nInputBytes, i)), + NInputZeroBytes: uint32(uintOrZero(nInputZeroBytes, i)), + NInputNonzeroBytes: uint32(uintOrZero(nInputNonzeroBytes, i)), + MetaNetworkName: meta.network, + }) + } + + return rows, nil +} + +type transactionColumns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionIndex proto.ColUInt64 + TransactionHash proto.ColFixedStr + Nonce proto.ColUInt64 + FromAddress proto.ColStr + ToAddress *proto.ColNullable[string] + Value proto.ColUInt256 + InputData *proto.ColNullable[string] + GasLimit proto.ColUInt64 + GasUsed proto.ColUInt64 + // GasPrice is wider than anything cryo can produce: the effective gas + // price is a uint64 on the wire, the target column simply reserves room. + GasPrice proto.ColUInt128 + TransactionType proto.ColUInt8 + MaxPriorityFeePerGas *proto.ColNullable[uint64] + MaxFeePerGas *proto.ColNullable[uint64] + Success proto.ColBool + NInputBytes proto.ColUInt32 + NInputZeroBytes proto.ColUInt32 + NInputNonzeroBytes proto.ColUInt32 + MetaNetworkName *proto.ColLowCardinality[string] +} + +func newTransactionColumns() *transactionColumns { + return &transactionColumns{ + TransactionHash: proto.ColFixedStr{Size: hashSize}, + ToAddress: new(proto.ColStr).Nullable(), + InputData: new(proto.ColStr).Nullable(), + MaxPriorityFeePerGas: new(proto.ColUInt64).Nullable(), + MaxFeePerGas: new(proto.ColUInt64).Nullable(), + MetaNetworkName: new(proto.ColStr).LowCardinality(), + } +} + +func (c *transactionColumns) Append(r transactionRow) error { + hash, err := fixedHash(r.TransactionHash) + if err != nil { + return err + } + + value, err := uint256(r.Value) + if err != nil { + return fmt.Errorf("value: %w", err) + } + + c.UpdatedDateTime.Append(r.UpdatedDateTime) + c.BlockNumber.Append(r.BlockNumber) + c.TransactionIndex.Append(r.TransactionIndex) + c.TransactionHash.Append(hash) + c.Nonce.Append(r.Nonce) + c.FromAddress.Append(r.FromAddress) + c.ToAddress.Append(r.ToAddress) + c.Value.Append(value) + c.InputData.Append(r.Input) + c.GasLimit.Append(r.GasLimit) + c.GasUsed.Append(r.GasUsed) + c.GasPrice.Append(proto.UInt128{Low: r.GasPrice}) + c.TransactionType.Append(r.TransactionType) + c.MaxPriorityFeePerGas.Append(r.MaxPriorityFeePerGas) + c.MaxFeePerGas.Append(r.MaxFeePerGas) + c.Success.Append(r.Success) + c.NInputBytes.Append(r.NInputBytes) + c.NInputZeroBytes.Append(r.NInputZeroBytes) + c.NInputNonzeroBytes.Append(r.NInputNonzeroBytes) + c.MetaNetworkName.Append(r.MetaNetworkName) + + return nil +} + +func (c *transactionColumns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionIndex.Reset() + c.TransactionHash.Reset() + c.Nonce.Reset() + c.FromAddress.Reset() + c.ToAddress.Reset() + c.Value.Reset() + c.InputData.Reset() + c.GasLimit.Reset() + c.GasUsed.Reset() + c.GasPrice.Reset() + c.TransactionType.Reset() + c.MaxPriorityFeePerGas.Reset() + c.MaxFeePerGas.Reset() + c.Success.Reset() + c.NInputBytes.Reset() + c.NInputZeroBytes.Reset() + c.NInputNonzeroBytes.Reset() + c.MetaNetworkName.Reset() +} + +func (c *transactionColumns) Rows() int { return c.BlockNumber.Rows() } + +func (c *transactionColumns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "nonce", Data: &c.Nonce}, + {Name: "from_address", Data: &c.FromAddress}, + {Name: "to_address", Data: c.ToAddress}, + {Name: "value", Data: &c.Value}, + {Name: "input", Data: c.InputData}, + {Name: "gas_limit", Data: &c.GasLimit}, + {Name: "gas_used", Data: &c.GasUsed}, + {Name: "gas_price", Data: &c.GasPrice}, + {Name: "transaction_type", Data: &c.TransactionType}, + {Name: "max_priority_fee_per_gas", Data: c.MaxPriorityFeePerGas}, + {Name: "max_fee_per_gas", Data: c.MaxFeePerGas}, + {Name: "success", Data: &c.Success}, + {Name: "n_input_bytes", Data: &c.NInputBytes}, + {Name: "n_input_zero_bytes", Data: &c.NInputZeroBytes}, + {Name: "n_input_nonzero_bytes", Data: &c.NInputNonzeroBytes}, + {Name: "meta_network_name", Data: c.MetaNetworkName}, + } +} diff --git a/pkg/processor/cryo/ds_transactions_test.go b/pkg/processor/cryo/ds_transactions_test.go new file mode 100644 index 0000000..39374ba --- /dev/null +++ b/pkg/processor/cryo/ds_transactions_test.go @@ -0,0 +1,90 @@ +package cryo + +import ( + "testing" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" +) + +func TestDecodeTransactions(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "transactions", decodeTransactions) + require.NoError(t, err) + require.Len(t, rows, 137) + + r := rows[0] + require.Equal(t, fixtureMeta.updated, r.UpdatedDateTime) + require.Equal(t, uint64(23000000), r.BlockNumber) + require.Equal(t, uint64(0), r.TransactionIndex) + require.Equal(t, "0x5946ef0de28db53ffe00f0fcdb4cc19c9da234819430bf957cc4c15200d8bcac", r.TransactionHash) + require.Equal(t, uint64(12963272), r.Nonce) + require.Equal(t, "0x28c6c06298d514db089934071355e5743bf21d60", r.FromAddress) + require.Equal(t, "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", r.ToAddress.Value) + require.Equal(t, "0", r.Value) + require.Equal(t, + "0xa9059cbb000000000000000000000000f2abf514dfbaf3323a5cb95bb2e4ab180a5ba3e80000000000000000000000000000000000000000000000000000000129d08a20", + r.Input.Value) + require.Equal(t, uint64(207128), r.GasLimit) + require.Equal(t, uint64(62272), r.GasUsed) + require.Equal(t, uint64(2239712557), r.GasPrice) + require.Equal(t, uint8(2), r.TransactionType) + require.Equal(t, uint64(2000000000), r.MaxPriorityFeePerGas.Value) + require.Equal(t, uint64(102000000000), r.MaxFeePerGas.Value) + require.True(t, r.Success) + require.Equal(t, uint32(68), r.NInputBytes) + require.Equal(t, uint32(39), r.NInputZeroBytes) + require.Equal(t, uint32(29), r.NInputNonzeroBytes) + require.Equal(t, "mainnet", r.MetaNetworkName) + + // A plain transfer carries no calldata, which cryo emits as "0x". + require.False(t, rows[1].Input.Set) + + // Legacy transactions have no 1559 fee fields at all, which is distinct + // from having them set to zero. + require.Equal(t, uint8(0), rows[2].TransactionType) + require.False(t, rows[2].MaxPriorityFeePerGas.Set) + require.False(t, rows[2].MaxFeePerGas.Set) + require.Equal(t, uint64(31022955380952), rows[2].GasPrice) + + missing := 0 + + for _, row := range rows { + if !row.MaxFeePerGas.Set { + missing++ + } + + require.Equal(t, !row.MaxFeePerGas.Set, !row.MaxPriorityFeePerGas.Set) + } + + require.Equal(t, 18, missing) + + require.Equal(t, uint8(4), rows[3].TransactionType) + require.Equal(t, uint8(3), rows[45].TransactionType) + + // Values routinely exceed uint64, so they must survive as decimal text. + require.Equal(t, "82520860000000000000", rows[56].Value) + + require.False(t, rows[88].Success) + require.Equal(t, "0x879d8faa84bfd73fd019a39e26a819d430636f5244089389b1003c407224dac7", rows[88].TransactionHash) +} + +func TestTransactionColumnsAppend(t *testing.T) { + t.Parallel() + + rows, err := decodeFixture(t, "b23000000", "transactions", decodeTransactions) + require.NoError(t, err) + + cols := newTransactionColumns() + for _, r := range rows { + require.NoError(t, cols.Append(r)) + } + + require.Equal(t, len(rows), cols.Rows()) + + require.Equal(t, proto.UInt256{Low: proto.UInt128{Low: 8733883705161793536, High: 4}}, cols.Value[56]) + require.Equal(t, proto.UInt128{Low: 31022955380952}, cols.GasPrice[2]) + + requireInputMatchesDDL(t, "canonical_execution_transaction", cols.Input()) +} diff --git a/pkg/processor/cryo/fetch.go b/pkg/processor/cryo/fetch.go new file mode 100644 index 0000000..c301f16 --- /dev/null +++ b/pkg/processor/cryo/fetch.go @@ -0,0 +1,400 @@ +package cryo + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + "unicode/utf8" + + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +const ( + // tempDirPattern marks every scratch directory so the startup sweep can + // tell them apart from anything else sharing the temp filesystem. + tempDirPattern = "cryo-*" + + // killGrace is how long a killed cryo has to exit before Wait gives up on + // it, so a wedged subprocess cannot hold a worker forever. + killGrace = 5 * time.Second + + // stderrLimit bounds what is retained from a failing cryo, so a chatty + // failure cannot grow without limit. + stderrLimit = 8 << 10 + + // sweepMinAge is how old an orphan must look before startup removes it. + // Replicas can share a temp filesystem, so this must comfortably exceed any + // plausible in-flight fetch or one replica will delete another's working + // directory mid-collection. + sweepMinAge = time.Hour + + // redacted replaces the RPC endpoint wherever cryo echoes it back, because + // the endpoint usually carries basic-auth credentials and a failed fetch's + // error text ends up in the logs. + redacted = "[redacted]" +) + +// Fetch failure causes. Every error Fetch returns wraps exactly one of these, +// so a caller can attribute the failure without matching on message text. +var ( + // ErrNoEndpoint means the pool offered no healthy node to collect from, so + // cryo was never started. + ErrNoEndpoint = errors.New("no rpc endpoint available") + + // ErrFetchTimeout means the cryo subprocess outlived its fetch timeout and + // was killed. + ErrFetchTimeout = errors.New("cryo subprocess timed out") + + // ErrFetchFailed means cryo ran and exited non-zero. + ErrFetchFailed = errors.New("cryo subprocess failed") + + // ErrParquetMissing means cryo exited successfully without writing the + // output for a dataset the group expects, which must never be read as a + // block that legitimately had no rows. + ErrParquetMissing = errors.New("cryo wrote no parquet") + + // ErrParquetAmbiguous means more than one file matched a dataset, so the + // one holding this block's rows cannot be identified. + ErrParquetAmbiguous = errors.New("cryo wrote more than one parquet") + + // ErrDecodeFailed means the parquet cryo wrote could not be decoded. + ErrDecodeFailed = errors.New("parquet decode failed") +) + +// boundedBuffer keeps at most stderrLimit bytes and counts what it discards. +type boundedBuffer struct { + buf []byte + dropped int +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + room := stderrLimit - len(b.buf) + + switch { + case room <= 0: + b.dropped += len(p) + case room >= len(p): + b.buf = append(b.buf, p...) + default: + b.buf = append(b.buf, p[:room]...) + b.dropped += len(p) - room + } + + return len(p), nil +} + +// String renders what was kept. The byte cap can land mid-rune, so the result +// is scrubbed to valid UTF-8 before it reaches an error or a log line. +func (b *boundedBuffer) String() string { + kept := strings.ToValidUTF8(string(b.buf), "") + + if b.dropped == 0 { + return kept + } + + return fmt.Sprintf("%s… (%d more bytes)", kept, b.dropped) +} + +// Fetcher retrieves decoded rows for one group over a contiguous block range, +// keyed by dataset name. +// +// Per-block processing calls this with from == to. A range-batched +// implementation would change only this seam and the enqueue path, not the +// tracking model, so cryo's command line must not leak past it. +type Fetcher interface { + Fetch(ctx context.Context, g *Group, from, to uint64) (map[string]*decode.Table, error) +} + +// endpointFunc resolves the JSON-RPC URL for one invocation. It is called per +// fetch rather than captured once so that cryo follows the pool away from an +// unhealthy node. +type endpointFunc func() (string, error) + +// execFetcher runs cryo as a subprocess and decodes the parquet it writes. +type execFetcher struct { + log logrus.FieldLogger + binaryPath string + endpoint endpointFunc + tempDir string + globalArgs []string + timeout time.Duration +} + +func newExecFetcher(log logrus.FieldLogger, cfg *Config, endpoint endpointFunc) *execFetcher { + return &execFetcher{ + log: log, + binaryPath: cfg.BinaryPath, + endpoint: endpoint, + tempDir: cfg.TempDir, + globalArgs: cfg.GlobalArgs, + timeout: cfg.FetchTimeout, + } +} + +// Version returns the bundled cryo's version string. +func (f *execFetcher) Version(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + //nolint:gosec // G204: the binary path is operator configuration, not request data + out, err := exec.CommandContext(ctx, f.binaryPath, "--version").Output() + if err != nil { + return "", fmt.Errorf("exec %s --version: %w%s", f.binaryPath, err, exitStderr(err)) + } + + return strings.TrimSpace(string(out)), nil +} + +// exitStderr renders what an *exec.ExitError captured on stderr, or an empty +// string when there is none. Without it the error reads only "exit status 127", +// which hides the reason a binary that exists still will not run — a missing +// shared library such as libssl.so.3 says so only on stderr. +func exitStderr(err error) string { + var exitErr *exec.ExitError + + if !errors.As(err, &exitErr) { + return "" + } + + text := strings.TrimSpace(string(exitErr.Stderr)) + if text == "" { + return "" + } + + return ": " + truncate(text, stderrLimit) +} + +// Fetch runs one cryo invocation and decodes every dataset the group expects. +func (f *execFetcher) Fetch(ctx context.Context, g *Group, from, to uint64) (map[string]*decode.Table, error) { + dir, err := os.MkdirTemp(f.tempDir, tempDirPattern) + if err != nil { + return nil, fmt.Errorf("create temp dir: %w", err) + } + + defer func() { + if rmErr := os.RemoveAll(dir); rmErr != nil { + f.log.WithError(rmErr).WithField("dir", dir).Error("Failed to remove cryo temp dir") + } + }() + + if err := f.run(ctx, g, from, to, dir); err != nil { + return nil, err + } + + collected := g.collected() + tables := make(map[string]*decode.Table, len(collected)) + + for _, ds := range collected { + path, findErr := findParquet(dir, ds.Name, from, to) + if findErr != nil { + return nil, findErr + } + + table, decodeErr := decode.File(path) + if decodeErr != nil { + return nil, fmt.Errorf("%w: dataset %s: %w", ErrDecodeFailed, ds.Name, decodeErr) + } + + tables[ds.Name] = table + } + + return tables, nil +} + +// run executes cryo once for the whole group. +func (f *execFetcher) run(ctx context.Context, g *Group, from, to uint64, dir string) error { + ctx, cancel := context.WithTimeout(ctx, f.timeout) + defer cancel() + + rpcURL, err := f.endpoint() + if err != nil { + return fmt.Errorf("%w: %w", ErrNoEndpoint, err) + } + + // cryo's range end is exclusive. + args := make([]string, 0, len(g.Datatypes)+len(f.globalArgs)+len(g.ExtraArgs)+6) + args = append(args, g.Datatypes...) + args = append(args, + "--blocks", fmt.Sprintf("%d:%d", from, to+1), + "--output-dir", dir, + "--no-report", + ) + args = append(args, f.globalArgs...) + + if required := g.RequiredColumns(); len(required) > 0 { + args = append(args, "--include-columns", strings.Join(required, ",")) + } + + args = append(args, g.ExtraArgs...) + + //nolint:gosec // G204: the binary path and arguments are operator configuration, not request data + cmd := exec.CommandContext(ctx, f.binaryPath, args...) + + // The endpoint usually carries basic-auth credentials, so it goes through + // the environment rather than argv, which any process sharing the PID + // namespace can read out of /proc. + cmd.Env = append(os.Environ(), "ETH_RPC_URL="+rpcURL) + + // Killing a hung cryo must also kill anything it spawned: a grandchild that + // outlived its parent would keep an RPC connection open against a node the + // worker has already given up on. WaitDelay additionally stops the parent + // blocking on pipes that group inherited. + killProcessGroup(cmd) + + cmd.WaitDelay = killGrace + + var stderr boundedBuffer + + cmd.Stderr = &stderr + cmd.Stdout = nil + + start := time.Now() + + if err := cmd.Run(); err != nil { + cause := ErrFetchFailed + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + cause = ErrFetchTimeout + } + + // Redact before truncating: cutting the text first could leave a + // fragment of the credentials behind. + detail := truncate(redactEndpoint(stderr.String(), rpcURL), 2048) + + return fmt.Errorf("%w: datatypes %s blocks %d-%d: %w: %s", + cause, strings.Join(g.Datatypes, " "), from, to, err, detail) + } + + f.log.WithFields(logrus.Fields{ + "group": g.Name, + "from": from, + "to": to, + "duration": time.Since(start), + }).Debug("cryo invocation complete") + + return nil +} + +// findParquet locates the file cryo wrote for one dataset. +// +// Only the dataset is matched. cryo decorates the rest of the name in ways that +// are not worth predicting: the prefix is whatever it resolved the chain id to, +// and the block numbers are zero-padded to a width it chooses, so a chain whose +// heights are shorter than mainnet's produces 01647952 where the request said +// 1647952. The invocation writes into a directory of its own, so the dataset +// alone identifies the file, and two matches are treated as an error rather +// than guessed between. +func findParquet(dir, dataset string, from, to uint64) (string, error) { + pattern := filepath.Join(dir, fmt.Sprintf("*__%s__*.parquet", dataset)) + + matches, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("glob %s: %w", pattern, err) + } + + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf("%w for dataset %s at blocks %d-%d", ErrParquetMissing, dataset, from, to) + default: + return "", fmt.Errorf("%w: %d files for dataset %s at blocks %d-%d", ErrParquetAmbiguous, len(matches), dataset, from, to) + } +} + +// redactEndpoint removes the RPC endpoint from text cryo produced. cryo echoes +// its target in several failure messages and the endpoint usually carries +// basic-auth credentials, which would otherwise travel with the error into the +// logs. The credentials are also matched on their own, because cryo may print a +// normalised or partial form of the URL. +func redactEndpoint(text, endpoint string) string { + if endpoint == "" || text == "" { + return text + } + + for _, form := range []string{endpoint, strings.TrimSuffix(endpoint, "/")} { + text = strings.ReplaceAll(text, form, redacted) + } + + parsed, err := url.Parse(endpoint) + if err != nil || parsed.User == nil { + return text + } + + if userinfo := parsed.User.String(); userinfo != "" { + text = strings.ReplaceAll(text, userinfo, redacted) + } + + if password, ok := parsed.User.Password(); ok && password != "" { + text = strings.ReplaceAll(text, password, redacted) + } + + return text +} + +// sweepTempDirs removes scratch directories left behind by a process that was +// killed before its deferred cleanup could run. Without this they accumulate in +// the container's writable layer until the node evicts the pod for disk +// pressure. +func sweepTempDirs(log logrus.FieldLogger, parent string) { + if parent == "" { + parent = os.TempDir() + } + + matches, err := filepath.Glob(filepath.Join(parent, tempDirPattern)) + if err != nil { + log.WithError(err).Warn("Failed to scan for orphaned cryo temp dirs") + + return + } + + var removed int + + cutoff := time.Now().Add(-sweepMinAge) + + for _, dir := range matches { + info, statErr := os.Stat(dir) + if statErr != nil || !info.IsDir() { + continue + } + + if info.ModTime().After(cutoff) { + continue + } + + if rmErr := os.RemoveAll(dir); rmErr != nil { + log.WithError(rmErr).WithField("dir", dir).Warn("Failed to remove orphaned cryo temp dir") + + continue + } + + removed++ + } + + if removed > 0 { + log.WithField("count", removed).Info("Removed orphaned cryo temp dirs") + } +} + +// truncate keeps at most limit bytes of s. The cut is moved back to a rune +// boundary so a multi-byte character straddling the limit is dropped whole +// rather than left as invalid UTF-8 in a log line. +func truncate(s string, limit int) string { + if len(s) <= limit { + return s + } + + cut := limit + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + + return s[:cut] + "…" +} diff --git a/pkg/processor/cryo/fetch_other.go b/pkg/processor/cryo/fetch_other.go new file mode 100644 index 0000000..e5bbddb --- /dev/null +++ b/pkg/processor/cryo/fetch_other.go @@ -0,0 +1,9 @@ +//go:build !unix + +package cryo + +import "os/exec" + +// killProcessGroup leaves the default behaviour in place: platforms without +// process groups fall back to signalling the direct child only. +func killProcessGroup(_ *exec.Cmd) {} diff --git a/pkg/processor/cryo/fetch_test.go b/pkg/processor/cryo/fetch_test.go new file mode 100644 index 0000000..9783004 --- /dev/null +++ b/pkg/processor/cryo/fetch_test.go @@ -0,0 +1,524 @@ +package cryo + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + "unicode/utf8" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +// fakeCryo writes an executable stub in place of cryo. The body receives cryo's +// arguments and is responsible for producing whatever output the test needs. +// +// Callers must not run in parallel: exec'ing a freshly written file races with +// any concurrent fork, which inherits the write descriptor and makes exec fail +// with ETXTBSY. +func fakeCryo(t *testing.T, body string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "cryo") + script := "#!/bin/sh\n" + body + "\n" + + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + + return path +} + +func testFetcher(t *testing.T, binary string) *execFetcher { + t.Helper() + + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: binary, TempDir: t.TempDir()} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + return newExecFetcher(log, cfg, func() (string, error) { return "https://user:secret@node.example/", nil }) +} + +func testGroup(t *testing.T, datatypes ...string) *Group { + t.Helper() + + g, err := newGroup(&GroupConfig{Name: "t", Enabled: true, Datatypes: datatypes}) + require.NoError(t, err) + + return g +} + +// TestFetchPassesEndpointThroughEnvironment is the security-relevant one: the +// endpoint carries basic-auth credentials, and anything in argv is readable +// from /proc by every process sharing the PID namespace. +func TestFetchPassesEndpointThroughEnvironment(t *testing.T) { + binary := fakeCryo(t, ` +for arg in "$@"; do + case "$arg" in + *secret*) echo "credential leaked into argv: $arg" >&2; exit 3 ;; + esac +done +[ "$ETH_RPC_URL" = "https://user:secret@node.example/" ] || { echo "env not set: $ETH_RPC_URL" >&2; exit 4; } +exit 9 +`) + + _, err := testFetcher(t, binary).Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.NotContains(t, err.Error(), "leaked into argv") + require.NotContains(t, err.Error(), "env not set") + require.Contains(t, err.Error(), "exit status 9") +} + +func TestFetchBuildsArguments(t *testing.T) { + argsFile := filepath.Join(t.TempDir(), "args") + binary := fakeCryo(t, `printf '%s\n' "$@" > `+argsFile+`; exit 1`) + + group := testGroup(t, "blocks", "state_diffs") + + _, err := testFetcher(t, binary).Fetch(t.Context(), group, 100, 100) + require.Error(t, err) + + raw, readErr := os.ReadFile(argsFile) + require.NoError(t, readErr) + + args := strings.Split(strings.TrimSpace(string(raw)), "\n") + + require.Equal(t, "blocks", args[0]) + require.Equal(t, "state_diffs", args[1]) + require.Contains(t, args, "--hex") + require.Contains(t, args, "--u256-types") + require.Contains(t, args, "--no-report") + + // Derived from the datasets rather than taken from config, so it cannot be + // configured away. + require.Contains(t, args, "--include-columns") + require.Contains(t, args, "gas_limit") + + // cryo's range end is exclusive, so one block is N:N+1. + blocks := args[indexOf(t, args, "--blocks")+1] + require.Equal(t, "100:101", blocks) +} + +func TestFetchDecodesEveryDatasetInTheGroup(t *testing.T) { + // state_diffs is one argument yielding three datasets, so a group naming it + // must come back with all three decoded. + src, err := filepath.Abs(filepath.Join("testdata", "b23000000")) + require.NoError(t, err) + + binary := fakeCryo(t, ` +out="" +while [ $# -gt 0 ]; do + case "$1" in --output-dir) out="$2"; shift 2 ;; *) shift ;; esac +done +for f in `+src+`/*.parquet; do + base=$(basename "$f") + ds=$(echo "$base" | sed 's/^ethereum__//; s/__.*//') + cp "$f" "$out/ethereum__${ds}__100_to_100.parquet" +done +`) + + tables, err := testFetcher(t, binary).Fetch(t.Context(), testGroup(t, "blocks", "state_diffs"), 100, 100) + require.NoError(t, err) + + require.Len(t, tables, 4) + require.Equal(t, 1, tables["blocks"].Rows()) + require.Equal(t, 377, tables["balance_diffs"].Rows()) + require.Equal(t, 139, tables["nonce_diffs"].Rows()) + require.Equal(t, 359, tables["storage_diffs"].Rows()) +} + +// TestFetchFailsWhenDatasetMissing covers cryo exiting successfully having +// written nothing, which must not be mistaken for a block with no rows. +func TestFetchFailsWhenDatasetMissing(t *testing.T) { + binary := fakeCryo(t, "exit 0") + + _, err := testFetcher(t, binary).Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.Contains(t, err.Error(), "no parquet for dataset blocks") +} + +func TestFetchRemovesTempDirOnEveryPath(t *testing.T) { + parent := t.TempDir() + + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: fakeCryo(t, "echo boom >&2; exit 1"), TempDir: parent} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { return "http://node", nil }) + + for range 3 { + _, err := f.Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + require.Error(t, err) + } + + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Empty(t, entries, "every failed fetch must remove its scratch directory") +} + +func TestFetchKillsHungSubprocess(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: fakeCryo(t, "sleep 120"), TempDir: t.TempDir(), FetchTimeout: 300 * time.Millisecond} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { return "http://node", nil }) + + start := time.Now() + _, err := f.Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.Less(t, time.Since(start), 30*time.Second, "a hung cryo must not hold the worker") +} + +// TestFetchStopsOnContextCancel covers pod drain: an in-flight fetch must +// unblock rather than run to its own timeout. +func TestFetchStopsOnContextCancel(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: fakeCryo(t, "sleep 120"), TempDir: t.TempDir()} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { return "http://node", nil }) + + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := f.Fetch(ctx, testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.Less(t, time.Since(start), 30*time.Second) +} + +func TestFetchStderrIsBounded(t *testing.T) { + binary := fakeCryo(t, `i=0; while [ $i -lt 4000 ]; do echo "noisy failure line $i"; i=$((i+1)); done >&2; exit 1`) + + _, err := testFetcher(t, binary).Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.Less(t, len(err.Error()), 3*stderrLimit, "a chatty failure must not balloon the error") +} + +func TestSweepTempDirsSparesRecentDirectories(t *testing.T) { + t.Parallel() + + parent := t.TempDir() + + live := filepath.Join(parent, "cryo-live") + require.NoError(t, os.Mkdir(live, 0o750)) + + orphan := filepath.Join(parent, "cryo-orphan") + require.NoError(t, os.Mkdir(orphan, 0o750)) + + old := time.Now().Add(-2 * sweepMinAge) + require.NoError(t, os.Chtimes(orphan, old, old)) + + unrelated := filepath.Join(parent, "something-else") + require.NoError(t, os.Mkdir(unrelated, 0o750)) + require.NoError(t, os.Chtimes(unrelated, old, old)) + + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + sweepTempDirs(log, parent) + + require.DirExists(t, live, "another replica's in-flight directory must survive") + require.NoDirExists(t, orphan) + require.DirExists(t, unrelated, "the sweep must only touch its own directories") +} + +func TestVersionReportsBinary(t *testing.T) { + version, err := testFetcher(t, fakeCryo(t, `echo "cryo 0.3.2-37-g559b654"`)).Version(t.Context()) + + require.NoError(t, err) + require.Equal(t, "cryo 0.3.2-37-g559b654", version) +} + +func TestVersionFailsWhenBinaryMissing(t *testing.T) { + t.Parallel() + + _, err := testFetcher(t, "/nonexistent/cryo").Version(t.Context()) + + require.Error(t, err) +} + +// TestFetchRedactsEndpointFromStderr is the other half of keeping the +// credentials out of reach: they stay out of argv, but cryo echoes its target +// back on stderr, and that text is embedded in the error the worker logs. +func TestFetchRedactsEndpointFromStderr(t *testing.T) { + for name, body := range map[string]string{ + "full url": `echo "error sending request for url ($ETH_RPC_URL): connection refused" >&2; exit 1`, + "credentials": `echo "authentication failed for user:secret" >&2; exit 1`, + } { + t.Run(name, func(t *testing.T) { + _, err := testFetcher(t, fakeCryo(t, body)).Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.NotContains(t, err.Error(), "secret", "the endpoint's basic-auth credentials must not reach the logs") + require.Contains(t, err.Error(), redacted) + }) + } +} + +func TestFetchErrorsAreDistinguishable(t *testing.T) { + parquet := filepath.Join(t.TempDir(), "blocks.parquet") + require.NoError(t, os.WriteFile(parquet, []byte("not parquet at all"), 0o600)) + + tests := []struct { + name string + body string + want error + }{ + { + name: "subprocess failure", + body: "echo boom >&2; exit 1", + want: ErrFetchFailed, + }, + { + name: "missing parquet", + body: "exit 0", + want: ErrParquetMissing, + }, + { + name: "decode failure", + body: `out="" +while [ $# -gt 0 ]; do + case "$1" in --output-dir) out="$2"; shift 2 ;; *) shift ;; esac +done +cp ` + parquet + ` "$out/ethereum__blocks__100_to_100.parquet"`, + want: ErrDecodeFailed, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := testFetcher(t, fakeCryo(t, tt.body)).Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.Error(t, err) + require.ErrorIs(t, err, tt.want) + }) + } +} + +func TestFetchErrorIsTimeoutWhenSubprocessOverruns(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: fakeCryo(t, "sleep 120"), TempDir: t.TempDir(), FetchTimeout: 300 * time.Millisecond} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { return "http://node", nil }) + + _, err := f.Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.ErrorIs(t, err, ErrFetchTimeout) + require.NotErrorIs(t, err, ErrFetchFailed, "a timeout is a different operational problem to a non-zero exit") +} + +func TestFetchErrorIsNoEndpointWhenPoolIsUnhealthy(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{BinaryPath: fakeCryo(t, "exit 0"), TempDir: t.TempDir()} + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { + return "", errors.New("no healthy execution node exposes an RPC endpoint for cryo") + }) + + _, err := f.Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + + require.ErrorIs(t, err, ErrNoEndpoint) +} + +// TestFetchKillsGrandchildren covers what the direct child alone does not: cryo +// spawning a helper that outlives it would keep an RPC connection open against a +// node the worker has already given up on. +func TestFetchKillsGrandchildren(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") + + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + cfg := &Config{ + BinaryPath: fakeCryo(t, "sleep 120 & echo $! > "+pidFile+"; sleep 120"), + TempDir: t.TempDir(), + FetchTimeout: 300 * time.Millisecond, + } + cfg.Addr = "localhost:9000" + cfg.Enabled = true + cfg.Groups = []GroupConfig{{Name: "t", Enabled: true, Datatypes: []string{"blocks"}}} + require.NoError(t, cfg.Validate()) + + f := newExecFetcher(log, cfg, func() (string, error) { return "http://node", nil }) + + _, err := f.Fetch(t.Context(), testGroup(t, "blocks"), 100, 100) + require.Error(t, err) + + raw, readErr := os.ReadFile(pidFile) + require.NoError(t, readErr) + + pid, parseErr := strconv.Atoi(strings.TrimSpace(string(raw))) + require.NoError(t, parseErr) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) != nil + }, 10*time.Second, 50*time.Millisecond, "a process cryo spawned must not survive the kill") +} + +func TestVersionErrorIncludesStderr(t *testing.T) { + // The documented failure is a binary that exists but cannot load a shared + // library, which says so only on stderr while the status says only 127. + binary := fakeCryo(t, `echo "cryo: error while loading shared libraries: libssl.so.3: cannot open shared object file" >&2; exit 127`) + + _, err := testFetcher(t, binary).Version(t.Context()) + + require.Error(t, err) + require.Contains(t, err.Error(), "libssl.so.3") + require.Contains(t, err.Error(), "exit status 127") +} + +func TestRedactEndpoint(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + text string + endpoint string + want string + }{ + { + name: "full url", + text: "dialing https://user:secret@node.example/ failed", + endpoint: "https://user:secret@node.example/", + want: "dialing " + redacted + " failed", + }, + { + name: "url without trailing slash", + text: "dialing https://user:secret@node.example failed", + endpoint: "https://user:secret@node.example/", + want: "dialing " + redacted + " failed", + }, + { + name: "credentials alone", + text: "rejected credentials user:secret", + endpoint: "https://user:secret@node.example/", + want: "rejected credentials " + redacted, + }, + { + name: "password alone", + text: "bad password secret", + endpoint: "https://user:secret@node.example/", + want: "bad password " + redacted, + }, + { + name: "endpoint without credentials", + text: "dialing http://node.example failed", + endpoint: "http://node.example", + want: "dialing " + redacted + " failed", + }, + { + name: "unrelated text", + text: "block 100 not found", + endpoint: "https://user:secret@node.example/", + want: "block 100 not found", + }, + { + name: "no endpoint", + text: "block 100 not found", + endpoint: "", + want: "block 100 not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tt.want, redactEndpoint(tt.text, tt.endpoint)) + }) + } +} + +func TestTruncateKeepsValidUTF8(t *testing.T) { + t.Parallel() + + // "…" is three bytes, so every limit that lands inside one exercises the + // rune boundary walk. + for limit := range 10 { + got := truncate("……………", limit) + + require.True(t, utf8.ValidString(got), "limit %d split a rune: %q", limit, got) + require.LessOrEqual(t, len(got), limit+len("…")) + } + + require.Equal(t, "short", truncate("short", 5)) + require.Equal(t, "shor…", truncate("short", 4)) +} + +func indexOf(t *testing.T, args []string, want string) int { + t.Helper() + + for i, arg := range args { + if arg == want { + return i + } + } + + t.Fatalf("argument %q not passed to cryo: %v", want, args) + + return -1 +} + +// TestFetchFindsZeroPaddedOutput covers cryo padding block numbers in the file +// name. Mainnet heights are already eight digits so this is invisible there, +// but it breaks every shorter height: the whole of mainnet below block +// 10,000,000, and every testnet. +func TestFetchFindsZeroPaddedOutput(t *testing.T) { + src, err := filepath.Abs(filepath.Join("testdata", "b23000000")) + require.NoError(t, err) + + binary := fakeCryo(t, ` +out="" +while [ $# -gt 0 ]; do + case "$1" in --output-dir) out="$2"; shift 2 ;; *) shift ;; esac +done +cp `+src+`/ethereum__blocks__*.parquet "$out/network_560048__blocks__01647952_to_01647952.parquet" +`) + + tables, err := testFetcher(t, binary).Fetch(t.Context(), testGroup(t, "blocks"), 1647952, 1647952) + + require.NoError(t, err, "a zero-padded, non-ethereum-prefixed name must still be found") + require.Equal(t, 1, tables["blocks"].Rows()) +} diff --git a/pkg/processor/cryo/fetch_unix.go b/pkg/processor/cryo/fetch_unix.go new file mode 100644 index 0000000..c5f1a4b --- /dev/null +++ b/pkg/processor/cryo/fetch_unix.go @@ -0,0 +1,15 @@ +//go:build unix + +package cryo + +import ( + "os/exec" + "syscall" +) + +// killProcessGroup makes cryo lead its own process group so that cancelling +// the command signals everything it spawned, not just the direct child. +func killProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } +} diff --git a/pkg/processor/cryo/fixture_test.go b/pkg/processor/cryo/fixture_test.go new file mode 100644 index 0000000..1b301af --- /dev/null +++ b/pkg/processor/cryo/fixture_test.go @@ -0,0 +1,126 @@ +package cryo + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "testing" + "time" + + "github.com/ClickHouse/ch-go/proto" + "github.com/stretchr/testify/require" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// fixtureMeta is the per-batch metadata every fixture decode uses, fixed so +// golden assertions do not move with the clock. +var fixtureMeta = rowMeta{ + network: "mainnet", + updated: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), +} + +// fixtureTable decodes a committed cryo parquet fixture. +func fixtureTable(t *testing.T, block, dataset string) *decode.Table { + t.Helper() + + matches, err := filepath.Glob(filepath.Join("testdata", block, fmt.Sprintf("ethereum__%s__*.parquet", dataset))) + require.NoError(t, err) + require.Len(t, matches, 1, "expected exactly one %s fixture in %s", dataset, block) + + table, err := decode.File(matches[0]) + require.NoError(t, err) + + return table +} + +// decodeFixture runs a dataset mapping over its committed fixture. +func decodeFixture[R any](t *testing.T, block, dataset string, fn decodeFunc[R]) ([]R, error) { + t.Helper() + + return fn(fixtureTable(t, block, dataset), fixtureMeta) +} + +var ( + ddlTableRe = regexp.MustCompile(`(?s)CREATE TABLE IF NOT EXISTS (\w+)\s*\((.*?)\n\)`) + ddlColumnRe = regexp.MustCompile("^\\s*`([^`]+)`\\s+(.+?)(?:\\s+COMMENT\\s|\\s+CODEC\\(|,?$)") +) + +// schemaDDL parses the committed target DDL into table -> column -> type. +func schemaDDL(t *testing.T) map[string]map[string]string { + t.Helper() + + raw, err := os.ReadFile(filepath.Join("testdata", "schema.sql")) + require.NoError(t, err) + + tables := make(map[string]map[string]string) + + for _, match := range ddlTableRe.FindAllStringSubmatch(string(raw), -1) { + cols := make(map[string]string) + + for line := range splitLines(match[2]) { + parts := ddlColumnRe.FindStringSubmatch(line) + if parts == nil { + continue + } + + cols[parts[1]] = trimTrailingComma(parts[2]) + } + + tables[match[1]] = cols + } + + require.Len(t, tables, 16, "expected 16 tables in testdata/schema.sql") + + return tables +} + +func splitLines(s string) func(func(string) bool) { + return func(yield func(string) bool) { + start := 0 + + for i := range len(s) { + if s[i] != '\n' { + continue + } + + if !yield(s[start:i]) { + return + } + + start = i + 1 + } + + if start < len(s) { + yield(s[start:]) + } + } +} + +func trimTrailingComma(s string) string { + for len(s) > 0 && (s[len(s)-1] == ',' || s[len(s)-1] == ' ') { + s = s[:len(s)-1] + } + + return s +} + +// requireInputMatchesDDL asserts that a dataset's column block writes exactly +// the columns its target table declares, with exactly matching types. This is +// the compile-time-adjacent guard on the mappings: a wrong width or a missed +// Nullable would otherwise only surface as a ClickHouse insert error. +func requireInputMatchesDDL(t *testing.T, table string, input proto.Input) { + t.Helper() + + want, ok := schemaDDL(t)[table] + require.True(t, ok, "table %s missing from testdata/schema.sql", table) + + got := make(map[string]string, len(input)) + + for _, col := range input { + got[col.Name] = string(col.Data.Type()) + } + + require.Equal(t, want, got, "column mapping for %s does not match its DDL", table) +} diff --git a/pkg/processor/cryo/group.go b/pkg/processor/cryo/group.go new file mode 100644 index 0000000..321ac7d --- /dev/null +++ b/pkg/processor/cryo/group.go @@ -0,0 +1,118 @@ +package cryo + +import ( + "fmt" + "slices" +) + +// Group is one cryo invocation: the datatypes collected together, and the +// datasets that come back. It is also the unit of progress tracking, because +// an invocation succeeds or fails as a whole and one completion flag should +// represent one thing that actually happened. +type Group struct { + // Name is the suffix of the processor name, and therefore the checkpoint + // key. It is permanent for the lifetime of the data. + Name string + // Datatypes are the arguments passed to cryo. + Datatypes []string + // Datasets are the parquet files that come back, one per entry. + Datasets []*Dataset + // ExtraArgs are appended to the invocation. + ExtraArgs []string + // TablePrefix is prepended to each dataset's table name. + TablePrefix string + // MinBlock is the highest floor among the group's datasets, since one + // invocation cannot start at two different heights. + MinBlock uint64 + // Canary is collected and checked but not written, when the group does not + // already produce the dataset that serves as its coverage proof. + Canary *Dataset +} + +// newGroup resolves a group's configuration into its datasets. +// +// Every group collects the blocks dataset, because it is the only one whose row +// count is knowable in advance: exactly one per block, whatever the block +// contains. Every other dataset can legitimately be empty, and cryo reports a +// block it cannot serve by writing an empty parquet and exiting zero — which is +// indistinguishable from an empty block, and is how production came to hold +// whole ranges with no rows at all. A group that does not write blocks still +// collects it as a canary. +func newGroup(cfg *GroupConfig) (*Group, error) { + members, err := datasetsFor(cfg.Datatypes) + if err != nil { + return nil, err + } + + minBlock := cfg.MinBlock + + var canary *Dataset + + writesBlocks := false + + for _, ds := range members { + if ds.MinBlock > minBlock { + minBlock = ds.MinBlock + } + + if ds.Name == blocksDataset.Name { + writesBlocks = true + } + } + + datatypes := append([]string(nil), cfg.Datatypes...) + + if !writesBlocks { + canary = blocksDataset + datatypes = append(datatypes, blocksDataset.Name) + } + + return &Group{ + Name: cfg.Name, + Datatypes: datatypes, + Datasets: members, + ExtraArgs: append([]string(nil), cfg.ExtraArgs...), + TablePrefix: cfg.TablePrefix, + MinBlock: minBlock, + Canary: canary, + }, nil +} + +// RequiredColumns returns the columns cryo must be asked for, gathered from +// the group's datasets. These are not operator choices: a mapping that reads a +// column cryo did not emit fails on every block, and ClickHouse would fill the +// gap with a type default rather than complain. +func (g *Group) RequiredColumns() []string { + var cols []string + + for _, ds := range g.collected() { + for _, col := range ds.RequiredColumns { + if !slices.Contains(cols, col) { + cols = append(cols, col) + } + } + } + + return cols +} + +// collected returns every dataset the invocation produces that is worth +// decoding, which is the written members plus any canary. +func (g *Group) collected() []*Dataset { + if g.Canary == nil { + return g.Datasets + } + + return append(append(make([]*Dataset, 0, len(g.Datasets)+1), g.Datasets...), g.Canary) +} + +// ProcessorName is the group's identity in queue names, metrics labels and the +// processor column of the block ledger. +func (g *Group) ProcessorName() string { + return fmt.Sprintf("cryo_%s", g.Name) +} + +// TableFor returns a dataset's target table under this group's prefix. +func (g *Group) TableFor(ds *Dataset) string { + return g.TablePrefix + ds.Table +} diff --git a/pkg/processor/cryo/handlers.go b/pkg/processor/cryo/handlers.go new file mode 100644 index 0000000..7684539 --- /dev/null +++ b/pkg/processor/cryo/handlers.go @@ -0,0 +1,196 @@ +package cryo + +import ( + "context" + "fmt" + "time" + + "github.com/hibiken/asynq" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + + "github.com/ethpandaops/execution-processor/pkg/common" + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// GetHandlers returns the task handlers for this processor. +func (p *Processor) GetHandlers() map[string]asynq.HandlerFunc { + return map[string]asynq.HandlerFunc{ + p.processForwardsTaskType(): p.handleProcessTask, + p.processBackwardsTaskType(): p.handleProcessTask, + } +} + +// handleProcessTask collects one block for the whole group and writes every +// dataset it produced. The invocation succeeds or fails as a unit, so the block +// is only marked complete once every dataset has been flushed. +func (p *Processor) handleProcessTask(ctx context.Context, task *asynq.Task) error { + start := time.Now() + queue := p.queueFor(p.processingMode) + + defer func() { + common.TaskProcessingDuration.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), + ).Observe(time.Since(start).Seconds()) + }() + + var payload ProcessPayload + if err := payload.UnmarshalBinary(task.Payload()); err != nil { + common.TasksErrored.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), "unmarshal_error", + ).Inc() + + return fmt.Errorf("failed to unmarshal payload: %w", err) + } + + blockNumber := payload.BlockNumber.Uint64() + + if blockNumber < p.group.MinBlock { + p.log.WithFields(logrus.Fields{ + "block_number": blockNumber, + "min_block": p.group.MinBlock, + }).Debug("Block below the group's floor, marking complete without collecting") + + return p.completeBlock(ctx, blockNumber, payload.ProcessingMode) + } + + tables, err := p.fetch(ctx, blockNumber) + if err != nil { + common.TasksErrored.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), "fetch_error", + ).Inc() + + return err + } + + if coverageErr := verifyCoverage(p.group, tables, blockNumber, blockNumber); coverageErr != nil { + CryoFetchErrors.WithLabelValues(p.network.Name, p.name, "coverage").Inc() + common.TasksErrored.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), "coverage_error", + ).Inc() + + return fmt.Errorf("block %d: %w", blockNumber, coverageErr) + } + + rowCount, err := p.consume(ctx, tables) + if err != nil { + common.TasksErrored.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), "insert_error", + ).Inc() + + return err + } + + common.TasksProcessed.WithLabelValues( + p.network.Name, p.name, queue, task.Type(), "success", + ).Inc() + + if err := p.completeBlock(ctx, blockNumber, payload.ProcessingMode); err != nil { + return err + } + + p.log.WithFields(logrus.Fields{ + "block_number": blockNumber, + "rows": rowCount, + "duration": time.Since(start), + }).Info("Processed block") + + return nil +} + +// fetch runs cryo for one block and returns the decoded tables per dataset. +func (p *Processor) fetch(ctx context.Context, blockNumber uint64) (map[string]*decode.Table, error) { + start := time.Now() + + tables, err := p.fetcher.Fetch(ctx, p.group, blockNumber, blockNumber) + + CryoFetchDuration.WithLabelValues(p.network.Name, p.name).Observe(time.Since(start).Seconds()) + + if err != nil { + CryoFetchErrors.WithLabelValues(p.network.Name, p.name, "collect").Inc() + + return nil, fmt.Errorf("collect block %d: %w", blockNumber, err) + } + + return tables, nil +} + +// consume maps every dataset's table into its row buffer. +// +// The datasets are submitted concurrently because each Submit blocks until its +// own buffer flushes. Sequentially, a block would wait one flush interval per +// dataset — sixteen of them, dwarfing the cryo invocation itself. +func (p *Processor) consume(ctx context.Context, tables map[string]*decode.Table) (int, error) { + meta := rowMeta{ + network: p.network.Name, + updated: time.Now().UTC().Truncate(time.Second), + } + + start := time.Now() + + for _, s := range p.sinks { + if _, ok := tables[s.Dataset()]; !ok { + return 0, fmt.Errorf("cryo produced no output for dataset %s", s.Dataset()) + } + } + + var ( + group errgroup.Group + counts = make([]int, len(p.sinks)) + ) + + for i, s := range p.sinks { + group.Go(func() error { + rows, err := s.Consume(ctx, tables[s.Dataset()], meta) + if err != nil { + return err + } + + counts[i] = rows + + return nil + }) + } + + if err := group.Wait(); err != nil { + return 0, err + } + + var total int + + for i, s := range p.sinks { + if counts[i] > 0 { + CryoRowsDecoded.WithLabelValues(p.network.Name, p.name, s.Dataset()).Add(float64(counts[i])) + } + + total += counts[i] + } + + CryoSubmitDuration.WithLabelValues(p.network.Name, p.name).Observe(time.Since(start).Seconds()) + + return total, nil +} + +// completeBlock records the single task this block had as done. +func (p *Processor) completeBlock(ctx context.Context, blockNumber uint64, mode string) error { + taskID := GenerateTaskID(p.name, p.network.Name, blockNumber) + + allComplete, err := p.completionTracker.TrackTaskCompletion(ctx, taskID, blockNumber, p.network.Name, p.name, mode) + if err != nil { + // Non-fatal: stale detection will pick the block up. + p.log.WithError(err).WithFields(logrus.Fields{ + "block_number": blockNumber, + "task_id": taskID, + }).Warn("Failed to track task completion") + } + + if !allComplete { + return nil + } + + if err := p.completionTracker.MarkBlockComplete(ctx, blockNumber, p.network.Name, p.name, mode); err != nil { + return fmt.Errorf("mark block %d complete: %w", blockNumber, err) + } + + return nil +} diff --git a/pkg/processor/cryo/integration_test.go b/pkg/processor/cryo/integration_test.go new file mode 100644 index 0000000..ea414f3 --- /dev/null +++ b/pkg/processor/cryo/integration_test.go @@ -0,0 +1,169 @@ +package cryo + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/proto" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + "github.com/ethpandaops/execution-processor/pkg/clickhouse" +) + +// integrationDSN names a ClickHouse holding the tables in testdata/schema.sql. +// The insert path cannot be proved without one: UInt256 limb order, +// FixedString padding and LowCardinality encoding are all wire-format +// properties that only a real server validates. +const integrationDSNEnv = "CRYO_TEST_CLICKHOUSE" + +func integrationClient(t *testing.T) clickhouse.ClientInterface { + t.Helper() + + addr := os.Getenv(integrationDSNEnv) + if addr == "" { + t.Skipf("set %s to addr/database (e.g. localhost:9000/cryo_e2e) to run", integrationDSNEnv) + } + + host, database := addr, "default" + for i := range len(addr) { + if addr[i] == '/' { + host, database = addr[:i], addr[i+1:] + + break + } + } + + cfg := &clickhouse.Config{Addr: host, Database: database, Processor: "cryo_test", Network: "mainnet"} + cfg.SetDefaults() + + client, err := clickhouse.New(cfg) + require.NoError(t, err) + require.NoError(t, client.Start()) + + t.Cleanup(func() { _ = client.Stop() }) + + return client +} + +// TestInsertEveryDataset runs every dataset's real column mapping into a real +// ClickHouse and reads the row count back, which is the only way to catch a +// column block that encodes to something the server rejects. +func TestInsertEveryDataset(t *testing.T) { + client := integrationClient(t) + ctx := t.Context() + + log := logrus.New() + log.SetLevel(logrus.ErrorLevel) + + for _, ds := range datasets { + t.Run(ds.Name, func(t *testing.T) { + s := ds.newSink(sinkDeps{ + log: log, + clickhouse: client, + dataset: ds, + table: ds.Table, + network: "mainnet", + processor: "cryo_test", + bufferMaxRows: 1, + bufferFlushInterval: time.Hour, + }) + + require.NoError(t, s.ValidateSchema(ctx), "startup schema check must accept the real table") + + require.NoError(t, s.Start(ctx)) + t.Cleanup(func() { _ = s.Stop(ctx) }) + + var expected int + + for _, block := range []string{"b23000000", "b23000026", "b15000051"} { + table := fixtureTable(t, block, ds.Name) + + rows, err := s.Consume(ctx, table, fixtureMeta) + require.NoError(t, err) + require.Equal(t, table.Rows(), rows, "every parquet row must reach the buffer") + + expected += rows + } + + require.Equal(t, expected, countRows(t, client, ds.Table)) + }) + } +} + +// TestUint256RoundTrip proves the limb order of the UInt256 encoding against +// ClickHouse itself rather than against the code that produced it. +func TestUint256RoundTrip(t *testing.T) { + client := integrationClient(t) + ctx := t.Context() + + values := []string{ + "0", + "1", + "18446744073709551615", + "18446744073709551616", + "82520860000000000000", + "2227376088180124203790135", + "340282366920938463463374607431768211455", + "340282366920938463463374607431768211456", + "90200608809960061854851247237396347024452973173411525954377011279129548947580", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + } + + var col proto.ColUInt256 + + for _, v := range values { + parsed, err := uint256(v) + require.NoError(t, err, "value %s", v) + + col.Append(parsed) + } + + table := "cryo_uint256_roundtrip" + require.NoError(t, client.Execute(ctx, "DROP TABLE IF EXISTS "+table)) + require.NoError(t, client.Execute(ctx, "CREATE TABLE "+table+" (v UInt256) ENGINE = Memory")) + + t.Cleanup(func() { _ = client.Execute(context.Background(), "DROP TABLE IF EXISTS "+table) }) + + input := proto.Input{{Name: "v", Data: &col}} + require.NoError(t, client.Do(ctx, ch.Query{Body: input.Into(table), Input: input})) + + var readBack proto.ColStr + + got := make([]string, 0, len(values)) + + require.NoError(t, client.Do(ctx, ch.Query{ + Body: "SELECT toString(v) AS v FROM " + table + " ORDER BY v", + Result: proto.Results{{Name: "v", Data: &readBack}}, + OnResult: func(_ context.Context, _ proto.Block) error { + for i := range readBack.Rows() { + got = append(got, readBack.Row(i)) + } + + return nil + }, + })) + + require.ElementsMatch(t, values, got) +} + +// countRows counts only the rows this test wrote, identified by the fixed +// fixture timestamp. Counting the whole table would break against leftovers +// from an earlier run and against any processor writing concurrently, and +// truncating a shared database to avoid that is worse than either. +func countRows(t *testing.T, client clickhouse.ClientInterface, table string) int { + t.Helper() + + query := fmt.Sprintf("SELECT count() AS c FROM %s WHERE updated_date_time = '%s'", + table, fixtureMeta.updated.Format("2006-01-02 15:04:05")) + + count, err := client.QueryUInt64(t.Context(), query, "c") + require.NoError(t, err) + require.NotNil(t, count) + + return int(*count) +} diff --git a/pkg/processor/cryo/memory_test.go b/pkg/processor/cryo/memory_test.go new file mode 100644 index 0000000..5c36157 --- /dev/null +++ b/pkg/processor/cryo/memory_test.go @@ -0,0 +1,87 @@ +package cryo + +import ( + "runtime" + "testing" + + "golang.org/x/sync/errgroup" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// TestPeakHeapUnderConcurrency reports the resident cost of processing blocks +// in parallel, which is what sizes a replica. Allocation churn is not the same +// number: most of a block's allocation is released as soon as its columns are +// handed to ClickHouse. +// +// It reports rather than asserts, because the figure depends on the machine and +// a threshold here would be noise. Run with -v to see it. +func TestPeakHeapUnderConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("allocates hundreds of megabytes") + } + + paths := make([]string, 0, len(datasets)) + sinks := make([]benchSink, 0, len(datasets)) + + for _, ds := range datasets { + paths = append(paths, benchFixturePath(t, ds.Name)) + sinks = append(sinks, benchSinkFor(ds)) + } + + for _, workers := range []int{1, 4, 8, 16, 32} { + runtime.GC() + + var before runtime.MemStats + + runtime.ReadMemStats(&before) + + var group errgroup.Group + + group.SetLimit(workers) + + // Each unit of work is one whole block: decode all sixteen parquet + // files and build every column block, exactly as a task does. + for range workers * 4 { + group.Go(func() error { + for i, path := range paths { + table, err := decode.File(path) + if err != nil { + return err + } + + if err := sinks[i].encodeOnly(table, fixtureMeta); err != nil { + return err + } + } + + return nil + }) + } + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + + var peak runtime.MemStats + + runtime.ReadMemStats(&peak) + + // HeapAlloc straight after the workload counts garbage that simply has + // not been collected yet. Collecting first is what distinguishes the + // memory a replica actually needs from the memory it churned through. + runtime.GC() + + var live runtime.MemStats + + runtime.ReadMemStats(&live) + + t.Logf("workers=%-3d heap_sys=%6.1f MiB live_after_gc=%6.1f MiB churned=%6.1f MiB gc_cycles=%d", + workers, + float64(peak.HeapSys)/(1<<20), + float64(live.HeapAlloc)/(1<<20), + float64(peak.TotalAlloc-before.TotalAlloc)/(1<<20), + peak.NumGC-before.NumGC, + ) + } +} diff --git a/pkg/processor/cryo/metrics.go b/pkg/processor/cryo/metrics.go new file mode 100644 index 0000000..1181fc6 --- /dev/null +++ b/pkg/processor/cryo/metrics.go @@ -0,0 +1,49 @@ +package cryo + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // CryoBuildInfo exposes which cryo produced the data, which is the first + // thing you need when the output looks wrong. + CryoBuildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_cryo_build_info", + Help: "Version of the cryo binary the processor invokes", + }, []string{"network", "version"}) + + // CryoFetchDuration measures one cryo invocation. + CryoFetchDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "execution_processor_cryo_fetch_duration_seconds", + Help: "Time taken by one cryo invocation, covering all datasets in a group", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 14), + }, []string{"network", "processor"}) + + // CryoDecodeDuration measures reading one group's parquet output into memory. + CryoDecodeDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "execution_processor_cryo_decode_duration_seconds", + Help: "Time taken to decode one group's parquet output", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), + }, []string{"network", "processor"}) + + // CryoSubmitDuration measures mapping rows and waiting for every dataset's + // buffer to flush, which is where ClickHouse backpressure shows up. + CryoSubmitDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "execution_processor_cryo_submit_duration_seconds", + Help: "Time taken to map and flush one group's rows to ClickHouse", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 16), + }, []string{"network", "processor"}) + + // CryoRowsDecoded counts rows produced per dataset. + CryoRowsDecoded = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_cryo_rows_decoded_total", + Help: "Rows decoded from cryo output", + }, []string{"network", "processor", "dataset"}) + + // CryoFetchErrors counts failed invocations by cause. + CryoFetchErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_cryo_fetch_errors_total", + Help: "Failed cryo invocations", + }, []string{"network", "processor", "reason"}) +) diff --git a/pkg/processor/cryo/processor.go b/pkg/processor/cryo/processor.go new file mode 100644 index 0000000..879f290 --- /dev/null +++ b/pkg/processor/cryo/processor.go @@ -0,0 +1,305 @@ +package cryo + +import ( + "context" + "fmt" + "math" + + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/execution-processor/pkg/clickhouse" + "github.com/ethpandaops/execution-processor/pkg/ethereum" + "github.com/ethpandaops/execution-processor/pkg/processor/tracker" + "github.com/ethpandaops/execution-processor/pkg/state" +) + +// Compile-time interface compliance check. +var _ tracker.BlockProcessor = (*Processor)(nil) + +// Dependencies contains the dependencies needed for the processor. +type Dependencies struct { + Log logrus.FieldLogger + Pool *ethereum.Pool + Network *ethereum.Network + State *state.Manager + AsynqClient *asynq.Client + AsynqInspector *asynq.Inspector + RedisClient *redis.Client + RedisPrefix string +} + +// Processor collects one cryo group per block and writes each dataset it +// produces to its own table. +type Processor struct { + log logrus.FieldLogger + name string + group *Group + pool *ethereum.Pool + stateManager *state.Manager + clickhouse clickhouse.ClientInterface + config *Config + network *ethereum.Network + asynqClient *asynq.Client + asynqInspector *asynq.Inspector + processingMode string + redisPrefix string + + fetcher Fetcher + + // One sink per dataset: a single invocation writes several tables whose + // volumes differ by orders of magnitude, so they flush independently. + sinks []sink + + *tracker.Limiter + + completionTracker *tracker.BlockCompletionTracker +} + +// New creates a processor for one enabled group. +func New(deps *Dependencies, config *Config, groupCfg *GroupConfig) (*Processor, error) { + group, err := newGroup(groupCfg) + if err != nil { + return nil, fmt.Errorf("invalid group %q: %w", groupCfg.Name, err) + } + + name := group.ProcessorName() + + clickhouseConfig := config.Config + clickhouseConfig.Network = deps.Network.Name + clickhouseConfig.Processor = name + + clickhouseClient, err := clickhouse.New(&clickhouseConfig) + if err != nil { + return nil, fmt.Errorf("failed to create clickhouse client: %w", err) + } + + maxPending := groupCfg.MaxPendingBlockRange + if maxPending <= 0 { + maxPending = tracker.DefaultMaxPendingBlockRange + } + + log := deps.Log.WithField("processor", name) + + limiter := tracker.NewLimiter( + &tracker.LimiterDeps{ + Log: log, + StateProvider: deps.State, + Network: deps.Network.Name, + Processor: name, + }, + tracker.LimiterConfig{ + MaxPendingBlockRange: maxPending, + }, + ) + + completionTracker := tracker.NewBlockCompletionTracker( + deps.RedisClient, + deps.RedisPrefix, + log, + deps.State, + tracker.BlockCompletionTrackerConfig{ + StaleThreshold: tracker.DefaultStaleThreshold, + AutoRetryStale: true, + }, + ) + + p := &Processor{ + log: log, + name: name, + group: group, + pool: deps.Pool, + stateManager: deps.State, + clickhouse: clickhouseClient, + config: config, + network: deps.Network, + asynqClient: deps.AsynqClient, + asynqInspector: deps.AsynqInspector, + processingMode: tracker.FORWARDS_MODE, + redisPrefix: deps.RedisPrefix, + Limiter: limiter, + completionTracker: completionTracker, + } + + p.fetcher = newExecFetcher(log, config, p.rpcEndpoint) + + p.sinks = make([]sink, 0, len(group.Datasets)) + for _, ds := range group.Datasets { + p.sinks = append(p.sinks, ds.newSink(sinkDeps{ + log: log, + clickhouse: clickhouseClient, + dataset: ds, + table: group.TableFor(ds), + network: deps.Network.Name, + processor: name, + bufferMaxRows: config.BufferMaxRows, + bufferFlushInterval: config.BufferFlushInterval, + })) + } + + log.WithFields(logrus.Fields{ + "network": deps.Network.Name, + "datatypes": group.Datatypes, + "datasets": len(group.Datasets), + "min_block": group.MinBlock, + "max_pending_block_range": maxPending, + }).Info("Cryo processor initialized") + + return p, nil +} + +// Name returns the processor name, which is the checkpoint key in the block +// ledger and must stay stable for the lifetime of the data. +func (p *Processor) Name() string { return p.name } + +// Group returns the group this processor collects. +func (p *Processor) Group() *Group { return p.group } + +// rpcEndpoint picks a healthy node for cryo to dial. Nodes that expose no +// endpoint cannot drive a subprocess, so they are not candidates. +func (p *Processor) rpcEndpoint() (string, error) { + for _, node := range p.pool.GetHealthyExecutionNodes() { + if endpoint := node.RPCEndpoint(); endpoint != "" { + return endpoint, nil + } + } + + return "", fmt.Errorf("no healthy execution node exposes an RPC endpoint for cryo") +} + +// Start starts the processor, verifying the cryo binary and every target table +// before accepting work. +func (p *Processor) Start(ctx context.Context) error { + p.log.Info("Starting cryo processor") + + sweepTempDirs(p.log, p.config.TempDir) + + if err := p.clickhouse.Start(); err != nil { + return fmt.Errorf("failed to start ClickHouse client: %w", err) + } + + version, err := p.cryoVersion(ctx) + if err != nil { + return err + } + + for _, s := range p.sinks { + if err := s.ValidateSchema(ctx); err != nil { + return fmt.Errorf("schema check failed: %w", err) + } + + if err := s.Start(ctx); err != nil { + return fmt.Errorf("failed to start %s row buffer: %w", s.Dataset(), err) + } + } + + p.log.WithFields(logrus.Fields{ + "network": p.network.Name, + "cryo_version": version, + }).Info("Cryo processor ready") + + return nil +} + +// cryoVersion resolves the bundled cryo's version, failing startup if the +// binary is missing or unrunnable rather than at the first task. +func (p *Processor) cryoVersion(ctx context.Context) (string, error) { + versioner, ok := p.fetcher.(interface { + Version(context.Context) (string, error) + }) + if !ok { + return "", nil + } + + version, err := versioner.Version(ctx) + if err != nil { + return "", fmt.Errorf("cryo binary %q is not runnable: %w", p.config.BinaryPath, err) + } + + CryoBuildInfo.WithLabelValues(p.network.Name, version).Set(1) + + return version, nil +} + +// Stop stops the processor, flushing every dataset's buffered rows. +func (p *Processor) Stop(ctx context.Context) error { + p.log.Info("Stopping cryo processor") + + for _, s := range p.sinks { + if err := s.Stop(ctx); err != nil { + p.log.WithError(err).WithField("dataset", s.Dataset()).Error("Failed to stop row buffer") + } + } + + return p.clickhouse.Stop() +} + +// SetProcessingMode sets the processing mode for the processor. +func (p *Processor) SetProcessingMode(mode string) { + p.processingMode = mode + p.log.WithField("mode", mode).Info("Processing mode updated") +} + +// GetCompletionTracker returns the block completion tracker. +func (p *Processor) GetCompletionTracker() *tracker.BlockCompletionTracker { + return p.completionTracker +} + +// GetLimiter returns the block completion limiter. +func (p *Processor) GetLimiter() *tracker.Limiter { + return p.Limiter +} + +// EnqueueTask enqueues a task to the specified queue with infinite retries. +func (p *Processor) EnqueueTask(ctx context.Context, task *asynq.Task, opts ...asynq.Option) error { + opts = append(opts, asynq.MaxRetry(math.MaxInt32)) + + _, err := p.asynqClient.EnqueueContext(ctx, task, opts...) + + return err +} + +// GetQueues returns the queues used by this processor. +func (p *Processor) GetQueues() []tracker.QueueInfo { + return []tracker.QueueInfo{ + {Name: p.getProcessReprocessForwardsQueue(), Priority: 20}, + {Name: p.getProcessReprocessBackwardsQueue(), Priority: 15}, + {Name: p.getProcessForwardsQueue(), Priority: 10}, + {Name: p.getProcessBackwardsQueue(), Priority: 5}, + } +} + +func (p *Processor) getProcessForwardsQueue() string { + return tracker.PrefixedProcessForwardsQueue(p.name, p.redisPrefix) +} + +func (p *Processor) getProcessBackwardsQueue() string { + return tracker.PrefixedProcessBackwardsQueue(p.name, p.redisPrefix) +} + +func (p *Processor) getProcessReprocessForwardsQueue() string { + return tracker.PrefixedProcessReprocessForwardsQueue(p.name, p.redisPrefix) +} + +func (p *Processor) getProcessReprocessBackwardsQueue() string { + return tracker.PrefixedProcessReprocessBackwardsQueue(p.name, p.redisPrefix) +} + +// queueFor returns the main queue for a processing mode. +func (p *Processor) queueFor(mode string) string { + if mode == tracker.BACKWARDS_MODE { + return p.getProcessBackwardsQueue() + } + + return p.getProcessForwardsQueue() +} + +// reprocessQueueFor returns the high-priority queue for a processing mode. +func (p *Processor) reprocessQueueFor(mode string) string { + if mode == tracker.BACKWARDS_MODE { + return p.getProcessReprocessBackwardsQueue() + } + + return p.getProcessReprocessForwardsQueue() +} diff --git a/pkg/processor/cryo/schema.go b/pkg/processor/cryo/schema.go new file mode 100644 index 0000000..dbeead0 --- /dev/null +++ b/pkg/processor/cryo/schema.go @@ -0,0 +1,96 @@ +package cryo + +import ( + "context" + "fmt" + "strings" + + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/clickhouse" + "github.com/ethpandaops/execution-processor/pkg/processor/tracker" +) + +// describeTable returns the column name to ClickHouse type mapping of a table. +func describeTable(ctx context.Context, client clickhouse.ClientInterface, table string) (map[string]string, error) { + database, name, err := splitTableName(table) + if err != nil { + return nil, err + } + + queryCtx, cancel := context.WithTimeout(ctx, tracker.DefaultClickHouseTimeout) + defer cancel() + + var ( + names proto.ColStr + types proto.ColStr + out = make(map[string]string) + ) + + body := fmt.Sprintf( + "SELECT name, type FROM system.columns WHERE database = %s AND table = '%s'", + database, name, + ) + + err = client.Do(queryCtx, ch.Query{ + Body: body, + Result: proto.Results{ + {Name: "name", Data: &names}, + {Name: "type", Data: &types}, + }, + OnResult: func(_ context.Context, _ proto.Block) error { + for i := range names.Rows() { + out[names.Row(i)] = types.Row(i) + } + + return nil + }, + }) + if err != nil { + return nil, fmt.Errorf("describe table %s: %w", table, err) + } + + if len(out) == 0 { + return nil, fmt.Errorf("table %s does not exist or has no columns", table) + } + + return out, nil +} + +// splitTableName resolves a configured table into the database expression and +// table name to look up, rejecting anything that is not a plain identifier so +// the values can be interpolated into the lookup. +func splitTableName(table string) (database, name string, err error) { + database, name = "currentDatabase()", table + + if db, rest, found := strings.Cut(table, "."); found { + if !isIdentifier(db) { + return "", "", fmt.Errorf("table %q has an invalid database name", table) + } + + database, name = "'"+db+"'", rest + } + + if !isIdentifier(name) { + return "", "", fmt.Errorf("table %q is not a plain identifier", table) + } + + return database, name, nil +} + +func isIdentifier(s string) bool { + if s == "" { + return false + } + + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_': + default: + return false + } + } + + return true +} diff --git a/pkg/processor/cryo/sink.go b/pkg/processor/cryo/sink.go new file mode 100644 index 0000000..ed3c73f --- /dev/null +++ b/pkg/processor/cryo/sink.go @@ -0,0 +1,240 @@ +package cryo + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/proto" + "github.com/sirupsen/logrus" + + "github.com/ethpandaops/execution-processor/pkg/clickhouse" + "github.com/ethpandaops/execution-processor/pkg/common" + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" + "github.com/ethpandaops/execution-processor/pkg/processor/tracker" + "github.com/ethpandaops/execution-processor/pkg/rowbuffer" +) + +// rowMeta carries the per-batch literals every dataset appends to its rows. +type rowMeta struct { + network string + updated time.Time +} + +// sinkDeps are the shared dependencies a dataset needs to buffer and insert. +type sinkDeps struct { + log logrus.FieldLogger + clickhouse clickhouse.ClientInterface + dataset *Dataset + table string + network string + processor string + bufferMaxRows int + bufferFlushInterval time.Duration +} + +// sink owns one dataset's row buffer and ClickHouse mapping. A single cryo +// invocation produces rows for several datasets, so a processor holds one sink +// per dataset in its group and flushes each on its own thresholds. +type sink interface { + Dataset() string + Table() string + Start(ctx context.Context) error + Stop(ctx context.Context) error + // Consume decodes a parquet table and submits its rows for insertion, + // blocking until they are flushed. It returns the number of rows submitted. + Consume(ctx context.Context, t *decode.Table, meta rowMeta) (int, error) + // ValidateSchema checks the target table's columns against the mapping. + ValidateSchema(ctx context.Context) error +} + +// columnar is a dataset's ch-go column block. +type columnar[R any] interface { + Append(R) error + Input() proto.Input + Reset() + Rows() int +} + +// decodeFunc turns a decoded parquet table into rows, preserving file order. +type decodeFunc[R any] func(t *decode.Table, meta rowMeta) ([]R, error) + +// datasetSink is the single implementation of sink; datasets differ only in +// their row type, decoder and column block. +type datasetSink[R any] struct { + deps sinkDeps + log logrus.FieldLogger + buffer *rowbuffer.Buffer[R] + decode decodeFunc[R] + newCols func() columnar[R] + + // cols recycles column blocks between flushes. A block holds one buffer + // per column sized for the batch, so at these row counts rebuilding it + // every flush is the single largest allocation in the insert path. + cols sync.Pool +} + +// acquireCols returns a reset column block, reusing a retired one when there is +// one to hand. +func (s *datasetSink[R]) acquireCols() columnar[R] { + if v := s.cols.Get(); v != nil { + cols, ok := v.(columnar[R]) + if ok { + cols.Reset() + + return cols + } + } + + return s.newCols() +} + +// releaseCols retires a column block once ClickHouse has consumed it. +func (s *datasetSink[R]) releaseCols(cols columnar[R]) { + cols.Reset() + s.cols.Put(cols) +} + +func newDatasetSink[R any](deps sinkDeps, dec decodeFunc[R], newCols func() columnar[R]) sink { + log := deps.log.WithField("dataset", deps.dataset.Name) + + s := &datasetSink[R]{ + deps: deps, + log: log, + decode: dec, + newCols: newCols, + } + + s.buffer = rowbuffer.New( + rowbuffer.Config{ + MaxRows: deps.bufferMaxRows, + FlushInterval: deps.bufferFlushInterval, + Network: deps.network, + Processor: deps.processor, + Table: deps.table, + }, + s.flush, + log, + ) + + return s +} + +func (s *datasetSink[R]) Dataset() string { return s.deps.dataset.Name } + +func (s *datasetSink[R]) Table() string { return s.deps.table } + +func (s *datasetSink[R]) Start(ctx context.Context) error { + return s.buffer.Start(ctx) +} + +func (s *datasetSink[R]) Stop(ctx context.Context) error { + return s.buffer.Stop(ctx) +} + +func (s *datasetSink[R]) Consume(ctx context.Context, t *decode.Table, meta rowMeta) (int, error) { + rows, err := s.decode(t, meta) + if err != nil { + return 0, fmt.Errorf("decode %s: %w", s.deps.dataset.Name, err) + } + + if len(rows) == 0 { + return 0, nil + } + + if err := s.buffer.Submit(ctx, rows); err != nil { + return 0, fmt.Errorf("buffer %s: %w", s.deps.dataset.Name, err) + } + + return len(rows), nil +} + +// flush maps buffered rows into a ch-go column block and inserts it. +func (s *datasetSink[R]) flush(ctx context.Context, rows []R) error { + if len(rows) == 0 { + return nil + } + + insertCtx, cancel := context.WithTimeout(ctx, tracker.DefaultClickHouseTimeout) + defer cancel() + + cols := s.acquireCols() + defer s.releaseCols(cols) + + for i := range rows { + if err := cols.Append(rows[i]); err != nil { + return fmt.Errorf("append %s row %d: %w", s.deps.dataset.Name, i, err) + } + } + + input := cols.Input() + + if err := s.deps.clickhouse.Do(insertCtx, ch.Query{ + Body: input.Into(s.deps.table), + Input: input, + }); err != nil { + common.ClickHouseInsertsRows.WithLabelValues( + s.deps.network, s.deps.processor, s.deps.table, "failed", "", + ).Add(float64(len(rows))) + + return fmt.Errorf("insert into %s: %w", s.deps.table, err) + } + + common.ClickHouseInsertsRows.WithLabelValues( + s.deps.network, s.deps.processor, s.deps.table, "success", "", + ).Add(float64(len(rows))) + + return nil +} + +// ValidateSchema compares the target table against the column block this +// dataset writes, so that schema drift fails at boot rather than at the first +// insert of a block that happens to exercise the drifted column. +// +// The check runs in both directions. A column the mapping does not write is as +// dangerous as one it writes wrongly: ClickHouse fills the gap with the type +// default and the insert succeeds, so the column silently becomes zero for +// every row. +func (s *datasetSink[R]) ValidateSchema(ctx context.Context) error { + actual, err := describeTable(ctx, s.deps.clickhouse, s.deps.table) + if err != nil { + return err + } + + written := make(map[string]struct{}, len(actual)) + + for _, col := range s.newCols().Input() { + written[col.Name] = struct{}{} + + got, ok := actual[col.Name] + if !ok { + return fmt.Errorf("table %s has no column %q", s.deps.table, col.Name) + } + + want := string(col.Data.Type()) + if got != want { + return fmt.Errorf("table %s column %q is %s, mapping writes %s", s.deps.table, col.Name, got, want) + } + } + + missing := make([]string, 0, len(actual)) + + for name := range actual { + if _, ok := written[name]; !ok { + missing = append(missing, name) + } + } + + if len(missing) > 0 { + sort.Strings(missing) + + return fmt.Errorf("table %s has columns the %s mapping never writes, which would be silently defaulted: %s", + s.deps.table, s.deps.dataset.Name, strings.Join(missing, ", ")) + } + + return nil +} diff --git a/pkg/processor/cryo/tasks.go b/pkg/processor/cryo/tasks.go new file mode 100644 index 0000000..f0d1e6d --- /dev/null +++ b/pkg/processor/cryo/tasks.go @@ -0,0 +1,63 @@ +package cryo + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/hibiken/asynq" + + "github.com/ethpandaops/execution-processor/pkg/processor/tracker" +) + +// ProcessPayload represents the payload for processing one block of one group. +// +//nolint:tagliatelle // snake_case matches the queued payloads of the other processors +type ProcessPayload struct { + BlockNumber big.Int `json:"block_number"` + NetworkName string `json:"network_name"` + ProcessingMode string `json:"processing_mode"` +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (p *ProcessPayload) MarshalBinary() ([]byte, error) { + return json.Marshal(p) +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (p *ProcessPayload) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, p) +} + +// Task types are derived from the group rather than fixed at package level, +// because one config yields one processor per enabled group. +func (p *Processor) processForwardsTaskType() string { + return p.name + "_process_forwards" +} + +func (p *Processor) processBackwardsTaskType() string { + return p.name + "_process_backwards" +} + +// GenerateTaskID creates a deterministic task ID for deduplication. +func GenerateTaskID(processorName, network string, blockNumber uint64) string { + return fmt.Sprintf("%s:%s:%d:block", processorName, network, blockNumber) +} + +// newTask builds the task for a processing mode, returning it with the ID used +// for deduplication. +func (p *Processor) newTask(payload *ProcessPayload, mode string) (*asynq.Task, string, error) { + payload.ProcessingMode = mode + + taskType := p.processForwardsTaskType() + if mode == tracker.BACKWARDS_MODE { + taskType = p.processBackwardsTaskType() + } + + data, err := payload.MarshalBinary() + if err != nil { + return nil, "", fmt.Errorf("marshal payload: %w", err) + } + + return asynq.NewTask(taskType, data), GenerateTaskID(p.name, payload.NetworkName, payload.BlockNumber.Uint64()), nil +} diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__address_appearances__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__address_appearances__15000051_to_15000051.parquet new file mode 100644 index 0000000..74f852e Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__address_appearances__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__balance_diffs__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__balance_diffs__15000051_to_15000051.parquet new file mode 100644 index 0000000..6431997 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__balance_diffs__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__balance_reads__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__balance_reads__15000051_to_15000051.parquet new file mode 100644 index 0000000..5a830d3 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__balance_reads__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__blocks__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__blocks__15000051_to_15000051.parquet new file mode 100644 index 0000000..55d68ab Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__blocks__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__code_diffs__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__code_diffs__15000051_to_15000051.parquet new file mode 100644 index 0000000..9d3139d Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__code_diffs__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__contracts__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__contracts__15000051_to_15000051.parquet new file mode 100644 index 0000000..5a669d7 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__contracts__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__erc20_transfers__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__erc20_transfers__15000051_to_15000051.parquet new file mode 100644 index 0000000..7dbafa8 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__erc20_transfers__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__erc721_transfers__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__erc721_transfers__15000051_to_15000051.parquet new file mode 100644 index 0000000..e9d29c3 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__erc721_transfers__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__four_byte_counts__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__four_byte_counts__15000051_to_15000051.parquet new file mode 100644 index 0000000..46ca267 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__four_byte_counts__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__logs__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__logs__15000051_to_15000051.parquet new file mode 100644 index 0000000..a06c4dd Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__logs__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__native_transfers__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__native_transfers__15000051_to_15000051.parquet new file mode 100644 index 0000000..c493e58 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__native_transfers__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_diffs__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_diffs__15000051_to_15000051.parquet new file mode 100644 index 0000000..1ad3963 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_diffs__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_reads__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_reads__15000051_to_15000051.parquet new file mode 100644 index 0000000..90ef048 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__nonce_reads__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__storage_diffs__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__storage_diffs__15000051_to_15000051.parquet new file mode 100644 index 0000000..094a139 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__storage_diffs__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__storage_reads__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__storage_reads__15000051_to_15000051.parquet new file mode 100644 index 0000000..299f6bc Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__storage_reads__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__traces__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__traces__15000051_to_15000051.parquet new file mode 100644 index 0000000..2433886 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__traces__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b15000051/ethereum__transactions__15000051_to_15000051.parquet b/pkg/processor/cryo/testdata/b15000051/ethereum__transactions__15000051_to_15000051.parquet new file mode 100644 index 0000000..4fd3876 Binary files /dev/null and b/pkg/processor/cryo/testdata/b15000051/ethereum__transactions__15000051_to_15000051.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__address_appearances__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__address_appearances__23000000_to_23000000.parquet new file mode 100644 index 0000000..3f46d10 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__address_appearances__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__balance_diffs__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__balance_diffs__23000000_to_23000000.parquet new file mode 100644 index 0000000..9ab7428 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__balance_diffs__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__balance_reads__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__balance_reads__23000000_to_23000000.parquet new file mode 100644 index 0000000..e323c69 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__balance_reads__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__blocks__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__blocks__23000000_to_23000000.parquet new file mode 100644 index 0000000..c03b9df Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__blocks__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__code_diffs__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__code_diffs__23000000_to_23000000.parquet new file mode 100644 index 0000000..9d3139d Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__code_diffs__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__contracts__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__contracts__23000000_to_23000000.parquet new file mode 100644 index 0000000..1a1bced Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__contracts__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__erc20_transfers__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__erc20_transfers__23000000_to_23000000.parquet new file mode 100644 index 0000000..9717328 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__erc20_transfers__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__erc721_transfers__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__erc721_transfers__23000000_to_23000000.parquet new file mode 100644 index 0000000..63aede2 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__erc721_transfers__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__four_byte_counts__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__four_byte_counts__23000000_to_23000000.parquet new file mode 100644 index 0000000..39af25f Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__four_byte_counts__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__logs__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__logs__23000000_to_23000000.parquet new file mode 100644 index 0000000..20fd0da Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__logs__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__native_transfers__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__native_transfers__23000000_to_23000000.parquet new file mode 100644 index 0000000..5cabbb7 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__native_transfers__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_diffs__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_diffs__23000000_to_23000000.parquet new file mode 100644 index 0000000..878a1a3 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_diffs__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_reads__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_reads__23000000_to_23000000.parquet new file mode 100644 index 0000000..c3d5de2 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__nonce_reads__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__storage_diffs__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__storage_diffs__23000000_to_23000000.parquet new file mode 100644 index 0000000..211439c Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__storage_diffs__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__storage_reads__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__storage_reads__23000000_to_23000000.parquet new file mode 100644 index 0000000..5ce2f18 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__storage_reads__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__traces__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__traces__23000000_to_23000000.parquet new file mode 100644 index 0000000..9a2fdfa Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__traces__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000000/ethereum__transactions__23000000_to_23000000.parquet b/pkg/processor/cryo/testdata/b23000000/ethereum__transactions__23000000_to_23000000.parquet new file mode 100644 index 0000000..5ca9bc1 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000000/ethereum__transactions__23000000_to_23000000.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__address_appearances__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__address_appearances__23000026_to_23000026.parquet new file mode 100644 index 0000000..1f51742 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__address_appearances__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__balance_diffs__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__balance_diffs__23000026_to_23000026.parquet new file mode 100644 index 0000000..3a5ffc2 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__balance_diffs__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__balance_reads__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__balance_reads__23000026_to_23000026.parquet new file mode 100644 index 0000000..8a47c83 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__balance_reads__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__blocks__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__blocks__23000026_to_23000026.parquet new file mode 100644 index 0000000..3ff9375 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__blocks__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__code_diffs__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__code_diffs__23000026_to_23000026.parquet new file mode 100644 index 0000000..ab1eb67 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__code_diffs__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__contracts__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__contracts__23000026_to_23000026.parquet new file mode 100644 index 0000000..a4eef9b Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__contracts__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__erc20_transfers__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__erc20_transfers__23000026_to_23000026.parquet new file mode 100644 index 0000000..b9a4166 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__erc20_transfers__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__erc721_transfers__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__erc721_transfers__23000026_to_23000026.parquet new file mode 100644 index 0000000..ca211e1 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__erc721_transfers__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__four_byte_counts__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__four_byte_counts__23000026_to_23000026.parquet new file mode 100644 index 0000000..7413d44 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__four_byte_counts__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__logs__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__logs__23000026_to_23000026.parquet new file mode 100644 index 0000000..4db0d6c Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__logs__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__native_transfers__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__native_transfers__23000026_to_23000026.parquet new file mode 100644 index 0000000..34ca977 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__native_transfers__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_diffs__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_diffs__23000026_to_23000026.parquet new file mode 100644 index 0000000..99f90ce Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_diffs__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_reads__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_reads__23000026_to_23000026.parquet new file mode 100644 index 0000000..f105670 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__nonce_reads__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__storage_diffs__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__storage_diffs__23000026_to_23000026.parquet new file mode 100644 index 0000000..84e7d54 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__storage_diffs__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__storage_reads__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__storage_reads__23000026_to_23000026.parquet new file mode 100644 index 0000000..4393cca Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__storage_reads__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__traces__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__traces__23000026_to_23000026.parquet new file mode 100644 index 0000000..71e6ddc Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__traces__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/b23000026/ethereum__transactions__23000026_to_23000026.parquet b/pkg/processor/cryo/testdata/b23000026/ethereum__transactions__23000026_to_23000026.parquet new file mode 100644 index 0000000..51c2e02 Binary files /dev/null and b/pkg/processor/cryo/testdata/b23000026/ethereum__transactions__23000026_to_23000026.parquet differ diff --git a/pkg/processor/cryo/testdata/schema.sql b/pkg/processor/cryo/testdata/schema.sql new file mode 100644 index 0000000..06586e4 --- /dev/null +++ b/pkg/processor/cryo/testdata/schema.sql @@ -0,0 +1,316 @@ +-- The 16 canonical_execution_* tables the cryo processors write. +-- +-- Single-node form: no ON CLUSTER, no Replicated engine, no Distributed +-- wrappers. Column types, CODECs, PARTITION BY and ORDER BY match the cluster +-- definitions exactly, which is what the tests and the startup schema check +-- assert against. + +CREATE TABLE IF NOT EXISTS canonical_execution_block +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_date_time` DateTime64(3) COMMENT 'The block timestamp' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `block_hash` FixedString(66) COMMENT 'The block hash' CODEC(ZSTD(1)), + `author` Nullable(String) COMMENT 'The block author' CODEC(ZSTD(1)), + `gas_used` Nullable(UInt64) COMMENT 'The block gas used' CODEC(DoubleDelta, ZSTD(1)), + `gas_limit` UInt64 COMMENT 'The block gas limit' CODEC(DoubleDelta, ZSTD(1)), + `extra_data` Nullable(String) COMMENT 'The block extra data in hex' CODEC(ZSTD(1)), + `extra_data_string` Nullable(String) COMMENT 'The block extra data in UTF-8 string' CODEC(ZSTD(1)), + `base_fee_per_gas` Nullable(UInt64) COMMENT 'The block base fee per gas' CODEC(DoubleDelta, ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number) +COMMENT 'Contains canonical execution block data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_transaction +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `nonce` UInt64 COMMENT 'The transaction nonce' CODEC(ZSTD(1)), + `from_address` String COMMENT 'The transaction from address' CODEC(ZSTD(1)), + `to_address` Nullable(String) COMMENT 'The transaction to address' CODEC(ZSTD(1)), + `value` UInt256 COMMENT 'The transaction value in float64' CODEC(ZSTD(1)), + `input` Nullable(String) COMMENT 'The transaction input in hex' CODEC(ZSTD(1)), + `gas_limit` UInt64 COMMENT 'The transaction gas limit' CODEC(ZSTD(1)), + `gas_used` UInt64 COMMENT 'The transaction gas used' CODEC(ZSTD(1)), + `gas_price` UInt128 COMMENT 'The transaction gas price' CODEC(ZSTD(1)), + `transaction_type` UInt8 COMMENT 'The transaction type' CODEC(ZSTD(1)), + `max_priority_fee_per_gas` Nullable(UInt64) COMMENT 'The transaction max priority fee per gas' CODEC(ZSTD(1)), + `max_fee_per_gas` Nullable(UInt64) COMMENT 'The transaction max fee per gas' CODEC(ZSTD(1)), + `success` Bool COMMENT 'The transaction success' CODEC(ZSTD(1)), + `n_input_bytes` UInt32 COMMENT 'The transaction input bytes' CODEC(ZSTD(1)), + `n_input_zero_bytes` UInt32 COMMENT 'The transaction input zero bytes' CODEC(ZSTD(1)), + `n_input_nonzero_bytes` UInt32 COMMENT 'The transaction input nonzero bytes' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash) +COMMENT 'Contains canonical execution transaction data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_four_byte_counts +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the four byte count within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `signature` String COMMENT 'The signature of the four byte count' CODEC(ZSTD(1)), + `size` UInt64 COMMENT 'The size of the four byte count' CODEC(ZSTD(1)), + `count` UInt64 COMMENT 'The count of the four byte count' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution four byte count data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_balance_diffs +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that caused the balance diff' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the balance diff within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address of the balance diff' CODEC(ZSTD(1)), + `from_value` UInt256 COMMENT 'The from value of the balance diff' CODEC(ZSTD(1)), + `to_value` UInt256 COMMENT 'The to value of the balance diff' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution balance diff data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_balance_reads +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that caused the balance read' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the balance read within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address of the balance read' CODEC(ZSTD(1)), + `balance` UInt256 COMMENT 'The balance that was read' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution balance read data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_erc20_transfers +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the transfer within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `log_index` UInt64 COMMENT 'The log index in the block' CODEC(DoubleDelta, ZSTD(1)), + `erc20` String COMMENT 'The erc20 address' CODEC(ZSTD(1)), + `from_address` String COMMENT 'The from address' CODEC(ZSTD(1)), + `to_address` String COMMENT 'The to address' CODEC(ZSTD(1)), + `value` UInt256 COMMENT 'The value of the transfer' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution erc20 transfer data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_erc721_transfers +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the transfer within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `log_index` UInt64 COMMENT 'The log index in the block' CODEC(DoubleDelta, ZSTD(1)), + `erc721` String COMMENT 'The erc20 address' CODEC(ZSTD(1)), + `from_address` String COMMENT 'The from address' CODEC(ZSTD(1)), + `to_address` String COMMENT 'The to address' CODEC(ZSTD(1)), + `token` UInt256 COMMENT 'The token id' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution erc721 transfer data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_native_transfers +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the transfer within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `transfer_index` UInt64 COMMENT 'The transfer index' CODEC(DoubleDelta, ZSTD(1)), + `from_address` String COMMENT 'The from address' CODEC(ZSTD(1)), + `to_address` String COMMENT 'The to address' CODEC(ZSTD(1)), + `value` UInt256 COMMENT 'The value of the approval' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution native transfer data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_nonce_diffs +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that caused the nonce diff' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the nonce diff within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address of the nonce diff' CODEC(ZSTD(1)), + `from_value` UInt64 COMMENT 'The from value of the nonce diff' CODEC(ZSTD(1)), + `to_value` UInt64 COMMENT 'The to value of the nonce diff' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution nonce diff data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_nonce_reads +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index in the block' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that caused the nonce read' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the nonce read within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address of the nonce read' CODEC(ZSTD(1)), + `nonce` UInt64 COMMENT 'The nonce that was read' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution nonce read data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_address_appearances +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that caused the address appearance' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the address appearance within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address of the address appearance' CODEC(ZSTD(1)), + `relationship` LowCardinality(String) COMMENT 'The relationship of the address to the transaction', + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution address appearance data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_contracts +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash that created the contract' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the contract creation within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `create_index` UInt32 COMMENT 'The create index' CODEC(DoubleDelta, ZSTD(1)), + `contract_address` String COMMENT 'The contract address' CODEC(ZSTD(1)), + `deployer` String COMMENT 'The address of the contract deployer' CODEC(ZSTD(1)), + `factory` String COMMENT 'The address of the factory that deployed the contract' CODEC(ZSTD(1)), + `init_code` String COMMENT 'The initialization code of the contract' CODEC(ZSTD(1)), + `code` Nullable(String) COMMENT 'The code of the contract' CODEC(ZSTD(1)), + `init_code_hash` String COMMENT 'The hash of the initialization code' CODEC(ZSTD(1)), + `n_init_code_bytes` UInt32 COMMENT 'Number of bytes in the initialization code' CODEC(DoubleDelta, ZSTD(1)), + `n_code_bytes` UInt32 COMMENT 'Number of bytes in the contract code' CODEC(DoubleDelta, ZSTD(1)), + `code_hash` String COMMENT 'The hash of the contract code' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution contract data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_traces +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the trace within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `action_from` String COMMENT 'The from address of the action' CODEC(ZSTD(1)), + `action_to` Nullable(String) COMMENT 'The to address of the action' CODEC(ZSTD(1)), + `action_value` UInt256 COMMENT 'The value of the action' CODEC(ZSTD(1)), + `action_gas` Nullable(UInt64) COMMENT 'The gas provided for the action' CODEC(DoubleDelta, ZSTD(1)), + `action_input` Nullable(String) COMMENT 'The input data for the action' CODEC(ZSTD(1)), + `action_call_type` LowCardinality(String) COMMENT 'The call type of the action' CODEC(ZSTD(1)), + `action_init` Nullable(String) COMMENT 'The initialization code for the action' CODEC(ZSTD(1)), + `action_reward_type` String COMMENT 'The reward type for the action' CODEC(ZSTD(1)), + `action_type` LowCardinality(String) COMMENT 'The type of the action' CODEC(ZSTD(1)), + `result_gas_used` Nullable(UInt64) COMMENT 'The gas used in the result' CODEC(DoubleDelta, ZSTD(1)), + `result_output` Nullable(String) COMMENT 'The output of the result' CODEC(ZSTD(1)), + `result_code` Nullable(String) COMMENT 'The code returned in the result' CODEC(ZSTD(1)), + `result_address` Nullable(String) COMMENT 'The address returned in the result' CODEC(ZSTD(1)), + `trace_address` Nullable(String) COMMENT 'The trace address' CODEC(ZSTD(1)), + `subtraces` UInt32 COMMENT 'The number of subtraces' CODEC(DoubleDelta, ZSTD(1)), + `error` Nullable(String) COMMENT 'The error, if any, in the trace' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution traces data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_logs +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash associated with the log' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the log within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `log_index` UInt32 COMMENT 'The log index within the block' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address associated with the log' CODEC(ZSTD(1)), + `topic0` Nullable(String) COMMENT 'The first topic of the log' CODEC(ZSTD(1)), + `topic1` Nullable(String) COMMENT 'The second topic of the log' CODEC(ZSTD(1)), + `topic2` Nullable(String) COMMENT 'The third topic of the log' CODEC(ZSTD(1)), + `topic3` Nullable(String) COMMENT 'The fourth topic of the log' CODEC(ZSTD(1)), + `data` Nullable(String) COMMENT 'The data associated with the log' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution logs data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_storage_diffs +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash associated with the storage diff' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the storage diff within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `address` String COMMENT 'The address associated with the storage diff' CODEC(ZSTD(1)), + `slot` String COMMENT 'The storage slot key' CODEC(ZSTD(1)), + `from_value` String COMMENT 'The original value before the storage diff' CODEC(ZSTD(1)), + `to_value` String COMMENT 'The new value after the storage diff' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution storage diffs data.'; + +CREATE TABLE IF NOT EXISTS canonical_execution_storage_reads +( + `updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)), + `block_number` UInt64 COMMENT 'The block number' CODEC(DoubleDelta, ZSTD(1)), + `transaction_index` UInt64 COMMENT 'The transaction index' CODEC(DoubleDelta, ZSTD(1)), + `transaction_hash` FixedString(66) COMMENT 'The transaction hash associated with the storage read' CODEC(ZSTD(1)), + `internal_index` UInt32 COMMENT 'The internal index of the storage read within the transaction' CODEC(DoubleDelta, ZSTD(1)), + `contract_address` String COMMENT 'The contract address associated with the storage read' CODEC(ZSTD(1)), + `slot` String COMMENT 'The storage slot key' CODEC(ZSTD(1)), + `value` String COMMENT 'The value read from the storage slot' CODEC(ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name' +) +ENGINE = ReplacingMergeTree(updated_date_time) +PARTITION BY (meta_network_name, intDiv(block_number, 5000000)) +ORDER BY (meta_network_name, block_number, transaction_hash, internal_index) +COMMENT 'Contains canonical execution storage reads data.'; diff --git a/pkg/processor/cryo/values.go b/pkg/processor/cryo/values.go new file mode 100644 index 0000000..8abf67c --- /dev/null +++ b/pkg/processor/cryo/values.go @@ -0,0 +1,204 @@ +package cryo + +import ( + "fmt" + "math/big" + + "github.com/ClickHouse/ch-go/proto" + + "github.com/ethpandaops/execution-processor/pkg/processor/cryo/decode" +) + +// hashSize is the width of the FixedString columns holding 0x-prefixed hashes. +const hashSize = 66 + +// emptyHex is what cryo's --hex emits for a zero-length byte string, and what +// ClickHouse produces when hex-encoding a NULL coerced to an empty String. +const emptyHex = "0x" + +// picker resolves the columns a dataset mapping needs, holding the first +// failure so a mapping can list its columns without an error check on each. +// A missing or wrongly typed column is a cryo schema change, and stopping the +// whole block is the intended response: the alternative is a silently absent +// column, which the target schema cannot distinguish from absent data. +type picker struct { + table *decode.Table + err error +} + +func newPicker(t *decode.Table) *picker { + return &picker{table: t} +} + +func (p *picker) str(name string) *decode.Column { return p.take(p.table.Str(name)) } +func (p *picker) int(name string) *decode.Column { return p.take(p.table.Int(name)) } +func (p *picker) bool(name string) *decode.Column { return p.take(p.table.Bool(name)) } + +func (p *picker) take(c *decode.Column, err error) *decode.Column { + if err != nil && p.err == nil { + p.err = err + } + + return c +} + +// hexOrEmpty returns a hex column's value, mapping NULL to "0x" so that a row +// cryo could not attribute to a transaction still lands under a stable key. +func hexOrEmpty(c *decode.Column, i int) string { + if c.IsNull(i) { + return emptyHex + } + + return c.Str(i) +} + +// nullableHex returns the null value for a hex column that is either NULL or +// encodes a zero-length byte string, matching the empty-to-NULL guard the +// target schema expects. +// +// These return ch-go's nullable representation rather than a pointer. A *string +// costs a heap allocation per non-null value and adds an indirection the +// collector has to follow, on rows the buffer holds until the next flush. +// Carrying the value inline removes both. +func nullableHex(c *decode.Column, i int) proto.Nullable[string] { + if c.IsNull(i) { + return proto.Null[string]() + } + + v := c.Str(i) + if v == "" || v == emptyHex { + return proto.Null[string]() + } + + return proto.NewNullable(v) +} + +// nullableStr returns the null value for a plain string column that is NULL or +// empty. +func nullableStr(c *decode.Column, i int) proto.Nullable[string] { + if c.IsNull(i) { + return proto.Null[string]() + } + + v := c.Str(i) + if v == "" { + return proto.Null[string]() + } + + return proto.NewNullable(v) +} + +// strOrEmpty returns a plain string column's value, mapping NULL to "". +func strOrEmpty(c *decode.Column, i int) string { + if c.IsNull(i) { + return "" + } + + return c.Str(i) +} + +// uintOrZero returns an integer column's value, mapping NULL to 0. +func uintOrZero(c *decode.Column, i int) uint64 { + if c.IsNull(i) { + return 0 + } + + return c.Int(i) +} + +// nullableUint returns the null value for a NULL integer column. +func nullableUint(c *decode.Column, i int) proto.Nullable[uint64] { + if c.IsNull(i) { + return proto.Null[uint64]() + } + + return proto.NewNullable(c.Int(i)) +} + +// boolOrFalse returns a boolean column's value, mapping NULL to false. +func boolOrFalse(c *decode.Column, i int) bool { + if c.IsNull(i) { + return false + } + + return c.Bool(i) +} + +// fixedHash renders a 0x-prefixed hash into the fixed-width buffer the target +// column uses, right-padding with zero bytes exactly as ClickHouse does. +func fixedHash(s string) ([]byte, error) { + if len(s) > hashSize { + return nil, fmt.Errorf("hash %q is %d bytes, exceeding FixedString(%d)", s, len(s), hashSize) + } + + buf := make([]byte, hashSize) + copy(buf, s) + + return buf, nil +} + +// uint256 parses a decimal string into ch-go's little-endian limb layout. +// cryo emits every U256 as a decimal string under --u256-types string, and the +// values routinely exceed uint64, so this must never route through a float. +func uint256(s string) (proto.UInt256, error) { + if s == "" { + return proto.UInt256{}, nil + } + + v, ok := new(big.Int).SetString(s, 10) + if !ok { + return proto.UInt256{}, fmt.Errorf("value %q is not a decimal integer", s) + } + + if v.Sign() < 0 { + return proto.UInt256{}, fmt.Errorf("value %q is negative", s) + } + + if v.BitLen() > 256 { + return proto.UInt256{}, fmt.Errorf("value %q exceeds 256 bits", s) + } + + var buf [32]byte + + v.FillBytes(buf[:]) + + // FillBytes writes big-endian; the limbs are little-endian by both word + // order and byte order within each word. + return proto.UInt256{ + Low: proto.UInt128{ + Low: beUint64(buf[24:32]), + High: beUint64(buf[16:24]), + }, + High: proto.UInt128{ + Low: beUint64(buf[8:16]), + High: beUint64(buf[0:8]), + }, + }, nil +} + +func beUint64(b []byte) uint64 { + return uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 | + uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7]) +} + +// internalIndex assigns a 1-based counter per transaction hash, walking rows in +// parquet file order. ClickHouse preserves neither insertion order nor, through +// a Distributed table, any recoverable order at all, so this column is the only +// record of the sequence cryo produced. +// +// Rows whose hash is NULL are counted under the same key they will be stored +// with rather than skipped. Skipping them collapses every block reward trace in +// a block onto one sort key, which loses one of the two rewards on pre-merge +// blocks with uncles. +func internalIndex(hashes *decode.Column, rows int) []uint32 { + out := make([]uint32, rows) + seen := make(map[string]uint32, rows) + + for i := range rows { + key := hexOrEmpty(hashes, i) + seen[key]++ + out[i] = seen[key] + } + + return out +} diff --git a/pkg/processor/manager.go b/pkg/processor/manager.go index d9bcf84..9f432b2 100644 --- a/pkg/processor/manager.go +++ b/pkg/processor/manager.go @@ -35,6 +35,7 @@ import ( "github.com/ethpandaops/execution-processor/pkg/common" "github.com/ethpandaops/execution-processor/pkg/ethereum" "github.com/ethpandaops/execution-processor/pkg/leaderelection" + "github.com/ethpandaops/execution-processor/pkg/processor/cryo" "github.com/ethpandaops/execution-processor/pkg/processor/tracker" transaction_simple "github.com/ethpandaops/execution-processor/pkg/processor/transaction/simple" transaction_structlog "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" @@ -487,11 +488,61 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { m.log.Debug("Transaction structlog_agg processor is disabled") } + if err := m.initializeCryoProcessors(ctx); err != nil { + return err + } + m.log.WithField("total_processors", len(m.processors)).Info("Completed processor initialization") return nil } +// initializeCryoProcessors creates one processor per enabled cryo group. Unlike +// the other processors this is one config to many processors, because a group +// is one cryo invocation and therefore one unit of progress. +func (m *Manager) initializeCryoProcessors(ctx context.Context) error { + if !m.config.Cryo.Enabled { + m.log.Debug("Cryo processors are disabled") + + return nil + } + + for i := range m.config.Cryo.Groups { + groupCfg := &m.config.Cryo.Groups[i] + + if !groupCfg.Enabled { + continue + } + + processor, err := cryo.New(&cryo.Dependencies{ + Log: m.log, + Pool: m.pool, + State: m.state, + AsynqClient: m.asynqClient, + AsynqInspector: m.asynqInspector, + RedisClient: m.redisClient, + Network: m.network, + RedisPrefix: m.redisPrefix, + }, &m.config.Cryo, groupCfg) + if err != nil { + return fmt.Errorf("failed to create cryo processor for group %s: %w", groupCfg.Name, err) + } + + name := processor.Name() + m.processors[name] = processor + + processor.SetProcessingMode(m.config.Mode) + + m.log.WithField("processor", name).Info("Initialized processor") + + if err := m.startProcessorWithRetry(ctx, processor, name); err != nil { + return fmt.Errorf("failed to start %s processor: %w", name, err) + } + } + + return nil +} + // startProcessorWithRetry starts a processor with infinite retry and capped exponential backoff. // This ensures processors can wait for their dependencies (like ClickHouse) to become available. func (m *Manager) startProcessorWithRetry(ctx context.Context, processor tracker.BlockProcessor, name string) error { @@ -1228,82 +1279,21 @@ func (m *Manager) QueueBlockManually(ctx context.Context, processorName string, return nil, fmt.Errorf("failed to fetch block %d: %w", blockNumber, err) } - var tasksCreated int - - // Handle different processor types - switch p := processor.(type) { - case *transaction_structlog.Processor: - // Enqueue transaction tasks using the processor's method - tasksCreated, err = p.EnqueueTransactionTasks(ctx, block) - if err != nil { - return nil, fmt.Errorf("failed to enqueue tasks for block %d: %w", blockNumber, err) - } - - case *transaction_simple.Processor: - // For simple processor, enqueue a single block processing task - tasksCreated, err = m.enqueueSimpleBlockTask(ctx, p, blockNumber) - if err != nil { - return nil, fmt.Errorf("failed to enqueue block task for block %d: %w", blockNumber, err) - } - - case *transaction_structlog_agg.Processor: - // Enqueue transaction tasks using the processor's method - tasksCreated, err = p.EnqueueTransactionTasks(ctx, block) - if err != nil { - return nil, fmt.Errorf("failed to enqueue tasks for block %d: %w", blockNumber, err) - } - - default: - return nil, fmt.Errorf("processor %s has unsupported type", processorName) - } - - // Update execution_block table to mark block as processed - if err := m.state.MarkBlockProcessed(ctx, blockNumber, m.network.Name, processorName); err != nil { - return nil, fmt.Errorf("failed to update execution_block table: %w", err) + // Every processor's own ProcessBlock records the ledger row, registers the + // block for completion tracking and enqueues its work, so the manual path + // produces exactly the state the automatic path does. + if err := processor.ProcessBlock(ctx, block); err != nil { + return nil, fmt.Errorf("failed to queue block %d for %s: %w", blockNumber, processorName, err) } return &QueueResult{ TransactionCount: len(block.Transactions()), - TasksCreated: tasksCreated, }, nil } -// enqueueSimpleBlockTask enqueues a block processing task for the simple processor. -func (m *Manager) enqueueSimpleBlockTask(ctx context.Context, p *transaction_simple.Processor, blockNumber uint64) (int, error) { - payload := &transaction_simple.ProcessPayload{ - BlockNumber: *big.NewInt(int64(blockNumber)), //nolint:gosec // validated above - NetworkName: m.network.Name, - } - - var task *asynq.Task - - var queue string - - var err error - - if m.config.Mode == tracker.BACKWARDS_MODE { - task, _, err = transaction_simple.NewProcessBackwardsTask(payload) - queue = tracker.PrefixedProcessBackwardsQueue(transaction_simple.ProcessorName, m.redisPrefix) - } else { - task, _, err = transaction_simple.NewProcessForwardsTask(payload) - queue = tracker.PrefixedProcessForwardsQueue(transaction_simple.ProcessorName, m.redisPrefix) - } - - if err != nil { - return 0, fmt.Errorf("failed to create task: %w", err) - } - - if err := p.EnqueueTask(ctx, task, asynq.Queue(queue)); err != nil { - return 0, fmt.Errorf("failed to enqueue task: %w", err) - } - - return 1, nil -} - // QueueResult contains the result of queuing a block. type QueueResult struct { TransactionCount int - TasksCreated int } // initializeBlocksStoredMetrics sets initial values for blocks stored metrics to ensure visibility. @@ -1465,18 +1455,7 @@ func (m *Manager) checkGaps(ctx context.Context) { default: } - // Get the limiter from each processor - var limiter *tracker.Limiter - - switch p := processor.(type) { - case *transaction_structlog.Processor: - limiter = p.Limiter - case *transaction_simple.Processor: - limiter = p.Limiter - case *transaction_structlog_agg.Processor: - limiter = p.Limiter - } - + limiter := processor.GetLimiter() if limiter == nil { continue } diff --git a/pkg/processor/tracker/block_tracker.go b/pkg/processor/tracker/block_tracker.go index cd51547..b34a097 100644 --- a/pkg/processor/tracker/block_tracker.go +++ b/pkg/processor/tracker/block_tracker.go @@ -243,17 +243,40 @@ func (t *BlockCompletionTracker) MarkBlockComplete( blockNum uint64, network, processor, mode string, ) error { - // Write to ClickHouse - if err := t.stateProvider.MarkBlockComplete(ctx, blockNum, network, processor); err != nil { - return fmt.Errorf("failed to mark block complete in ClickHouse: %w", err) - } - - // Cleanup Redis keys completedKey := t.completedKey(network, processor, mode, blockNum) expectedKey := t.expectedKey(network, processor, mode, blockNum) metaKey := t.metaKey(network, processor, mode, blockNum) pendingKey := t.pendingBlocksKey(network, processor, mode) + // Read the expected task count before the cleanup pipeline deletes it + taskCount := 0 + + expectedStr, err := t.redis.Get(ctx, expectedKey).Result() + if err != nil { + t.log.WithError(err).WithFields(logrus.Fields{ + "block_number": blockNum, + "network": network, + "processor": processor, + "mode": mode, + }).Debug("Failed to read expected task count, recording 0") + } else if parsed, parseErr := strconv.Atoi(expectedStr); parseErr != nil { + t.log.WithError(parseErr).WithFields(logrus.Fields{ + "block_number": blockNum, + "expected": expectedStr, + "network": network, + "processor": processor, + "mode": mode, + }).Debug("Failed to parse expected task count, recording 0") + } else { + taskCount = parsed + } + + // Write to ClickHouse + if err := t.stateProvider.MarkBlockComplete(ctx, blockNum, network, processor, taskCount); err != nil { + return fmt.Errorf("failed to mark block complete in ClickHouse: %w", err) + } + + // Cleanup Redis keys pipe := t.redis.Pipeline() pipe.Del(ctx, completedKey, expectedKey, metaKey) pipe.ZRem(ctx, pendingKey, blockNum) diff --git a/pkg/processor/tracker/block_tracker_test.go b/pkg/processor/tracker/block_tracker_test.go index 90c13b6..068d89c 100644 --- a/pkg/processor/tracker/block_tracker_test.go +++ b/pkg/processor/tracker/block_tracker_test.go @@ -39,7 +39,7 @@ func (m *mockStateProviderForTracker) GetNewestIncompleteBlock( } func (m *mockStateProviderForTracker) MarkBlockComplete( - _ context.Context, _ uint64, _, _ string, + _ context.Context, _ uint64, _, _ string, _ int, ) error { return m.markCompleteErr } @@ -274,7 +274,7 @@ func (m *mockStateProviderForLimiter) GetNewestIncompleteBlock( } func (m *mockStateProviderForLimiter) MarkBlockComplete( - _ context.Context, _ uint64, _, _ string, + _ context.Context, _ uint64, _, _ string, _ int, ) error { return nil } diff --git a/pkg/processor/tracker/limiter.go b/pkg/processor/tracker/limiter.go index 0a90777..3d936c9 100644 --- a/pkg/processor/tracker/limiter.go +++ b/pkg/processor/tracker/limiter.go @@ -14,7 +14,7 @@ import ( type StateProvider interface { GetOldestIncompleteBlock(ctx context.Context, network, processor string, minBlockNumber uint64) (*uint64, error) GetNewestIncompleteBlock(ctx context.Context, network, processor string, maxBlockNumber uint64) (*uint64, error) - MarkBlockComplete(ctx context.Context, blockNumber uint64, network, processor string) error + MarkBlockComplete(ctx context.Context, blockNumber uint64, network, processor string, taskCount int) error } // GapStateProvider extends StateProvider with gap detection capabilities. diff --git a/pkg/processor/tracker/limiter_test.go b/pkg/processor/tracker/limiter_test.go index fa61bd8..f5b14b1 100644 --- a/pkg/processor/tracker/limiter_test.go +++ b/pkg/processor/tracker/limiter_test.go @@ -381,7 +381,7 @@ func (m *mockStateProvider) GetNewestIncompleteBlock( } func (m *mockStateProvider) MarkBlockComplete( - _ context.Context, _ uint64, _, _ string, + _ context.Context, _ uint64, _, _ string, _ int, ) error { return nil } diff --git a/pkg/processor/tracker/processor.go b/pkg/processor/tracker/processor.go index 9f0322f..69b222f 100644 --- a/pkg/processor/tracker/processor.go +++ b/pkg/processor/tracker/processor.go @@ -129,6 +129,9 @@ type BlockProcessor interface { // GetCompletionTracker returns the block completion tracker for checking tracking status. GetCompletionTracker() *BlockCompletionTracker + + // GetLimiter returns the block completion limiter, which owns gap scanning. + GetLimiter() *Limiter } // QueueInfo contains information about a processor queue. diff --git a/pkg/processor/transaction/simple/block_processing.go b/pkg/processor/transaction/simple/block_processing.go index 8f9e477..e2124db 100644 --- a/pkg/processor/transaction/simple/block_processing.go +++ b/pkg/processor/transaction/simple/block_processing.go @@ -189,7 +189,7 @@ func (p *Processor) ProcessBlock(ctx context.Context, block execution.Block) err if len(block.Transactions()) == 0 { p.log.WithField("block", blockNumber.Uint64()).Debug("Empty block, marking as complete") - return p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name()) + return p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name(), 0) } // 1. Mark block as enqueued in ClickHouse FIRST (complete=0) @@ -331,7 +331,7 @@ func (p *Processor) ReprocessBlock(ctx context.Context, blockNum uint64) error { if len(block.Transactions()) == 0 { p.log.WithField("block", blockNum).Debug("Empty orphaned block, marking as complete") - return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name()) + return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name(), 0) } // Use the high-priority reprocess queue for orphaned/stale blocks (mode-specific) diff --git a/pkg/processor/transaction/simple/processor.go b/pkg/processor/transaction/simple/processor.go index f8b0ada..321a9c9 100644 --- a/pkg/processor/transaction/simple/processor.go +++ b/pkg/processor/transaction/simple/processor.go @@ -200,6 +200,11 @@ func (p *Processor) GetCompletionTracker() *tracker.BlockCompletionTracker { return p.completionTracker } +// GetLimiter returns the block completion limiter. +func (p *Processor) GetLimiter() *tracker.Limiter { + return p.Limiter +} + // EnqueueTask enqueues a task to the specified queue with infinite retries. func (p *Processor) EnqueueTask(ctx context.Context, task *asynq.Task, opts ...asynq.Option) error { opts = append(opts, asynq.MaxRetry(math.MaxInt32)) diff --git a/pkg/processor/transaction/structlog/block_processing.go b/pkg/processor/transaction/structlog/block_processing.go index 9df6812..b399d68 100644 --- a/pkg/processor/transaction/structlog/block_processing.go +++ b/pkg/processor/transaction/structlog/block_processing.go @@ -204,7 +204,7 @@ func (p *Processor) ProcessBlock(ctx context.Context, block execution.Block) err }).Debug("skipping empty block") // Mark the block as complete immediately (no tasks to track) - if markErr := p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name()); markErr != nil { + if markErr := p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name(), 0); markErr != nil { p.log.WithError(markErr).WithFields(logrus.Fields{ "network": p.network.Name, "block_number": blockNumber, @@ -430,7 +430,7 @@ func (p *Processor) ReprocessBlock(ctx context.Context, blockNum uint64) error { if len(block.Transactions()) == 0 { p.log.WithField("block", blockNum).Debug("Empty orphaned block, marking as complete") - return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name()) + return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name(), 0) } expectedCount := len(block.Transactions()) diff --git a/pkg/processor/transaction/structlog/memory.go b/pkg/processor/transaction/structlog/memory.go index 1a1801b..c19f928 100644 --- a/pkg/processor/transaction/structlog/memory.go +++ b/pkg/processor/transaction/structlog/memory.go @@ -74,6 +74,7 @@ func ComputeMemoryWords(structlogs []execution.StructLog) (wordsBefore, wordsAft } // Resolve pending opcode at current depth: its wordsAfter is our wordsBefore. + //nolint:gosec // G602: i ranges over structlogs, so the index is in bounds wb := memoryWords(structlogs[i].MemorySize) wordsBefore[i] = wb diff --git a/pkg/processor/transaction/structlog/processor.go b/pkg/processor/transaction/structlog/processor.go index 34e1281..71f2226 100644 --- a/pkg/processor/transaction/structlog/processor.go +++ b/pkg/processor/transaction/structlog/processor.go @@ -252,6 +252,11 @@ func (p *Processor) GetCompletionTracker() *tracker.BlockCompletionTracker { return p.completionTracker } +// GetLimiter returns the block completion limiter. +func (p *Processor) GetLimiter() *tracker.Limiter { + return p.Limiter +} + // getProcessForwardsQueue returns the prefixed process forwards queue name. func (p *Processor) getProcessForwardsQueue() string { return tracker.PrefixedProcessForwardsQueue(ProcessorName, p.redisPrefix) diff --git a/pkg/processor/transaction/structlog_agg/block_processing.go b/pkg/processor/transaction/structlog_agg/block_processing.go index cdff57e..8de8908 100644 --- a/pkg/processor/transaction/structlog_agg/block_processing.go +++ b/pkg/processor/transaction/structlog_agg/block_processing.go @@ -204,7 +204,7 @@ func (p *Processor) ProcessBlock(ctx context.Context, block execution.Block) err }).Debug("skipping empty block") // Mark the block as complete immediately (no tasks to track) - if markErr := p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name()); markErr != nil { + if markErr := p.stateManager.MarkBlockComplete(ctx, blockNumber.Uint64(), p.network.Name, p.Name(), 0); markErr != nil { p.log.WithError(markErr).WithFields(logrus.Fields{ "network": p.network.Name, "block_number": blockNumber, @@ -430,7 +430,7 @@ func (p *Processor) ReprocessBlock(ctx context.Context, blockNum uint64) error { if len(block.Transactions()) == 0 { p.log.WithField("block", blockNum).Debug("Empty orphaned block, marking as complete") - return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name()) + return p.stateManager.MarkBlockComplete(ctx, blockNum, p.network.Name, p.Name(), 0) } expectedCount := len(block.Transactions()) diff --git a/pkg/processor/transaction/structlog_agg/processor.go b/pkg/processor/transaction/structlog_agg/processor.go index a39d044..abbebcf 100644 --- a/pkg/processor/transaction/structlog_agg/processor.go +++ b/pkg/processor/transaction/structlog_agg/processor.go @@ -262,6 +262,11 @@ func (p *Processor) GetCompletionTracker() *tracker.BlockCompletionTracker { return p.completionTracker } +// GetLimiter returns the block completion limiter. +func (p *Processor) GetLimiter() *tracker.Limiter { + return p.Limiter +} + // getProcessForwardsQueue returns the prefixed process forwards queue name. func (p *Processor) getProcessForwardsQueue() string { return tracker.PrefixedProcessForwardsQueue(ProcessorName, p.redisPrefix) diff --git a/pkg/rowbuffer/buffer.go b/pkg/rowbuffer/buffer.go index f511589..2fadeb5 100644 --- a/pkg/rowbuffer/buffer.go +++ b/pkg/rowbuffer/buffer.go @@ -189,6 +189,10 @@ func (b *Buffer[R]) Submit(ctx context.Context, rows []R) error { // Perform flush outside of lock if triggered by size if shouldFlush { + // Detached deliberately: the batch belongs to every waiter, not to the + // caller that happened to cross the threshold, so it must survive that + // caller's context being cancelled. + //nolint:gosec // G118: see above, the detachment is the point go func() { _ = b.doFlush(context.Background(), flushRows, flushWaiters, "size") }() diff --git a/pkg/state/manager.go b/pkg/state/manager.go index 6da2287..3771426 100644 --- a/pkg/state/manager.go +++ b/pkg/state/manager.go @@ -561,14 +561,15 @@ func (s *Manager) MarkBlockEnqueued(ctx context.Context, blockNumber uint64, tas // MarkBlockComplete inserts a block with complete=true to indicate all tasks finished. // This is the second phase of two-phase completion tracking. // ReplacingMergeTree will keep the latest row per (processor, network, block_number). -func (s *Manager) MarkBlockComplete(ctx context.Context, blockNumber uint64, network, processor string) error { +func (s *Manager) MarkBlockComplete(ctx context.Context, blockNumber uint64, network, processor string, taskCount int) error { query := fmt.Sprintf( - "INSERT INTO %s (updated_date_time, block_number, processor, meta_network_name, complete, task_count) VALUES ('%s', %d, '%s', '%s', 1, 0)", + "INSERT INTO %s (updated_date_time, block_number, processor, meta_network_name, complete, task_count) VALUES ('%s', %d, '%s', '%s', 1, %d)", s.storageTable, time.Now().Format("2006-01-02 15:04:05.000"), blockNumber, processor, network, + taskCount, ) err := s.storageClient.Execute(ctx, query) @@ -580,6 +581,7 @@ func (s *Manager) MarkBlockComplete(ctx context.Context, blockNumber uint64, net "block_number": blockNumber, "processor": processor, "network": network, + "task_count": taskCount, }).Debug("Marked block as complete") return nil