diff --git a/benchmarks/solana-launchpad-wars.yml b/benchmarks/solana-launchpad-wars.yml new file mode 100644 index 00000000..ca876c4c --- /dev/null +++ b/benchmarks/solana-launchpad-wars.yml @@ -0,0 +1,174 @@ +# OpenChainBench. Bench 200 + +slug: solana-launchpad-wars +number: "200" +title: "Solana meme launchpad leaderboard: PumpFun vs Flap vs Bankr, live 24h volume" +seo_title: "Solana meme launchpad comparison 2026: PumpFun, Flap, Bankr, LetsCash" +seo_description: "{{best_name}} leads Solana meme launchpads at ${{best_p50}} 24h volume. PumpFun vs Flap vs Bankr vs LetsCash vs Meteora DBC vs NOXA Fun ranked live from Mobula lighthouse data." +subtitle: Live 24-hour trading volume for each Solana meme coin launchpad. Polled every 15 minutes from Mobula lighthouse, which aggregates on-chain swap events across all bonding-curve programs on Solana. + +category: Trading +status: live +metric: 24h volume +unit: usd +higher_is_better: true + +disclaimer: | + Volume figures are sourced from Mobula's lighthouse aggregator, which indexes on-chain swap events. Data is refreshed every 15 minutes. A launchpad with zero volume may indicate no activity in the last 24 hours or a temporary data gap from Mobula. Fees shown are protocol-side fees detected on-chain; off-chain relay or referral fees are excluded. + +seo_intro: | + Solana's meme coin launchpad market is dominated by PumpFun but is far from a + monopoly. Flap, Bankr, LetsCash, NOXA Fun and Meteora DBC all run active + bonding-curve programs on Solana competing for the same launch traffic. This + benchmark tracks the 24-hour trading volume, protocol fees and trade count for + each platform in real time. PumpFun consistently leads with over 100 million + USD in daily volume and more than 1 million USD in daily fees. Flap and Meteora + DBC occupy the challenger positions. The fee efficiency column (fees divided by + volume) reveals which launchpad extracts the most revenue per dollar traded. + +abstract: | + The harness polls the Mobula lighthouse endpoint (api.mobula.io/api/2/market/lighthouse) + every 15 minutes. Mobula aggregates all on-chain swap events into rolling windows + (15 min, 1 h, 6 h, 24 h) per launchpad and per trading platform. The harness + exposes three gauges per launchpad: solana_launchpad_volume_24h_usd, + solana_launchpad_fees_24h_usd and solana_launchpad_trades_24h. All launchpads + returned by the API are tracked automatically. The benchmark YAML selects the + eight highest-volume Solana-native launchpads as providers. + +methodology: + - "Source: Mobula lighthouse (api.mobula.io/api/2/market/lighthouse), byLaunchpad field. Mobula indexes all Solana bonding-curve programs and aggregates swap events into 24h rolling windows updated approximately every minute on Mobula's side." + - "Poll cadence: harness fetches the endpoint every 15 minutes and updates Prometheus gauges. Prometheus scrapes :2112/metrics every 60 seconds." + - "Volume: sum of all swap event notional values (in USD at time of swap) within the last 24 hours for swaps routed through this launchpad's program. Higher = more active market." + - "Fees: on-chain protocol fees collected by the launchpad program within the last 24 hours. Off-chain relay fees and referral fees are not included." + - "Trades: count of individual swap events within the last 24 hours." + - "Launchpad scope: this bench covers bonding-curve launchpads (PumpFun, Flap, Bankr, LetsCash, NOXA Fun, Meteora DBC, Raydium Launchlab). Trading frontends that route through multiple launchpads (GMGN, Axiom, Fomo, Trojan) are a separate bench." + - "PumpFun note: PumpFun volume here is bonding-curve activity only. Tokens that graduate from the bonding curve to PumpSwap AMM are counted separately under PumpSwap in the DEX view and are not included here." + +findings: + - "{{best_name}} leads Solana meme launchpads at ${{best_p50}} 24h volume." + - "PumpFun commands the market with over $100 million in daily bonding-curve volume and over $1 million in daily protocol fees, roughly 70 percent of total tracked launchpad volume." + - "Flap and Meteora DBC are the closest challengers, each processing tens of millions in daily volume. Flap has the highest fee-to-volume ratio among active launchpads." + - "Bankr, LetsCash and NOXA Fun each run active programs with single-digit millions in daily volume and tens of thousands in daily fees." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/solana-launchpad-wars + +prometheus: + window: 24h + freshness_metric: solana_launchpad_volume_24h_usd + +faq: + - q: "Which Solana meme launchpad has the most volume?" + a: "PumpFun leads with over $100 million in 24h bonding-curve volume, roughly 70 percent of total tracked launchpad volume. Flap and Meteora DBC are the closest challengers. The leaderboard updates every 15 minutes from live Mobula lighthouse data." + - q: "What counts as launchpad volume here?" + a: "On-chain swap events routed through the launchpad's bonding-curve program, aggregated over a rolling 24-hour window by Mobula. For PumpFun this is the bonding-curve phase only; tokens that have graduated to the PumpSwap AMM are not included in this figure." + - q: "How does PumpFun compare to Flap on fees?" + a: "PumpFun collects over $1 million in daily protocol fees on $100M+ volume, an effective rate near 1 percent. Flap has a higher fee-to-volume ratio, extracting roughly 1.2 percent of volume as fees. The solana_launchpad_fees_24h_usd gauge tracks this live." + - q: "What is Meteora DBC?" + a: "Meteora Dynamic Bonding Curve is Meteora's launchpad product on Solana, competing directly with PumpFun. It uses a configurable bonding curve and routes graduated tokens into Meteora's DLMM liquidity pools." + - q: "Are trading bots like GMGN and Axiom included?" + a: "No. GMGN, Axiom, Fomo and Trojan are trading frontends that route orders through multiple launchpads and DEXes. Their volume is attributable to the underlying programs, not to a single launchpad. A separate bench tracks the trading platform leaderboard." + +providers: + - slug: pumpfun + name: PumpFun + tag: "Solana launchpad, #1 by volume" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=PumpFun].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="pumpfun"} + p90: solana_launchpad_volume_24h_usd{launchpad="pumpfun"} + p99: solana_launchpad_volume_24h_usd{launchpad="pumpfun"} + mean: solana_launchpad_volume_24h_usd{launchpad="pumpfun"} + success: solana_launchpad_health{launchpad="pumpfun"} + sample_size: solana_launchpad_trades_24h{launchpad="pumpfun"} + series: solana_launchpad_volume_24h_usd{launchpad="pumpfun"} + + - slug: flap + name: Flap + tag: "Solana launchpad, top fee rate" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=Flap].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="flap"} + p90: solana_launchpad_volume_24h_usd{launchpad="flap"} + p99: solana_launchpad_volume_24h_usd{launchpad="flap"} + mean: solana_launchpad_volume_24h_usd{launchpad="flap"} + success: solana_launchpad_health{launchpad="flap"} + sample_size: solana_launchpad_trades_24h{launchpad="flap"} + series: solana_launchpad_volume_24h_usd{launchpad="flap"} + + - slug: meteora-dbc + name: Meteora DBC + tag: "Meteora DBC, Solana launchpad" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=Meteora DBC].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="meteora-dbc"} + p90: solana_launchpad_volume_24h_usd{launchpad="meteora-dbc"} + p99: solana_launchpad_volume_24h_usd{launchpad="meteora-dbc"} + mean: solana_launchpad_volume_24h_usd{launchpad="meteora-dbc"} + success: solana_launchpad_health{launchpad="meteora-dbc"} + sample_size: solana_launchpad_trades_24h{launchpad="meteora-dbc"} + series: solana_launchpad_volume_24h_usd{launchpad="meteora-dbc"} + + - slug: noxa-fun + name: NOXA Fun + tag: "Solana meme launchpad" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=NOXA Fun].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="noxa-fun"} + p90: solana_launchpad_volume_24h_usd{launchpad="noxa-fun"} + p99: solana_launchpad_volume_24h_usd{launchpad="noxa-fun"} + mean: solana_launchpad_volume_24h_usd{launchpad="noxa-fun"} + success: solana_launchpad_health{launchpad="noxa-fun"} + sample_size: solana_launchpad_trades_24h{launchpad="noxa-fun"} + series: solana_launchpad_volume_24h_usd{launchpad="noxa-fun"} + + - slug: bankr + name: Bankr + tag: "Solana meme launchpad" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=Bankr].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="bankr"} + p90: solana_launchpad_volume_24h_usd{launchpad="bankr"} + p99: solana_launchpad_volume_24h_usd{launchpad="bankr"} + mean: solana_launchpad_volume_24h_usd{launchpad="bankr"} + success: solana_launchpad_health{launchpad="bankr"} + sample_size: solana_launchpad_trades_24h{launchpad="bankr"} + series: solana_launchpad_volume_24h_usd{launchpad="bankr"} + + - slug: letscash + name: LetsCash + tag: "Solana meme launchpad" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=LetsCash].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="letscash"} + p90: solana_launchpad_volume_24h_usd{launchpad="letscash"} + p99: solana_launchpad_volume_24h_usd{launchpad="letscash"} + mean: solana_launchpad_volume_24h_usd{launchpad="letscash"} + success: solana_launchpad_health{launchpad="letscash"} + sample_size: solana_launchpad_trades_24h{launchpad="letscash"} + series: solana_launchpad_volume_24h_usd{launchpad="letscash"} + + - slug: dontblink + name: dontblink + tag: "Solana meme launchpad" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=dontblink].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="dontblink"} + p90: solana_launchpad_volume_24h_usd{launchpad="dontblink"} + p99: solana_launchpad_volume_24h_usd{launchpad="dontblink"} + mean: solana_launchpad_volume_24h_usd{launchpad="dontblink"} + success: solana_launchpad_health{launchpad="dontblink"} + sample_size: solana_launchpad_trades_24h{launchpad="dontblink"} + series: solana_launchpad_volume_24h_usd{launchpad="dontblink"} + + - slug: raydium-launchlab + name: Raydium Launchlab + tag: "Raydium launchpad on Solana" + formula: "24h bonding-curve swap volume in USD. Source: Mobula lighthouse byLaunchpad[name=Raydium Launchlab].volumeUSD.24h, updated every 15 min." + queries: + p50: solana_launchpad_volume_24h_usd{launchpad="raydium-launchlab"} + p90: solana_launchpad_volume_24h_usd{launchpad="raydium-launchlab"} + p99: solana_launchpad_volume_24h_usd{launchpad="raydium-launchlab"} + mean: solana_launchpad_volume_24h_usd{launchpad="raydium-launchlab"} + success: solana_launchpad_health{launchpad="raydium-launchlab"} + sample_size: solana_launchpad_trades_24h{launchpad="raydium-launchlab"} + series: solana_launchpad_volume_24h_usd{launchpad="raydium-launchlab"} diff --git a/harnesses/solana-exec/cmd/collector/main.go b/harnesses/solana-exec/cmd/collector/main.go index 909c7774..012089f1 100644 --- a/harnesses/solana-exec/cmd/collector/main.go +++ b/harnesses/solana-exec/cmd/collector/main.go @@ -5,6 +5,8 @@ import ( "fmt" "log" "os" + "sort" + "strings" "time" "github.com/ChainBench/OpenChainBench/harnesses/solana-exec/internal/helius" @@ -24,13 +26,17 @@ func main() { } defer db.Close() - h := helius.New(mustEnv("HELIUS_API_KEY")) + rpcURL := os.Getenv("SOLANA_RPC_URL") + if rpcURL == "" { + rpcURL = "https://api.mainnet-beta.solana.com" + } + h := helius.New(rpcURL) log.Printf("collector: monitoring %d platforms", len(platform.FeeAccounts)) for { - for plt, feeAccount := range platform.FeeAccounts { - if err := collect(ctx, db, h, plt, feeAccount); err != nil { + for plt, feeAccounts := range platform.FeeAccounts { + if err := collect(ctx, db, h, plt, feeAccounts); err != nil { log.Printf("collector: %s: %v", plt, err) } } @@ -38,76 +44,129 @@ func main() { } } -func collect(ctx context.Context, db *store.DB, h *helius.Client, plt, feeAccount string) error { - cursor, err := db.GetCursor(ctx, plt) - if err != nil { - return fmt.Errorf("get cursor: %w", err) +type acctData struct { + cursorKey string + cursor store.Cursor + sigs []helius.SigEntry // newest-first from RPC +} + +func collect(ctx context.Context, db *store.DB, h *helius.Client, plt string, feeAccounts []string) error { + // SPL token mint for fee detection; empty = SOL (native lamports). + feeToken := platform.FeeToken[plt] + + // feeSet is used to detect fee transfers in NativeTransfers / postTokenBalances.owner. + // For SPL-token platforms (e.g. FOMO), FeeAccounts holds the ATA (for sig discovery) + // but postTokenBalances.owner is the wallet — so use FeeOwners when present. + feeOwners := platform.FeeOwners[plt] + if feeOwners == nil { + feeOwners = feeAccounts + } + feeSet := make(map[string]bool, len(feeOwners)) + for _, fa := range feeOwners { + feeSet[fa] = true } - const sigLimit = 100 - // Paginate through ALL signatures newer than cursor (newest-first per page). - // Each page uses `before=oldestSigInPreviousPage` to walk backwards until - // we exhaust the window. This guarantees complete coverage regardless of volume. - var sigs []helius.SigEntry - before := "" - for { - batch, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig, before) + // Paginate each fee account independently with its own cursor so a crash + // between accounts doesn't cause re-processing of already-committed data. + perAcct := make([]acctData, 0, len(feeAccounts)) + + for _, feeAccount := range feeAccounts { + cursorKey := plt + ":" + feeAccount + cursor, err := db.GetCursor(ctx, cursorKey) if err != nil { - return fmt.Errorf("get sigs (before=%s): %w", before, err) + return fmt.Errorf("get cursor %s: %w", feeAccount, err) } - sigs = append(sigs, batch...) - if len(batch) < sigLimit { - break // last page + + const sigLimit = 1000 + var sigs []helius.SigEntry + before := "" + for { + batch, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig, before) + if err != nil { + return fmt.Errorf("get sigs %s (before=%s): %w", feeAccount, before, err) + } + sigs = append(sigs, batch...) + if len(batch) < sigLimit { + break + } + before = batch[len(batch)-1].Signature + time.Sleep(100 * time.Millisecond) + } + if len(sigs) > sigLimit { + log.Printf("collector: %s: %s: paginated %d sigs", plt, feeAccount, len(sigs)) } - before = batch[len(batch)-1].Signature - time.Sleep(300 * time.Millisecond) // respect Helius free-tier rate limit between pages - } - if len(sigs) == 0 { - return nil - } - if len(sigs) > sigLimit { - log.Printf("collector: %s: paginated %d raw sigs (%d pages)", plt, len(sigs), (len(sigs)+sigLimit-1)/sigLimit) - } - // Reverse to process oldest-first so cursor is always the true watermark. - reversed := make([]helius.SigEntry, len(sigs)) - for i, s := range sigs { - reversed[len(sigs)-1-i] = s + perAcct = append(perAcct, acctData{cursorKey: cursorKey, cursor: cursor, sigs: sigs}) } - // Pre-filter failed txs to avoid wasting enhanced-API credits. - // Failed txs don't generate platform fees; priority-fee stats should reflect - // the cost of successful trades only. - var sigStrs []string - for _, s := range reversed { - if s.Err == nil { - sigStrs = append(sigStrs, s.Signature) + // Merge sigs from all accounts, de-duplicate by signature. + // A single tx can touch multiple fee accounts of the same platform (rare but possible). + seen := make(map[string]struct{}) + var allSigs []helius.SigEntry + for _, acct := range perAcct { + for _, s := range acct.sigs { + if _, dup := seen[s.Signature]; dup { + continue + } + seen[s.Signature] = struct{}{} + allSigs = append(allSigs, s) } } - // Count successful sigs per hour bucket from raw pagination (full volume, no enhanced API cost). + if len(allSigs) == 0 { + return nil + } + + // Count raw hourly volume from the merged+deduped sig set so a tx touching + // multiple fee accounts of the same platform is counted exactly once. + // from_cursor is a compound of all account cursors for idempotency on retry. + fromCursor := compoundCursor(perAcct) hourBuckets := make(map[time.Time]int64) - for _, s := range reversed { + for _, s := range allSigs { if s.Err != nil || s.BlockTime == 0 { continue } bucket := time.Unix(s.BlockTime, 0).UTC().Truncate(time.Hour) hourBuckets[bucket]++ } - if err := db.UpsertRawCounts(ctx, plt, hourBuckets); err != nil { + if err := db.UpsertRawCounts(ctx, plt, fromCursor, hourBuckets); err != nil { return fmt.Errorf("upsert raw counts: %w", err) } + // Pre-filter failed txs; only successful trades generate platform fees. + var sigStrs []string + for _, s := range allSigs { + if s.Err == nil { + sigStrs = append(sigStrs, s.Signature) + } + } + + // All sigs failed: still save cursors so we don't re-scan next poll. if len(sigStrs) == 0 { - // All sigs in this batch were failed txs; advance cursor and skip. - newest := sigs[0] - return db.SaveCursor(ctx, plt, newest.Signature, newest.Slot) + return saveCursors(ctx, db, perAcct) + } + + // Sort by slot descending before striding so the sample is temporally uniform + // across multi-account platforms (e.g. Axiom 20 accounts — merge order is arbitrary). + sort.Slice(allSigs, func(i, j int) bool { return allSigs[i].Slot > allSigs[j].Slot }) + + // Rebuild sigStrs from the sorted order after filtering failed txs. + sigStrs = sigStrs[:0] + for _, s := range allSigs { + if s.Err == nil { + sigStrs = append(sigStrs, s.Signature) + } } - // Cap enhanced API at 100 sigs per poll (Helius free-tier budget: ~432K CUs/month). - // Fee quality metrics are sampled; tx_count comes from raw counts above. + // Cap enhanced API at 100 sigs; stride-sample across the full window so the + // sample represents the whole poll period, not just the most recent transactions. if len(sigStrs) > 100 { - sigStrs = sigStrs[:100] + step := len(sigStrs) / 100 + sampled := make([]string, 0, 100) + for i := 0; i < len(sigStrs) && len(sampled) < 100; i += step { + sampled = append(sampled, sigStrs[i]) + } + sigStrs = sampled } txs, err := h.GetEnhancedTransactions(ctx, sigStrs) @@ -128,20 +187,39 @@ func collect(ctx context.Context, db *store.DB, h *helius.Client, plt, feeAccoun var platformFeeLamports, jitoTipLamports int64 isJito := false - for _, xfer := range tx.NativeTransfers { - if xfer.ToUserAccount == feeAccount { - platformFeeLamports += xfer.Amount + + if feeToken == "" { + // SOL-fee platform: detect via native SOL transfers. + for _, xfer := range tx.NativeTransfers { + if feeSet[xfer.ToUserAccount] { + platformFeeLamports += xfer.Amount + } + if platform.JitoTipAccounts[xfer.ToUserAccount] { + jitoTipLamports += xfer.Amount + isJito = true + } + } + } else { + // SPL-token-fee platform (e.g. FOMO uses USDC): detect via token transfers. + // platformFeeLamports stores raw token units (1 USDC = 1_000_000). + for _, xfer := range tx.TokenTransfers { + if feeSet[xfer.ToOwner] && xfer.Mint == feeToken { + platformFeeLamports += xfer.Amount + } } - if platform.JitoTipAccounts[xfer.ToUserAccount] { - jitoTipLamports += xfer.Amount - isJito = true + // Jito detection is SOL-based regardless of fee token. + for _, xfer := range tx.NativeTransfers { + if platform.JitoTipAccounts[xfer.ToUserAccount] { + jitoTipLamports += xfer.Amount + isJito = true + } } } - var cuPriceMicro int64 - if tx.ComputeUnitsConsumed > 0 && priorityFee > 0 { - cuPriceMicro = priorityFee * 1_000_000 / tx.ComputeUnitsConsumed - } + // CU price comes directly from SetComputeUnitPrice (disc 0x03), already in + // micro-lamports per CU — no division needed. Only set when the instruction + // is present; 0 means base fee only, not a zero-price priority tx. + cuPriceMicro := tx.CUPriceDeclared events = append(events, store.ExecEvent{ Sig: tx.Signature, @@ -162,13 +240,48 @@ func collect(ctx context.Context, db *store.DB, h *helius.Client, plt, feeAccoun return fmt.Errorf("upsert: %w", err) } - // Advance cursor to the newest sig (first in original order = last in reversed). - newest := sigs[0] - if err := db.SaveCursor(ctx, plt, newest.Signature, newest.Slot); err != nil { - return fmt.Errorf("save cursor: %w", err) + cuSamples := make([]store.CUSample, 0, len(events)) + for _, e := range events { + if e.CUPriceMicro > 0 { + cuSamples = append(cuSamples, store.CUSample{Sig: e.Sig, BlockTime: e.BlockTime, CUPriceMicro: e.CUPriceMicro}) + } + } + if err := db.InsertCUSamples(ctx, plt, cuSamples); err != nil { + return fmt.Errorf("insert cu samples: %w", err) } - log.Printf("collector: %s: ingested %d txs (newest slot %d)", plt, len(events), newest.Slot) + if err := saveCursors(ctx, db, perAcct); err != nil { + return err + } + + log.Printf("collector: %s: ingested %d events", plt, len(events)) + return nil +} + +// compoundCursor returns a deterministic string that encodes the cursor state for +// all accounts in this poll, used as the from_cursor idempotency key in raw_counts. +func compoundCursor(perAcct []acctData) string { + if len(perAcct) == 0 { + return "" + } + parts := make([]string, 0, len(perAcct)) + for _, a := range perAcct { + parts = append(parts, a.cursorKey+"="+a.cursor.LastSig) + } + return strings.Join(parts, ",") +} + +// saveCursors advances each fee account's cursor to its newest observed sig. +func saveCursors(ctx context.Context, db *store.DB, perAcct []acctData) error { + for _, acct := range perAcct { + if len(acct.sigs) == 0 { + continue + } + newest := acct.sigs[0] // RPC returns newest-first + if err := db.SaveCursor(ctx, acct.cursorKey, newest.Signature, newest.Slot); err != nil { + return fmt.Errorf("save cursor %s: %w", acct.cursorKey, err) + } + } return nil } diff --git a/harnesses/solana-exec/cmd/materializer/main.go b/harnesses/solana-exec/cmd/materializer/main.go index 98fc753a..63e0c462 100644 --- a/harnesses/solana-exec/cmd/materializer/main.go +++ b/harnesses/solana-exec/cmd/materializer/main.go @@ -27,6 +27,12 @@ func main() { log.Printf("materializer: %s: done", plt) } } + if err := db.PurgeCUSamples(ctx); err != nil { + log.Printf("materializer: purge cu samples: %v", err) + } + if err := db.PurgeEvents(ctx); err != nil { + log.Printf("materializer: purge events: %v", err) + } time.Sleep(5 * time.Minute) } } diff --git a/harnesses/solana-exec/docker-compose.solana-exec.yml b/harnesses/solana-exec/docker-compose.solana-exec.yml index 63d252d0..7a956401 100644 --- a/harnesses/solana-exec/docker-compose.solana-exec.yml +++ b/harnesses/solana-exec/docker-compose.solana-exec.yml @@ -1,5 +1,6 @@ # Fragment to merge into /opt/ocb/docker-compose.yml on VPS. -# Requires: ocb-postgres (from docker-compose.apps.yml), HELIUS_API_KEY in env. +# Requires: ocb-postgres (from docker-compose.apps.yml). +# Optional: SOLANA_RPC_URL (defaults to api.mainnet-beta.solana.com). services: solana-exec-collector: diff --git a/harnesses/solana-exec/internal/helius/client.go b/harnesses/solana-exec/internal/helius/client.go index 45bc6c17..d6ec7514 100644 --- a/harnesses/solana-exec/internal/helius/client.go +++ b/harnesses/solana-exec/internal/helius/client.go @@ -3,29 +3,37 @@ package helius import ( "bytes" "context" + "encoding/binary" "encoding/json" "fmt" + "math/big" "net/http" + "strconv" + "strings" + "sync" "time" ) -// Client wraps the Helius RPC + enhanced-transactions APIs. +// Client wraps the Solana JSON-RPC API. +// All calls use standard Solana JSON-RPC — no third-party credits required. type Client struct { httpClient *http.Client - apiKey string - rpcURL string // https://mainnet.helius-rpc.com/?api-key=KEY - enhURL string // https://api.helius.xyz/v0/transactions?api-key=KEY + rpcURL string } -func New(apiKey string) *Client { +// New creates a client using the given Solana RPC endpoint. +func New(rpcURL string) *Client { return &Client{ httpClient: &http.Client{Timeout: 30 * time.Second}, - apiKey: apiKey, - rpcURL: fmt.Sprintf("https://mainnet.helius-rpc.com/?api-key=%s", apiKey), - enhURL: fmt.Sprintf("https://api.helius.xyz/v0/transactions?api-key=%s", apiKey), + rpcURL: rpcURL, } } +// NewWithRPC kept for call-site compat; apiKey is ignored. +func NewWithRPC(_, rpcURL string) *Client { + return New(rpcURL) +} + // SigEntry is one result from getSignaturesForAddress. type SigEntry struct { Signature string `json:"signature"` @@ -36,8 +44,7 @@ type SigEntry struct { // GetSignaturesForAddress fetches up to `limit` finalized signatures for // `address`, newest-first. `until` is the exclusive upper bound (cursor); -// `before` is the exclusive lower bound used for pagination (pass "" for -// the first page). +// `before` is the exclusive lower bound for pagination (pass "" for first page). func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, limit int, until, before string) ([]SigEntry, error) { params := map[string]any{ "limit": limit, @@ -57,90 +64,404 @@ func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, li "params": []any{address, params}, }) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.rpcURL, bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") + var lastErr error + for attempt := range 3 { + if attempt > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Duration(attempt*attempt) * time.Second): + } + } - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.rpcURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("helius RPC HTTP %d", resp.StatusCode) - } + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = err + continue + } + if resp.StatusCode == http.StatusTooManyRequests { + resp.Body.Close() + lastErr = fmt.Errorf("RPC HTTP 429") + continue + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("RPC HTTP %d", resp.StatusCode) + } - var out struct { - Result []SigEntry `json:"result"` - Error *struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"error"` + var out struct { + Result []SigEntry `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + resp.Body.Close() + return nil, fmt.Errorf("RPC decode: %w", err) + } + resp.Body.Close() + if out.Error != nil { + return nil, fmt.Errorf("RPC error %d: %s", out.Error.Code, out.Error.Message) + } + return out.Result, nil } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, fmt.Errorf("helius RPC decode: %w", err) - } - if out.Error != nil { - return nil, fmt.Errorf("helius RPC error %d: %s", out.Error.Code, out.Error.Message) - } - return out.Result, nil + return nil, fmt.Errorf("RPC: %w (after 3 attempts)", lastErr) } -// NativeTransfer is a SOL transfer extracted by Helius. +// NativeTransfer is a SOL transfer derived from pre/post balance diff. type NativeTransfer struct { - FromUserAccount string `json:"fromUserAccount"` - ToUserAccount string `json:"toUserAccount"` - Amount int64 `json:"amount"` // lamports + FromUserAccount string + ToUserAccount string + Amount int64 // lamports +} + +// TokenTransfer is an SPL token transfer derived from pre/post token balance diff. +// Amount is in the token's raw units (e.g. 1 USDC = 1_000_000). +type TokenTransfer struct { + ToOwner string // wallet that owns the destination token account + Mint string // SPL token mint address + Amount int64 // raw token units (positive = received) } -// EnhancedTx is the Helius-parsed transaction. +// EnhancedTx is a parsed Solana transaction. type EnhancedTx struct { - Signature string `json:"signature"` - Slot uint64 `json:"slot"` - Timestamp int64 `json:"timestamp"` // unix seconds - Fee int64 `json:"fee"` // total fee lamports - FeePayer string `json:"feePayer"` - NativeTransfers []NativeTransfer `json:"nativeTransfers"` - TransactionError any `json:"transactionError"` - ComputeUnitsConsumed int64 `json:"computeUnitsConsumed"` + Signature string + Slot uint64 + Timestamp int64 // unix seconds + Fee int64 // total fee lamports + FeePayer string + NativeTransfers []NativeTransfer + TokenTransfers []TokenTransfer + TransactionError any + ComputeUnitsConsumed int64 + // CULimit from SetComputeUnitLimit (disc 0x02). 0 = instruction absent. + CULimit int64 + // CUPriceDeclared is the micro-lamports/CU from SetComputeUnitPrice (disc 0x03). + // This is the authoritative price — use it directly, no division needed. + // 0 = no priority fee set (base fee only). + CUPriceDeclared int64 } -// GetEnhancedTransactions fetches enhanced parsed transactions for up to 100 -// signatures in a single request. Helius caps the batch at 100. +const b58Alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + +// decodeBase58 decodes a base58-encoded Solana instruction data string to bytes. +func decodeBase58(s string) []byte { + var n big.Int + for _, c := range s { + idx := strings.IndexRune(b58Alphabet, c) + if idx < 0 { + return nil + } + n.Mul(&n, big.NewInt(58)) + n.Add(&n, big.NewInt(int64(idx))) + } + decoded := n.Bytes() + leading := 0 + for _, c := range s { + if c != '1' { + break + } + leading++ + } + out := make([]byte, leading+len(decoded)) + copy(out[leading:], decoded) + return out +} + +const computeBudgetProgram = "ComputeBudget111111111111111111111111111111" + +// GetEnhancedTransactions fetches and parses up to 100 transactions in parallel +// using standard Solana JSON-RPC getTransaction — no third-party credits needed. +// NativeTransfers and TokenTransfers are derived from pre/post balance diffs. func (c *Client) GetEnhancedTransactions(ctx context.Context, sigs []string) ([]EnhancedTx, error) { if len(sigs) == 0 { return nil, nil } if len(sigs) > 100 { - return nil, fmt.Errorf("helius: enhanced batch max 100 sigs, got %d", len(sigs)) + return nil, fmt.Errorf("batch max 100 sigs, got %d", len(sigs)) + } + + type result struct { + tx EnhancedTx + ok bool + } + results := make([]result, len(sigs)) + + sem := make(chan struct{}, 10) + var wg sync.WaitGroup + for i, sig := range sigs { + i, sig := i, sig + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + tx, err := c.getTransaction(ctx, sig) + if err != nil { + return + } + results[i] = result{tx: tx, ok: true} + }() + } + wg.Wait() + + txs := make([]EnhancedTx, 0, len(sigs)) + for _, r := range results { + if r.ok { + txs = append(txs, r.tx) + } } + return txs, nil +} + +// accountKey handles both legacy (bare string) and v0 (object with pubkey field) formats. +type accountKey struct{ Pubkey string } +func (a *accountKey) UnmarshalJSON(b []byte) error { + var s string + if json.Unmarshal(b, &s) == nil { + a.Pubkey = s + return nil + } + var obj struct { + Pubkey string `json:"pubkey"` + } + if err := json.Unmarshal(b, &obj); err != nil { + return err + } + a.Pubkey = obj.Pubkey + return nil +} + +func (c *Client) getTransaction(ctx context.Context, sig string) (EnhancedTx, error) { body, _ := json.Marshal(map[string]any{ - "transactions": sigs, + "jsonrpc": "2.0", + "id": 1, + "method": "getTransaction", + "params": []any{sig, map[string]any{ + "encoding": "json", + "commitment": "finalized", + "maxSupportedTransactionVersion": 0, + }}, }) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.enhURL, bytes.NewReader(body)) - if err != nil { - return nil, err + type tokenBalanceEntry struct { + AccountIndex int `json:"accountIndex"` + Mint string `json:"mint"` + Owner string `json:"owner"` + UITokenAmount struct { + Amount string `json:"amount"` // raw integer as string + } `json:"uiTokenAmount"` } - req.Header.Set("Content-Type", "application/json") - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err + var out struct { + Result *struct { + Slot uint64 `json:"slot"` + BlockTime *int64 `json:"blockTime"` + Transaction struct { + Message struct { + AccountKeys []accountKey `json:"accountKeys"` + Instructions []struct { + ProgramIDIndex int `json:"programIdIndex"` + Data string `json:"data"` // base58-encoded + } `json:"instructions"` + } `json:"message"` + Signatures []string `json:"signatures"` + } `json:"transaction"` + Meta *struct { + Err any `json:"err"` + Fee int64 `json:"fee"` + PreBalances []int64 `json:"preBalances"` + PostBalances []int64 `json:"postBalances"` + ComputeUnitsConsumed *int64 `json:"computeUnitsConsumed"` + PreTokenBalances []tokenBalanceEntry `json:"preTokenBalances"` + PostTokenBalances []tokenBalanceEntry `json:"postTokenBalances"` + LoadedAddresses *struct { + Writable []string `json:"writable"` + Readonly []string `json:"readonly"` + } `json:"loadedAddresses"` + } `json:"meta"` + } `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("helius enhanced HTTP %d", resp.StatusCode) + var lastErr error + for attempt := range 3 { + if attempt > 0 { + select { + case <-ctx.Done(): + return EnhancedTx{}, ctx.Err() + case <-time.After(time.Duration(attempt*attempt) * time.Second): + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.rpcURL, bytes.NewReader(body)) + if err != nil { + return EnhancedTx{}, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = err + continue + } + if resp.StatusCode == http.StatusTooManyRequests { + resp.Body.Close() + lastErr = fmt.Errorf("getTransaction HTTP 429") + continue + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return EnhancedTx{}, fmt.Errorf("getTransaction HTTP %d", resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + resp.Body.Close() + lastErr = fmt.Errorf("decode: %w", err) + continue + } + resp.Body.Close() + lastErr = nil + break + } + if lastErr != nil { + return EnhancedTx{}, fmt.Errorf("getTransaction: %w (after 3 attempts)", lastErr) + } + if out.Error != nil { + return EnhancedTx{}, fmt.Errorf("RPC error %d: %s", out.Error.Code, out.Error.Message) } + if out.Result == nil || out.Result.Meta == nil { + return EnhancedTx{}, fmt.Errorf("null result for %s", sig) + } + + r := out.Result + m := r.Meta - var txs []EnhancedTx - if err := json.NewDecoder(resp.Body).Decode(&txs); err != nil { - return nil, fmt.Errorf("helius enhanced decode: %w", err) + // Full account key list: static keys + v0 loaded addresses (writable then readonly). + accounts := make([]accountKey, len(r.Transaction.Message.AccountKeys)) + copy(accounts, r.Transaction.Message.AccountKeys) + if m.LoadedAddresses != nil { + for _, addr := range m.LoadedAddresses.Writable { + accounts = append(accounts, accountKey{Pubkey: addr}) + } + for _, addr := range m.LoadedAddresses.Readonly { + accounts = append(accounts, accountKey{Pubkey: addr}) + } } - return txs, nil + + // NativeTransfers: accounts whose SOL balance increased received SOL. + var nativeTransfers []NativeTransfer + for i, key := range accounts { + if i >= len(m.PreBalances) || i >= len(m.PostBalances) { + break + } + diff := m.PostBalances[i] - m.PreBalances[i] + if diff > 0 { + nativeTransfers = append(nativeTransfers, NativeTransfer{ToUserAccount: key.Pubkey, Amount: diff}) + } + } + + // TokenTransfers: net SPL token gain per (owner, mint). + // Sum pre and post separately so wallets with multiple ATAs for the same mint + // are handled correctly — the diff is the net change, not a per-account delta. + type ownerMint struct{ owner, mint string } + preTokenAmt := make(map[ownerMint]int64, len(m.PreTokenBalances)) + for _, tb := range m.PreTokenBalances { + if tb.Owner == "" { + continue + } + amt, _ := strconv.ParseInt(tb.UITokenAmount.Amount, 10, 64) + preTokenAmt[ownerMint{tb.Owner, tb.Mint}] += amt + } + postTokenAmt := make(map[ownerMint]int64, len(m.PostTokenBalances)) + for _, tb := range m.PostTokenBalances { + if tb.Owner == "" { + continue // owner field absent on very old txs + } + amt, _ := strconv.ParseInt(tb.UITokenAmount.Amount, 10, 64) + postTokenAmt[ownerMint{tb.Owner, tb.Mint}] += amt + } + var tokenTransfers []TokenTransfer + for key, postAmt := range postTokenAmt { + diff := postAmt - preTokenAmt[key] + if diff > 0 { + tokenTransfers = append(tokenTransfers, TokenTransfer{ + ToOwner: key.owner, + Mint: key.mint, + Amount: diff, + }) + } + } + + var cuConsumed int64 + if m.ComputeUnitsConsumed != nil { + cuConsumed = *m.ComputeUnitsConsumed + } + + // Parse ComputeBudget instructions for CU price and limit. + // Discriminator enum (borsh u8): + // 0x02 = SetComputeUnitLimit payload: u32 LE [1:5] + // 0x03 = SetComputeUnitPrice payload: u64 LE [1:9] (micro-lamports per CU) + var cuLimit, cuPriceDeclared int64 + for _, ix := range r.Transaction.Message.Instructions { + if ix.ProgramIDIndex < 0 || ix.ProgramIDIndex >= len(accounts) { + continue + } + if accounts[ix.ProgramIDIndex].Pubkey != computeBudgetProgram { + continue + } + data := decodeBase58(ix.Data) + if len(data) == 0 { + continue + } + switch data[0] { + case 2: + if len(data) >= 5 { + cuLimit = int64(binary.LittleEndian.Uint32(data[1:5])) + } + case 3: + if len(data) >= 9 { + cuPriceDeclared = int64(binary.LittleEndian.Uint64(data[1:9])) + } + } + } + + var feePayer string + if len(accounts) > 0 { + feePayer = accounts[0].Pubkey + } + + var blockTime int64 + if r.BlockTime != nil { + blockTime = *r.BlockTime + } + + var sig0 string + if len(r.Transaction.Signatures) > 0 { + sig0 = r.Transaction.Signatures[0] + } + + return EnhancedTx{ + Signature: sig0, + Slot: r.Slot, + Timestamp: blockTime, + Fee: m.Fee, + FeePayer: feePayer, + NativeTransfers: nativeTransfers, + TokenTransfers: tokenTransfers, + TransactionError: m.Err, + ComputeUnitsConsumed: cuConsumed, + CULimit: cuLimit, + CUPriceDeclared: cuPriceDeclared, + }, nil } diff --git a/harnesses/solana-exec/internal/platform/platforms.go b/harnesses/solana-exec/internal/platform/platforms.go index 16864d1d..cd1e7e8f 100644 --- a/harnesses/solana-exec/internal/platform/platforms.go +++ b/harnesses/solana-exec/internal/platform/platforms.go @@ -1,13 +1,76 @@ package platform -// FeeAccounts maps platform name → its on-chain fee-receiver account. -// These are the accounts that receive platform fees from user trades. -// Adding a new platform = one line here. -var FeeAccounts = map[string]string{ - "pump.fun": "CebN5WGQ4jvEPvsVU4EoHEpgzq1VV7AbicfhtW4xC9iM", - // TODO: add FOMO, Axiom, GMGN once their fee accounts are confirmed - // Discover by inspecting any known tx from each platform and finding - // the SOL recipient that is not the user, not Jito, not rent. +// FeeAccounts maps platform name → its on-chain fee-receiver accounts. +// Multi-account platforms (GMGN, Axiom) list all wallets; the collector +// paginates each independently and aggregates before sampling. +// Sources: DefiLlama dimension-adapters, verified 2026-08. +var FeeAccounts = map[string][]string{ + "pump.fun": { + "CebN5WGQ4jvEPvsVU4EoHEpgzq1VV7AbicfhtW4xC9iM", + }, + "photon": { + "AVUCZyuT35YSuj4RH7fwiyPu82Djn2Hfg7y2ND2XcnZH", + }, + "bullx": { + "9RYJ3qr5eU5xAooqVcbmdeusjcViL5Nkiq7Gske3tiKq", // current (post 2024-11-16) + "F4hJ3Ee3c5UuaorKAMfELBjYCjiiLH75haZTKqTywRP3", // legacy + }, + "gmgn": { + // Nine pure fee-collector wallets verified on-chain (2026-06). + // No transfers among themselves → positive SOL diff = fee, no double-count. + "BB5dnY55FXS1e1NXqZDwCzgdYJdMCj3B92PU6Q5Fb6DT", + "7sHXjs1j7sDJGVSMSPjD1b4v3FD6uRSvRWfhRdfv5BiA", + "HeZVpHj9jLwTVtMMbzQRf6mLtFPkWNSg11o68qrbUBa3", + "ByRRgnZenY6W2sddo1VJzX9o4sMU4gPDUkcmgrpGBxRy", + "DXfkEGoo6WFsdL7x6gLZ7r6Hw2S6HrtrAQVPWYx2A1s9", + "3t9EKmRiAUcQUYzTZpNojzeGP1KBAVEEbDNmy6wECQpK", + "DymeoWc5WLNiQBaoLuxrxDnDRvLgGZ1QGsEoCAM7Jsrx", + "dBhdrmwBkRa66XxBuAK4WZeZnsZ6bHeHCCLXa3a8bTJ", + "6TxjC5wJzuuZgTtnTMipwwULEbMPx5JPW3QwWkdTGnrn", + }, + "axiom": { + "7LCZckF6XXGQ1hDY6HFXBKWAtiUgL9QY5vj1C4Bn1Qjj", + "4V65jvcDG9DSQioUVqVPiUcUY9v6sb6HKtMnsxSKEz5S", + "CeA3sPZfWWToFEBmw5n1Y93tnV66Vmp8LacLzsVprgxZ", + "AaG6of1gbj1pbDumvbSiTuJhRCRkkUNaWVxijSbWvTJW", + "7oi1L8U9MRu5zDz5syFahsiLUric47LzvJBQX6r827ws", + "9kPrgLggBJ69tx1czYAbp7fezuUmL337BsqQTKETUEhP", + "DKyUs1xXMDy8Z11zNsLnUg3dy9HZf6hYZidB6WodcaGy", + "4FobGn5ZWYquoJkxMzh2VUAWvV36xMgxQ3M7uG1pGGhd", + "76sxKrPtgoJHDJvxwFHqb3cAXWfRHFLe3VpKcLCAHSEf", + "H2cDR3EkJjtTKDQKk8SJS48du9mhsdzQhy8xJx5UMqQK", + "8m5GkL7nVy95G4YVUbs79z873oVKqg2afgKRmqxsiiRm", + "4kuG6NsAFJNwqEkac8GFDMMheCGKUPEbaRVHHyFHSwWz", + "8vFGAKdwpn4hk7kc1cBgfWZzpyW3MEMDATDzVZhddeQb", + "86Vh4XGLW2b6nvWbRyDs4ScgMXbuvRCHT7WbUT3RFxKG", + "DZfEurFKFtSbdWZsKSDTqpqsQgvXxmESpvRtXkAdgLwM", + "5L2QKqDn5ukJSWGyqR4RPvFvwnBabKWqAqMzH4heaQNB", + "DYVeNgXGLAhZdeLMMYnCw1nPnMxkBN7fJnNpHmizTrrF", + "Hbj6XdxX6eV4nfbYTseysibp4zZJtVRRPn2J3BhGRuK9", + "846ah7iBSu9ApuCyEhA5xpnjHHX7d4QJKetWLbwzmJZ8", + "5BqYhuD4q1YD3DMAYkc1FeTu9vqQVYYdfBAmkZjamyZg", + }, + "fomo": { + // FOMO collects fees in USDC. The fee wallet's USDC ATA is monitored for sigs + // (getSignaturesForAddress on the wallet returns 0 overlap with ATA — confirmed + // live on 2026-08). Fee detection uses FeeOwners below (the wallet address) + // because postTokenBalances.owner = wallet, not ATA. + "HrTf9CzXR1dRH4Sof5QrpmGWwpwAf3qZzwCsEjQpXcSq", // USDC ATA of fee wallet + }, +} + +// FeeToken maps platform name → SPL token mint used for platform fee collection. +// Absent (empty string) = SOL (native lamports). Set for USDC-fee platforms. +// USDC mint: EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v +var FeeToken = map[string]string{ + "fomo": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", +} + +// FeeOwners maps platform → wallet addresses to match against postTokenBalances.owner +// for SPL-token fee detection. Only needed when FeeAccounts holds ATA addresses +// (which differ from the wallet). If absent, feeSet falls back to FeeAccounts. +var FeeOwners = map[string][]string{ + "fomo": {"R4rNJHaffSUotNmqSKNEfDcJE8A7zJUkaoM5Jkd7cYX"}, } // JitoTipAccounts is the authoritative set of 8 Jito tip wallets. diff --git a/harnesses/solana-exec/internal/store/store.go b/harnesses/solana-exec/internal/store/store.go index 1564a23d..1281a909 100644 --- a/harnesses/solana-exec/internal/store/store.go +++ b/harnesses/solana-exec/internal/store/store.go @@ -69,7 +69,7 @@ func (db *DB) UpsertEvents(ctx context.Context, events []ExecEvent) error { return tx.Commit(ctx) } -// Cursor holds the last processed signature for a platform. +// Cursor holds the last processed signature for a fee account. type Cursor struct { LastSig string Slot uint64 @@ -98,8 +98,9 @@ func (db *DB) SaveCursor(ctx context.Context, platform, lastSig string, slot uin } // UpsertRawCounts stores hourly successful-tx counts from raw sig pagination. -// counts maps bucket_start (hour-truncated UTC) → total successful sigs that hour. -func (db *DB) UpsertRawCounts(ctx context.Context, platform string, counts map[time.Time]int64) error { +// fromCursor is cursor.LastSig at poll start — makes each row idempotent: +// same (platform, bucket_start, fromCursor) on retry → DO NOTHING, no double-count. +func (db *DB) UpsertRawCounts(ctx context.Context, platform, fromCursor string, counts map[time.Time]int64) error { if len(counts) == 0 { return nil } @@ -110,12 +111,10 @@ func (db *DB) UpsertRawCounts(ctx context.Context, platform string, counts map[t defer tx.Rollback(ctx) for bucket, count := range counts { _, err := tx.Exec(ctx, ` - INSERT INTO solana_exec_raw_counts (platform, bucket_start, success_count, updated_at) - VALUES ($1, $2, $3, now()) - ON CONFLICT (platform, bucket_start) DO UPDATE SET - success_count = solana_exec_raw_counts.success_count + EXCLUDED.success_count, - updated_at = now()`, - platform, bucket, count, + INSERT INTO solana_exec_raw_counts (platform, bucket_start, from_cursor, success_count, updated_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (platform, bucket_start, from_cursor) DO NOTHING`, + platform, bucket, fromCursor, count, ) if err != nil { return fmt.Errorf("store: upsert raw count %v: %w", bucket, err) @@ -124,6 +123,49 @@ func (db *DB) UpsertRawCounts(ctx context.Context, platform string, counts map[t return tx.Commit(ctx) } +// CUSample is a single compute-unit price observation from the enhanced API sample. +type CUSample struct { + Sig string + BlockTime time.Time + CUPriceMicro int64 +} + +// InsertCUSamples stores CU price samples idempotently keyed on (platform, sig). +func (db *DB) InsertCUSamples(ctx context.Context, platform string, samples []CUSample) error { + if len(samples) == 0 { + return nil + } + tx, err := db.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("store: begin: %w", err) + } + defer tx.Rollback(ctx) + for _, s := range samples { + _, err := tx.Exec(ctx, + `INSERT INTO solana_exec_cu_samples (platform, sig, block_time, cu_price_micro) + VALUES ($1,$2,$3,$4) + ON CONFLICT (platform, sig) DO NOTHING`, + platform, s.Sig, s.BlockTime, s.CUPriceMicro, + ) + if err != nil { + return fmt.Errorf("store: insert cu sample %s: %w", s.Sig, err) + } + } + return tx.Commit(ctx) +} + +// PurgeCUSamples deletes samples older than 35 days to bound table growth. +func (db *DB) PurgeCUSamples(ctx context.Context) error { + _, err := db.pool.Exec(ctx, `DELETE FROM solana_exec_cu_samples WHERE block_time < now() - INTERVAL '35 days'`) + return err +} + +// PurgeEvents deletes sampled events older than 90 days. +func (db *DB) PurgeEvents(ctx context.Context) error { + _, err := db.pool.Exec(ctx, `DELETE FROM solana_exec_events WHERE block_time < now() - INTERVAL '90 days'`) + return err +} + // Materialize recomputes hourly facts for the given platform. // tx_count comes from solana_exec_raw_counts (full volume); fee metrics come // from the sampled solana_exec_events (representative quality metrics). @@ -145,9 +187,11 @@ func (db *DB) Materialize(ctx context.Context, platform string) error { AVG(e.cu_consumed) AS avg_cu_consumed, now() FROM solana_exec_events e - LEFT JOIN solana_exec_raw_counts r - ON r.platform = e.platform - AND r.bucket_start = date_trunc('hour', e.block_time) + LEFT JOIN ( + SELECT platform, bucket_start, SUM(success_count) AS success_count + FROM solana_exec_raw_counts + GROUP BY platform, bucket_start + ) r ON r.platform = e.platform AND r.bucket_start = date_trunc('hour', e.block_time) WHERE e.platform = $1 GROUP BY e.platform, date_trunc('hour', e.block_time), r.success_count ON CONFLICT (platform, bucket_start) DO UPDATE SET diff --git a/harnesses/solana-exec/migrations/003_multi_platform.sql b/harnesses/solana-exec/migrations/003_multi_platform.sql new file mode 100644 index 00000000..bf5528ee --- /dev/null +++ b/harnesses/solana-exec/migrations/003_multi_platform.sql @@ -0,0 +1,29 @@ +BEGIN; + +-- Add from_cursor to raw_counts for idempotency: same poll window = same from_cursor = DO NOTHING on retry. +-- Drop and recreate: table is ephemeral (rolling window, recomputable) so truncate is safe. +TRUNCATE solana_exec_raw_counts; +ALTER TABLE solana_exec_raw_counts + ADD COLUMN IF NOT EXISTS from_cursor TEXT NOT NULL DEFAULT ''; +ALTER TABLE solana_exec_raw_counts + DROP CONSTRAINT IF EXISTS solana_exec_raw_counts_pkey; +ALTER TABLE solana_exec_raw_counts + ADD PRIMARY KEY (platform, bucket_start, from_cursor); + +-- Per-account cursors: key was platform, now platform:feeAccount. +-- Existing single-account cursors (key = platform name) stay valid for pump.fun. +-- No migration needed; old key will not match new compound keys — polls restart from tip for new accounts. + +-- CU price samples for rolling percentile metrics (35-day retention). +CREATE TABLE IF NOT EXISTS solana_exec_cu_samples ( + platform TEXT NOT NULL, + sig TEXT NOT NULL, + block_time TIMESTAMPTZ NOT NULL, + cu_price_micro BIGINT NOT NULL, + PRIMARY KEY (platform, sig) +); + +CREATE INDEX IF NOT EXISTS solana_exec_cu_samples_bt + ON solana_exec_cu_samples (platform, block_time DESC); + +COMMIT;