diff --git a/.github/workflows/codex-push.yml b/.github/workflows/codex-push.yml new file mode 100644 index 00000000..bdefd9a0 --- /dev/null +++ b/.github/workflows/codex-push.yml @@ -0,0 +1,44 @@ +name: Codex token push + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + +jobs: + push-token: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: harnesses/aggregator-head-lag/go.mod + cache-dependency-path: harnesses/aggregator-head-lag/go.sum + + - name: Build scraper + working-directory: harnesses/aggregator-head-lag + run: go build -o /tmp/codex-scraper ./cmd/test-utls/ + + - name: Scrape and push token + run: | + set -euo pipefail + output=$(MODE=default /tmp/codex-scraper 2>&1) + echo "$output" | head -5 + token=$(echo "$output" | grep '^CODEX_TOKEN=' | cut -d= -f2-) + if [ -z "$token" ] || [ ${#token} -lt 100 ]; then + echo "ERROR: no token in output" + echo "$output" + exit 1 + fi + echo "Got token (len=${#token}), pushing to sidecar..." + http_code=$(curl -s -o /tmp/push_resp -w "%{http_code}" -X POST \ + -H "Content-Type: text/plain" \ + --data-raw "$token" \ + "http://57.130.19.92:8080/push") + if [ "$http_code" = "204" ]; then + echo "Push OK" + else + echo "Push failed (HTTP $http_code): $(cat /tmp/push_resp)" + exit 1 + fi diff --git a/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go b/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go index 6ec7cac7..41740bd4 100644 --- a/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go +++ b/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go @@ -103,9 +103,30 @@ func scrapeCodexToken() (string, error) { return "", fmt.Errorf("codex_token cookie not found after page load") } +// chromeAvailable returns true if a Chrome/Chromium binary is found on this host. +func chromeAvailable() bool { + for _, p := range []string{ + os.Getenv("CHROME_PATH"), + "/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + } { + if p != "" { + if _, err := os.Stat(p); err == nil { + return true + } + } + } + return false +} + // startInProcessScraper launches a background goroutine that refreshes the JWE every 5 min. -// Call once from main. Safe to call even if Chrome is not installed (logs error, no crash). +// Call once from main. No-ops silently if Chrome is not installed. func startInProcessScraper(stopChan <-chan struct{}) { + if !chromeAvailable() { + fmt.Println("[CODEX-SCRAPER] Chrome not found — in-process scraper disabled (sidecar will be used)") + return + } + go func() { // Initial delay: let the container fully start before launching Chrome. select { diff --git a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go index 987a1f13..c38dfd56 100644 --- a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go +++ b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go @@ -141,12 +141,20 @@ func GetDefinedJWTToken(sessionCookie string) (string, error) { fmt.Printf("[DEFINED-AUTH] Got token from in-process scraper (age=%v, len=%d)\n", time.Since(mintedAt).Round(time.Second), len(tok)) return tok, nil } - // Direct mint: JWE minted from this container's IP = same IP used for WS = no 4403. + // utls Chrome fingerprint scraper: visits defined.fi, gets CSRF, POSTs to /api/codex/token. + // Works if this container's IP is not in Vercel's datacenter blocklist. + if tok, err := tryUtlsCodexToken(); err == nil && tok != "" { + fmt.Printf("[DEFINED-AUTH] Got token via utls scraper (len=%d)\n", len(tok)) + return tok, nil + } else { + fmt.Printf("[DEFINED-AUTH] utls scraper failed: %v\n", err) + } + // Direct mint (standard Go TLS, usually blocked by Vercel bot check). if tok, err := tryDirectCodexToken(sessionCookie); err == nil && tok != "" { fmt.Printf("[DEFINED-AUTH] Got token via direct /api/codex/token (len=%d)\n", len(tok)) return tok, nil } - // Sidecar fallback (Paris box chromedp, auto-refreshes every 25 min — may be stale) + // Sidecar fallback (Paris box, Mac-push keeps it fresh every 5 min) if svcURL := os.Getenv("DEFINED_TOKEN_SERVICE_URL"); svcURL != "" { if tok, err := tryTokenService(svcURL); err == nil && tok != "" { fmt.Printf("[DEFINED-AUTH] Got token from sidecar (len=%d)\n", len(tok)) @@ -199,49 +207,39 @@ func GetDefinedJWTToken(sessionCookie string) (string, error) { return token, nil } -// generateDefinedJWTToken generates a new JWT token from Defined.fi session cookie -func generateDefinedJWTToken(sessionCookie string) (string, error) { - fmt.Println("[DEFINED-AUTH] Generating new JWT token from Defined.fi (local)...") - fmt.Println("[DEFINED-AUTH] Creating new HTTP client with fresh TCP connection (no keepalive)") - - // Create a new HTTP client with fresh connection for each request. - // CRITICAL: route through HTTP_PROXY/HTTPS_PROXY (webshare rotating proxy) - // so each JWT mint hits a fresh IP. Direct from the container IP gets us - // stuck on Vercel's bot ban (429 loop) when the container restarts often. - transport := &http.Transport{ - DisableKeepAlives: true, - MaxIdleConnsPerHost: 0, - Proxy: http.ProxyFromEnvironment, - } - client := &http.Client{ - Timeout: 10 * time.Second, - Transport: transport, - } +// generateDefinedJWTToken generates a new JWT token via un.defined.fi (old UI, still alive). +// No session cookie or CSRF needed. Falls back to www.defined.fi/api proxy path. +func generateDefinedJWTToken(_ string) (string, error) { + fmt.Println("[DEFINED-AUTH] Generating token via un.defined.fi legacy API...") reqBody := map[string]interface{}{ "operationName": "CreateApiToken", "query": "mutation CreateApiToken { createApiTokens(input: { count: 1 }) { token } }", "variables": map[string]interface{}{}, } - bodyBytes, _ := json.Marshal(reqBody) - req, _ := http.NewRequest("POST", "https://www.defined.fi/api", bytes.NewBuffer(bodyBytes)) + transport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + } + client := &http.Client{Timeout: 10 * time.Second, Transport: transport} + + req, _ := http.NewRequest("POST", "https://un.defined.fi/api", bytes.NewBuffer(bodyBytes)) req.Header.Set("Accept", "application/json") - req.Header.Set("Accept-Language", "en-US,en;q=0.9") req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://www.defined.fi") - req.Header.Set("Referer", "https://www.defined.fi/") req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Origin", "https://un.defined.fi") + req.Header.Set("Referer", "https://un.defined.fi/") req.Header.Set("sec-ch-ua", `"Not_A Brand";v="8", "Chromium";v="131", "Google Chrome";v="131"`) req.Header.Set("sec-ch-ua-mobile", "?0") req.Header.Set("sec-ch-ua-platform", `"macOS"`) req.Header.Set("sec-fetch-dest", "empty") req.Header.Set("sec-fetch-mode", "cors") req.Header.Set("sec-fetch-site", "same-origin") - req.AddCookie(&http.Cookie{Name: "defined-attestation-token", Value: sessionCookie}) - fmt.Println("[DEFINED-AUTH] Sending POST request to https://www.defined.fi/api...") + fmt.Println("[DEFINED-AUTH] POST https://un.defined.fi/api ...") resp, err := client.Do(req) if err != nil { fmt.Printf("[DEFINED-AUTH] ❌ Request failed: %v\n", err) @@ -253,18 +251,16 @@ func generateDefinedJWTToken(sessionCookie string) (string, error) { fmt.Printf("[DEFINED-AUTH] Response status: %d\n", resp.StatusCode) if resp.StatusCode == 429 { - // Parse retry-after header if available retryAfter := resp.Header.Get("Retry-After") fmt.Printf("[DEFINED-AUTH] ⚠ Rate limited! Retry-After: %s\n", retryAfter) - if retryAfter != "" { - return "", fmt.Errorf("rate limited (429), retry after: %s", retryAfter) - } - return "", fmt.Errorf("rate limited (429), too many token requests - will retry later") + return "", fmt.Errorf("rate limited (429)") } if resp.StatusCode != 200 { - fmt.Printf("[DEFINED-AUTH] ❌ Unexpected status %d: %s\n", resp.StatusCode, string(respBody[:min(len(respBody), 100)])) - return "", fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody[:min(len(respBody), 100)])) + n := len(respBody) + if n > 100 { n = 100 } + fmt.Printf("[DEFINED-AUTH] ❌ Unexpected status %d: %s\n", resp.StatusCode, string(respBody[:n])) + return "", fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody[:n])) } var tokenResp DefinedTokenResponse @@ -278,7 +274,7 @@ func generateDefinedJWTToken(sessionCookie string) (string, error) { return "", fmt.Errorf("no token returned") } - fmt.Printf("[DEFINED-AUTH] ✅ JWT token generated successfully (length: %d)\n", len(tokenResp.Data.CreateApiTokens[0].Token)) + fmt.Printf("[DEFINED-AUTH] ✅ Token generated via un.defined.fi (length: %d)\n", len(tokenResp.Data.CreateApiTokens[0].Token)) return tokenResp.Data.CreateApiTokens[0].Token, nil } diff --git a/harnesses/aggregator-head-lag/cmd/script/utls_codex.go b/harnesses/aggregator-head-lag/cmd/script/utls_codex.go new file mode 100644 index 00000000..a323bb64 --- /dev/null +++ b/harnesses/aggregator-head-lag/cmd/script/utls_codex.go @@ -0,0 +1,223 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "time" + + tls "github.com/refraction-networking/utls" +) + +func chromeH1Spec() tls.ClientHelloSpec { + return tls.ClientHelloSpec{ + TLSVersMax: tls.VersionTLS13, + TLSVersMin: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.GREASE_PLACEHOLDER, + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + }, + CompressionMethods: []byte{0x00}, + Extensions: tls.ShuffleChromeTLSExtensions([]tls.TLSExtension{ + &tls.UtlsGREASEExtension{}, + &tls.SNIExtension{}, + &tls.ExtendedMasterSecretExtension{}, + &tls.RenegotiationInfoExtension{Renegotiation: tls.RenegotiateOnceAsClient}, + &tls.SupportedCurvesExtension{[]tls.CurveID{ + tls.GREASE_PLACEHOLDER, tls.X25519, tls.CurveP256, tls.CurveP384, + }}, + &tls.SupportedPointsExtension{SupportedPoints: []byte{0x00}}, + &tls.SessionTicketExtension{}, + &tls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}, + &tls.StatusRequestExtension{}, + &tls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []tls.SignatureScheme{ + tls.ECDSAWithP256AndSHA256, tls.PSSWithSHA256, tls.PKCS1WithSHA256, + tls.ECDSAWithP384AndSHA384, tls.PSSWithSHA384, tls.PKCS1WithSHA384, + tls.PSSWithSHA512, tls.PKCS1WithSHA512, + }}, + &tls.SCTExtension{}, + &tls.KeyShareExtension{[]tls.KeyShare{ + {Group: tls.CurveID(tls.GREASE_PLACEHOLDER), Data: []byte{0}}, + {Group: tls.X25519}, + }}, + &tls.PSKKeyExchangeModesExtension{[]uint8{tls.PskModeDHE}}, + &tls.SupportedVersionsExtension{[]uint16{ + tls.GREASE_PLACEHOLDER, tls.VersionTLS13, tls.VersionTLS12, + }}, + &tls.UtlsCompressCertExtension{[]tls.CertCompressionAlgo{tls.CertCompressionBrotli}}, + &tls.UtlsGREASEExtension{}, + &tls.UtlsPaddingExtension{GetPaddingLen: tls.BoringPaddingStyle}, + }), + } +} + +func newUTLSClient() *http.Client { + jar, _ := cookiejar.New(nil) + return &http.Client{ + Timeout: 30 * time.Second, + Jar: jar, + Transport: &http.Transport{ + DisableKeepAlives: true, + ForceAttemptHTTP2: false, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, _ := net.SplitHostPort(addr) + conn, err := (&net.Dialer{Timeout: 15 * time.Second}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + spec := chromeH1Spec() + uc := tls.UClient(conn, &tls.Config{ServerName: host}, tls.HelloCustom) + if err := uc.ApplyPreset(&spec); err != nil { + conn.Close() + return nil, err + } + if err := uc.HandshakeContext(ctx); err != nil { + conn.Close() + return nil, err + } + return uc, nil + }, + }, + } +} + +// tryUtlsLegacyAPI calls un.defined.fi/api (old UI, still alive) with createApiTokens mutation. +// No cookies or CSRF needed — one request, much simpler. Works from residential IPs. +// Try this first since it's a single POST with no page-visit prerequisite. +func tryUtlsLegacyAPI() (string, error) { + client := newUTLSClient() + + body, _ := json.Marshal(map[string]interface{}{ + "operationName": "CreateApiToken", + "query": "mutation CreateApiToken { createApiTokens(input: { count: 1 }) { token } }", + "variables": map[string]interface{}{}, + }) + req, _ := http.NewRequest("POST", "https://un.defined.fi/api", bytes.NewBuffer(body)) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Origin", "https://un.defined.fi") + req.Header.Set("Referer", "https://un.defined.fi/") + req.Header.Set("Sec-Fetch-Site", "same-origin") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Dest", "empty") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + respBody, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP %d: %.100s", resp.StatusCode, string(respBody)) + } + + var parsed struct { + Data struct { + CreateApiTokens []struct { + Token string `json:"token"` + } `json:"createApiTokens"` + } `json:"data"` + } + if err := json.Unmarshal(respBody, &parsed); err != nil || len(parsed.Data.CreateApiTokens) == 0 || parsed.Data.CreateApiTokens[0].Token == "" { + return "", fmt.Errorf("no token in response: %.100s", string(respBody)) + } + return parsed.Data.CreateApiTokens[0].Token, nil +} + +// tryUtlsCodexToken uses Chrome TLS fingerprint spoofing to call www.defined.fi/api/codex/token. +// Requires a page visit first to get CSRF cookie. Fallback if tryUtlsLegacyAPI fails. +func tryUtlsCodexToken() (string, error) { + // Try the old un.defined.fi API first — no cookies or CSRF needed. + if tok, err := tryUtlsLegacyAPI(); err == nil && tok != "" { + return tok, nil + } + + client := newUTLSClient() + + req1, _ := http.NewRequest("GET", "https://www.defined.fi/", nil) + req1.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + req1.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req1.Header.Set("Accept-Language", "en-US,en;q=0.9") + req1.Header.Set("Sec-Fetch-Site", "none") + req1.Header.Set("Sec-Fetch-Mode", "navigate") + req1.Header.Set("Sec-Fetch-Dest", "document") + req1.Header.Set("Upgrade-Insecure-Requests", "1") + + resp1, err := client.Do(req1) + if err != nil { + return "", fmt.Errorf("page load failed: %w", err) + } + io.Copy(io.Discard, resp1.Body) + resp1.Body.Close() + + if resp1.StatusCode != 200 { + return "", fmt.Errorf("page load blocked (HTTP %d) — IP blocked by Vercel", resp1.StatusCode) + } + + u, _ := url.Parse("https://www.defined.fi/") + cookies := client.Jar.Cookies(u) + var csrfToken string + for _, c := range cookies { + if c.Name == "csrf-token" { + csrfToken = c.Value + } + } + if csrfToken == "" { + return "", fmt.Errorf("no csrf-token in page cookies") + } + + req2, _ := http.NewRequest("POST", "https://www.defined.fi/api/codex/token", bytes.NewBufferString("{}")) + req2.Header.Set("Accept", "application/json") + req2.Header.Set("Content-Type", "application/json") + req2.Header.Set("x-csrf-token", csrfToken) + req2.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req2.Header.Set("Accept-Language", "en-US,en;q=0.9") + req2.Header.Set("Origin", "https://www.defined.fi") + req2.Header.Set("Referer", "https://www.defined.fi/") + req2.Header.Set("Sec-Fetch-Site", "same-origin") + req2.Header.Set("Sec-Fetch-Mode", "cors") + req2.Header.Set("Sec-Fetch-Dest", "empty") + + resp2, err := client.Do(req2) + if err != nil { + return "", fmt.Errorf("codex/token request failed: %w", err) + } + body2, _ := io.ReadAll(resp2.Body) + resp2.Body.Close() + + if resp2.StatusCode != 200 { + return "", fmt.Errorf("codex/token HTTP %d: %.100s", resp2.StatusCode, string(body2)) + } + + var parsed struct { + Token string `json:"token"` + } + if err := json.Unmarshal(body2, &parsed); err != nil || parsed.Token == "" { + return "", fmt.Errorf("no token in response: %.100s", string(body2)) + } + + return parsed.Token, nil +} diff --git a/harnesses/aggregator-head-lag/cmd/test-utls/main.go b/harnesses/aggregator-head-lag/cmd/test-utls/main.go new file mode 100644 index 00000000..cbb8adbc --- /dev/null +++ b/harnesses/aggregator-head-lag/cmd/test-utls/main.go @@ -0,0 +1,261 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "time" + + tls "github.com/refraction-networking/utls" +) + +func chromeH1Spec() tls.ClientHelloSpec { + return tls.ClientHelloSpec{ + TLSVersMax: tls.VersionTLS13, + TLSVersMin: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.GREASE_PLACEHOLDER, + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + }, + CompressionMethods: []byte{0x00}, + Extensions: tls.ShuffleChromeTLSExtensions([]tls.TLSExtension{ + &tls.UtlsGREASEExtension{}, + &tls.SNIExtension{}, + &tls.ExtendedMasterSecretExtension{}, + &tls.RenegotiationInfoExtension{Renegotiation: tls.RenegotiateOnceAsClient}, + &tls.SupportedCurvesExtension{[]tls.CurveID{ + tls.GREASE_PLACEHOLDER, tls.X25519, tls.CurveP256, tls.CurveP384, + }}, + &tls.SupportedPointsExtension{SupportedPoints: []byte{0x00}}, + &tls.SessionTicketExtension{}, + &tls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}, + &tls.StatusRequestExtension{}, + &tls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []tls.SignatureScheme{ + tls.ECDSAWithP256AndSHA256, tls.PSSWithSHA256, tls.PKCS1WithSHA256, + tls.ECDSAWithP384AndSHA384, tls.PSSWithSHA384, tls.PKCS1WithSHA384, + tls.PSSWithSHA512, tls.PKCS1WithSHA512, + }}, + &tls.SCTExtension{}, + &tls.KeyShareExtension{[]tls.KeyShare{ + {Group: tls.CurveID(tls.GREASE_PLACEHOLDER), Data: []byte{0}}, + {Group: tls.X25519}, + }}, + &tls.PSKKeyExchangeModesExtension{[]uint8{tls.PskModeDHE}}, + &tls.SupportedVersionsExtension{[]uint16{ + tls.GREASE_PLACEHOLDER, tls.VersionTLS13, tls.VersionTLS12, + }}, + &tls.UtlsCompressCertExtension{[]tls.CertCompressionAlgo{tls.CertCompressionBrotli}}, + &tls.UtlsGREASEExtension{}, + &tls.UtlsPaddingExtension{GetPaddingLen: tls.BoringPaddingStyle}, + }), + } +} + +func newUTLSClient() *http.Client { + jar, _ := cookiejar.New(nil) + return &http.Client{ + Timeout: 30 * time.Second, + Jar: jar, + Transport: &http.Transport{ + DisableKeepAlives: true, + ForceAttemptHTTP2: false, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, _ := net.SplitHostPort(addr) + conn, err := (&net.Dialer{Timeout: 15 * time.Second}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + spec := chromeH1Spec() + uc := tls.UClient(conn, &tls.Config{ServerName: host}, tls.HelloCustom) + if err := uc.ApplyPreset(&spec); err != nil { + conn.Close() + return nil, err + } + if err := uc.HandshakeContext(ctx); err != nil { + conn.Close() + return nil, err + } + return uc, nil + }, + }, + } +} + +func main() { + mode := os.Getenv("MODE") + + switch mode { + case "fetch-html": + // Fetch URL via utls, print body to stdout + target := os.Getenv("URL") + if target == "" { + target = "https://www.defined.fi/" + } + client := newUTLSClient() + req, _ := http.NewRequest("GET", target, nil) + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + resp, err := client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(1) + } + defer resp.Body.Close() + fmt.Fprintf(os.Stderr, "HTTP %d\n", resp.StatusCode) + io.Copy(os.Stdout, resp.Body) + + case "scrape": + // Step 1 only: GET defined.fi, print cookies + client := newUTLSClient() + req, _ := http.NewRequest("GET", "https://www.defined.fi/", nil) + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Sec-Fetch-Site", "none") + req.Header.Set("Sec-Fetch-Mode", "navigate") + req.Header.Set("Sec-Fetch-Dest", "document") + req.Header.Set("Upgrade-Insecure-Requests", "1") + resp, err := client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(1) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + fmt.Fprintf(os.Stderr, "ERROR: HTTP %d\n", resp.StatusCode) + os.Exit(1) + } + u, _ := url.Parse("https://www.defined.fi/") + for _, c := range client.Jar.Cookies(u) { + fmt.Printf("%s=%s\n", c.Name, c.Value) + } + + case "mint-utls": + // Step 2 only via utls: POST /api/codex/token with pre-obtained cookies + attestation := os.Getenv("ATTESTATION") + csrf := os.Getenv("CSRF") + if attestation == "" || csrf == "" { + fmt.Fprintln(os.Stderr, "Need ATTESTATION and CSRF env vars") + os.Exit(1) + } + fmt.Printf("Calling /api/codex/token via utls (Chrome fingerprint)...\n") + client := newUTLSClient() + // Inject cookies into jar + u, _ := url.Parse("https://www.defined.fi/") + client.Jar.SetCookies(u, []*http.Cookie{ + {Name: "defined-attestation-token", Value: attestation}, + {Name: "csrf-token", Value: csrf}, + }) + req, _ := http.NewRequest("POST", "https://www.defined.fi/api/codex/token", bytes.NewBufferString("{}")) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-csrf-token", csrf) + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Origin", "https://www.defined.fi") + req.Header.Set("Referer", "https://www.defined.fi/") + req.Header.Set("Sec-Fetch-Site", "same-origin") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Dest", "empty") + resp, err := client.Do(req) + if err != nil { + fmt.Printf("ERROR: %v\n", err) + os.Exit(1) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + fmt.Printf("HTTP %d\n", resp.StatusCode) + if resp.StatusCode == 200 { + var parsed struct{ Token string `json:"token"` } + if err := json.Unmarshal(body, &parsed); err == nil && parsed.Token != "" { + fmt.Printf("✅ Got JWE (len=%d)\n", len(parsed.Token)) + fmt.Printf("CODEX_TOKEN=%s\n", parsed.Token) + return + } + } + n := len(body) + if n > 200 { + n = 200 + } + fmt.Printf("❌ Body: %s\n", string(body[:n])) + os.Exit(1) + + default: + // Full flow: scrape + mint on this machine + client := newUTLSClient() + req1, _ := http.NewRequest("GET", "https://www.defined.fi/", nil) + req1.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + req1.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req1.Header.Set("Accept-Language", "en-US,en;q=0.9") + req1.Header.Set("Sec-Fetch-Site", "none") + req1.Header.Set("Sec-Fetch-Mode", "navigate") + req1.Header.Set("Sec-Fetch-Dest", "document") + req1.Header.Set("Upgrade-Insecure-Requests", "1") + resp1, err := client.Do(req1) + if err != nil { + fmt.Printf("Page load failed: %v\n", err) + return + } + io.Copy(io.Discard, resp1.Body) + resp1.Body.Close() + fmt.Printf("Page: HTTP %d\n", resp1.StatusCode) + u, _ := url.Parse("https://www.defined.fi/") + cookies := client.Jar.Cookies(u) + var attestation, csrf string + for _, c := range cookies { + switch c.Name { + case "defined-attestation-token": + attestation = c.Value + case "csrf-token": + csrf = c.Value + } + } + fmt.Printf("attestation len=%d, csrf len=%d\n", len(attestation), len(csrf)) + req2, _ := http.NewRequest("POST", "https://www.defined.fi/api/codex/token", bytes.NewBufferString("{}")) + req2.Header.Set("Accept", "application/json") + req2.Header.Set("Content-Type", "application/json") + req2.Header.Set("x-csrf-token", csrf) + req2.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + req2.Header.Set("Origin", "https://www.defined.fi") + req2.Header.Set("Referer", "https://www.defined.fi/") + req2.Header.Set("Sec-Fetch-Site", "same-origin") + req2.Header.Set("Sec-Fetch-Mode", "cors") + resp2, err := client.Do(req2) + if err != nil { + fmt.Printf("API failed: %v\n", err) + return + } + body, _ := io.ReadAll(resp2.Body) + resp2.Body.Close() + fmt.Printf("API: HTTP %d\n", resp2.StatusCode) + if resp2.StatusCode == 200 { + var parsed struct{ Token string `json:"token"` } + if err := json.Unmarshal(body, &parsed); err == nil && parsed.Token != "" { + fmt.Printf("✅ JWE len=%d\n", len(parsed.Token)) + } + } + } +} diff --git a/harnesses/aggregator-head-lag/go.mod b/harnesses/aggregator-head-lag/go.mod index ed17b543..d6ccd878 100644 --- a/harnesses/aggregator-head-lag/go.mod +++ b/harnesses/aggregator-head-lag/go.mod @@ -10,6 +10,7 @@ require ( ) require ( + github.com/andybalholm/brotli v1.0.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect @@ -17,12 +18,15 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/sys v0.35.0 // indirect google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/harnesses/aggregator-head-lag/go.sum b/harnesses/aggregator-head-lag/go.sum index f594bc89..6379815d 100644 --- a/harnesses/aggregator-head-lag/go.sum +++ b/harnesses/aggregator-head-lag/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -47,6 +49,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -55,6 +59,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=